Repository: incubator-atlas Updated Branches: refs/heads/master 7cc347134 -> d64112d6a
ATLAS-957 Atlas is not capturing topologies that have $ in the data payload (shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/d64112d6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/d64112d6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/d64112d6 Branch: refs/heads/master Commit: d64112d6a95e6279fd54f2164e89857124450b67 Parents: 7cc3471 Author: Shwetha GS <[email protected]> Authored: Tue Jul 19 11:51:28 2016 +0530 Committer: Shwetha GS <[email protected]> Committed: Tue Jul 19 11:51:28 2016 +0530 ---------------------------------------------------------------------- release-log.txt | 1 + .../graph/DefaultGraphPersistenceStrategy.java | 6 +- .../graph/GraphBackedDiscoveryService.java | 5 +- .../atlas/repository/graph/DeleteHandler.java | 8 +- .../atlas/repository/graph/FullTextMapper.java | 3 +- .../graph/GraphBackedMetadataRepository.java | 6 +- .../graph/GraphBackedSearchIndexer.java | 2 +- .../atlas/repository/graph/GraphHelper.java | 73 +++++++- .../graph/GraphToTypedInstanceMapper.java | 58 +++--- .../graph/TypedInstanceToGraphMapper.java | 10 +- .../typestore/GraphBackedTypeStore.java | 36 ++-- .../org/apache/atlas/query/GremlinQuery.scala | 33 ++-- .../atlas/repository/graph/GraphHelperTest.java | 51 ++++++ .../service/DefaultMetadataServiceTest.java | 175 ++++++++++++------- 14 files changed, 312 insertions(+), 155 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d64112d6/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 3b2338e..965cafe 100644 --- a/release-log.txt +++ b/release-log.txt @@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES: ALL CHANGES: +ATLAS-957 Atlas is not capturing topologies that have $ in the data payload (shwethags) ATLAS-1032 Atlas hook package should not include libraries already present in host component - like log4j (mneethiraj via sumasai) ATLAS-1027 Atlas hooks should use properties from atlas-application.properties, instead of component's configuration (mneethiraj via sumasai) ATLAS-1030 Add instrumentation to measure performance: REST API (mneethiraj via sumasai) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d64112d6/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java b/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java index 8ddbe1b..e07a54e 100755 --- a/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java +++ b/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java @@ -140,9 +140,9 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi TypeSystem.IdType idType = TypeSystem.getInstance().getIdType(); if (dataType.getName().equals(idType.getName())) { - structInstance.set(idType.typeNameAttrName(), structVertex.getProperty(typeAttributeName())); - structInstance.set(idType.idAttrName(), structVertex.getProperty(idAttributeName())); - structInstance.set(idType.stateAttrName(), structVertex.getProperty(stateAttributeName())); + structInstance.set(idType.typeNameAttrName(), GraphHelper.getProperty(structVertex, typeAttributeName())); + structInstance.set(idType.idAttrName(), GraphHelper.getProperty(structVertex, idAttributeName())); + structInstance.set(idType.stateAttrName(), GraphHelper.getProperty(structVertex, stateAttributeName())); } else { metadataRepository.getGraphToInstanceMapper() .mapVertexToInstance(structVertex, structInstance, structType.fieldMapping().fields); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d64112d6/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java index f97b83d..6def78b 100755 --- a/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java @@ -36,6 +36,7 @@ import org.apache.atlas.query.QueryParser; import org.apache.atlas.query.QueryProcessor; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.MetadataRepository; +import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graph.GraphProvider; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; @@ -94,11 +95,11 @@ public class GraphBackedDiscoveryService implements DiscoveryService { Vertex vertex = result.getElement(); JSONObject row = new JSONObject(); - String guid = vertex.getProperty(Constants.GUID_PROPERTY_KEY); + String guid = GraphHelper.getIdFromVertex(vertex); if (guid != null) { //Filter non-class entities try { row.put("guid", guid); - row.put(AtlasClient.TYPENAME, vertex.<String>getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY)); + row.put(AtlasClient.TYPENAME, GraphHelper.getTypeName(vertex)); row.put(SCORE, result.getScore()); } catch (JSONException e) { LOG.error("Unable to create response", e); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d64112d6/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java b/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java index 91f9bd0..e74e57f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java @@ -154,7 +154,7 @@ public abstract class DeleteHandler { if (valueTypeCategory == DataTypes.TypeCategory.STRUCT || valueTypeCategory == DataTypes.TypeCategory.CLASS) { - List<String> keys = instanceVertex.getProperty(propertyName); + List<String> keys = GraphHelper.getProperty(instanceVertex, propertyName); if (keys != null) { for (String key : keys) { String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, key); @@ -286,7 +286,7 @@ public abstract class DeleteHandler { case ARRAY: //If its array attribute, find the right edge between the two vertices and update array property - List<String> elements = outVertex.getProperty(propertyName); + List<String> elements = GraphHelper.getProperty(outVertex, propertyName); if (elements != null) { elements = new ArrayList<>(elements); //Make a copy, else list.remove reflects on titan.getProperty() for (String elementEdgeId : elements) { @@ -327,12 +327,12 @@ public abstract class DeleteHandler { case MAP: //If its map attribute, find the right edge between two vertices and update map property - List<String> keys = outVertex.getProperty(propertyName); + List<String> keys = GraphHelper.getProperty(outVertex, propertyName); if (keys != null) { keys = new ArrayList<>(keys); //Make a copy, else list.remove reflects on titan.getProperty() for (String key : keys) { String keyPropertyName = GraphHelper.getQualifiedNameForMapKey(propertyName, key); - String mapEdgeId = outVertex.getProperty(keyPropertyName); + String mapEdgeId = GraphHelper.getProperty(outVertex, keyPropertyName); Edge mapEdge = graphHelper.getEdgeByEdgeId(outVertex, keyPropertyName, mapEdgeId); Vertex mapVertex = mapEdge.getVertex(Direction.IN); if (mapVertex.getId().toString().equals(inVertex.getId().toString())) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d64112d6/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java index 37cacb0..b342e27 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java @@ -19,7 +19,6 @@ package org.apache.atlas.repository.graph; import com.tinkerpop.blueprints.Vertex; import org.apache.atlas.AtlasException; -import org.apache.atlas.repository.Constants; import org.apache.atlas.typesystem.ITypedInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.types.AttributeInfo; @@ -51,7 +50,7 @@ public class FullTextMapper { } public String mapRecursive(Vertex instanceVertex, boolean followReferences) throws AtlasException { - String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY); + String guid = GraphHelper.getIdFromVertex(instanceVertex); ITypedReferenceableInstance typedReference; if (instanceCache.containsKey(guid)) { typedReference = instanceCache.get(guid); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d64112d6/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java index e5c0e46..73445cb 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java @@ -115,7 +115,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { if (aInfo.name.startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX)) { return aInfo.name; } - return GraphHelper.getQualifiedFieldName(dataType, aInfo.name); + return GraphHelper.encodePropertyKey(GraphHelper.getQualifiedFieldName(dataType, aInfo.name)); } public String getFieldNameInVertex(IDataType<?> dataType, String attrName) throws AtlasException { @@ -168,7 +168,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { Constants.ENTITY_TYPE_PROPERTY_KEY, entityType, Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name()); - String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY); + String guid = GraphHelper.getIdFromVertex(instanceVertex); return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex); } @@ -185,7 +185,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ArrayList<String> entityList = new ArrayList<>(); while (results.hasNext()) { Vertex vertex = results.next(); - entityList.add(vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY)); + entityList.add(GraphHelper.getIdFromVertex(vertex)); } return entityList; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d64112d6/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index 8c8134f..e960c2f 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -227,7 +227,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang } private void createIndexForAttribute(TitanManagement management, String typeName, AttributeInfo field) { - final String propertyName = typeName + "." + field.name; + final String propertyName = GraphHelper.encodePropertyKey(typeName + "." + field.name); switch (field.dataType().getTypeCategory()) { case PRIMITIVE: Cardinality cardinality = getCardinality(field.multiplicity); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d64112d6/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 4f531e2..b3711b5 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -18,6 +18,9 @@ package org.apache.atlas.repository.graph; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.TitanProperty; import com.thinkaurelius.titan.core.TitanVertex; @@ -41,11 +44,13 @@ import org.apache.atlas.typesystem.types.DataTypes; import org.apache.atlas.typesystem.types.HierarchicalType; import org.apache.atlas.typesystem.types.IDataType; import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -215,7 +220,7 @@ public final class GraphHelper { LOG.debug("Found {}", string(edge)); return edge; } else { - Long modificationTime = edge.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); + Long modificationTime = getProperty(edge, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); if (modificationTime != null && modificationTime >= latestDeletedEdgeTime) { latestDeletedEdgeTime = modificationTime; latestDeletedEdge = edge; @@ -244,21 +249,36 @@ public final class GraphHelper { public static <T extends Element> void setProperty(T element, String propertyName, Object value) { String elementStr = string(element); - LOG.debug("Setting property {} = \"{}\" to {}", propertyName, value, elementStr); - Object existValue = element.getProperty(propertyName); + String actualPropertyName = GraphHelper.encodePropertyKey(propertyName); + LOG.debug("Setting property {} = \"{}\" to {}", actualPropertyName, value, elementStr); + Object existValue = element.getProperty(actualPropertyName); if(value == null || (value instanceof Collection && ((Collection) value).isEmpty())) { if(existValue != null) { - LOG.info("Removing property - {} value from {}", propertyName, elementStr); - element.removeProperty(propertyName); + LOG.info("Removing property - {} value from {}", actualPropertyName, elementStr); + element.removeProperty(actualPropertyName); } } else { if (!value.equals(existValue)) { - element.setProperty(propertyName, value); - LOG.debug("Set property {} = \"{}\" to {}", propertyName, value, elementStr); + element.setProperty(actualPropertyName, value); + LOG.debug("Set property {} = \"{}\" to {}", actualPropertyName, value, elementStr); } } } + public static <T extends Element, O> O getProperty(T element, String propertyName) { + String elementStr = string(element); + String actualPropertyName = GraphHelper.encodePropertyKey(propertyName); + LOG.debug("Reading property {} from {}", actualPropertyName, elementStr); + return element.getProperty(actualPropertyName); + } + + public static Iterable<TitanProperty> getProperties(TitanVertex vertex, String propertyName) { + String elementStr = string(vertex); + String actualPropertyName = GraphHelper.encodePropertyKey(propertyName); + LOG.debug("Reading property {} from {}", actualPropertyName, elementStr); + return vertex.getProperties(actualPropertyName); + } + private static <T extends Element> String string(T element) { if (element instanceof Vertex) { return string((Vertex) element); @@ -339,8 +359,8 @@ public final class GraphHelper { } public static Id getIdFromVertex(String dataTypeName, Vertex vertex) { - return new Id(vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY), - vertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), dataTypeName); + return new Id(getIdFromVertex(vertex), + vertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), dataTypeName, getStateAsString(vertex)); } public static String getIdFromVertex(Vertex vertex) { @@ -425,4 +445,39 @@ public final class GraphHelper { return String.format("edge[id=%s]", edge.getId().toString()); } } + + @VisibleForTesting + //Keys copied from com.thinkaurelius.titan.graphdb.types.StandardRelationTypeMaker + //Titan checks that these chars are not part of any keys. So, encoding... + public static BiMap<String, String> RESERVED_CHARS_ENCODE_MAP = + HashBiMap.create(new HashMap<String, String>() {{ + put("{", "_o"); + put("}", "_c"); + put("\"", "_q"); + put("$", "_d"); + put("%", "_p"); + }}); + + + public static String encodePropertyKey(String key) { + if (StringUtils.isBlank(key)) { + return key; + } + + for (String str : RESERVED_CHARS_ENCODE_MAP.keySet()) { + key = key.replace(str, RESERVED_CHARS_ENCODE_MAP.get(str)); + } + return key; + } + + public static String decodePropertyKey(String key) { + if (StringUtils.isBlank(key)) { + return key; + } + + for (String encodedStr : RESERVED_CHARS_ENCODE_MAP.values()) { + key = key.replace(encodedStr, RESERVED_CHARS_ENCODE_MAP.inverse().get(encodedStr)); + } + return key; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d64112d6/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java index e240fb6..69223f5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java @@ -47,6 +47,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.apache.atlas.repository.graph.GraphHelper.getIdFromVertex; import static org.apache.atlas.repository.graph.GraphHelper.string; @Singleton @@ -66,11 +67,12 @@ public final class GraphToTypedInstanceMapper { throws AtlasException { LOG.debug("Mapping graph root vertex {} to typed instance for guid {}", instanceVertex, guid); - String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY); + String typeName = GraphHelper.getProperty(instanceVertex, Constants.ENTITY_TYPE_PROPERTY_KEY); List<String> traits = GraphHelper.getTraitNames(instanceVertex); String state = GraphHelper.getStateAsString(instanceVertex); - Id id = new Id(guid, instanceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), typeName, state); + Id id = new Id(guid, (Integer) GraphHelper.getProperty(instanceVertex, Constants.VERSION_PROPERTY_KEY), + typeName, state); LOG.debug("Created id {} for instance type {}", id, typeName); ClassType classType = typeSystem.getDataType(ClassType.class, typeName); @@ -115,13 +117,12 @@ public final class GraphToTypedInstanceMapper { break; // add only if vertex has this attribute case ENUM: - if (instanceVertex.getProperty(vertexPropertyName) == null) { + Object propertyValue = GraphHelper.getProperty(instanceVertex, vertexPropertyName); + if (propertyValue == null) { return; } - typedInstance.set(attributeInfo.name, - dataType.convert(instanceVertex.<String>getProperty(vertexPropertyName), - Multiplicity.REQUIRED)); + typedInstance.set(attributeInfo.name, dataType.convert(propertyValue, Multiplicity.REQUIRED)); break; case ARRAY: @@ -168,17 +169,14 @@ public final class GraphToTypedInstanceMapper { if (edge != null) { final Vertex referenceVertex = edge.getVertex(Direction.IN); - final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY); + final String guid = GraphHelper.getIdFromVertex(referenceVertex); LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel, guid); if (attributeInfo.isComposite) { //Also, when you retrieve a type's instance, you get the complete object graph of the composites LOG.debug("Found composite, mapping vertex to instance"); return mapGraphToTypedInstance(guid, referenceVertex); } else { - String state = GraphHelper.getStateAsString(referenceVertex); - Id referenceId = - new Id(guid, referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), - dataType.getName(), state); + Id referenceId = getIdFromVertex(dataType.getName(), referenceVertex); LOG.debug("Found non-composite, adding id {} ", referenceId); return referenceId; } @@ -191,7 +189,7 @@ public final class GraphToTypedInstanceMapper { private void mapVertexToArrayInstance(Vertex instanceVertex, ITypedInstance typedInstance, AttributeInfo attributeInfo, String propertyName) throws AtlasException { LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name); - List list = instanceVertex.getProperty(propertyName); + List list = GraphHelper.getProperty(instanceVertex, propertyName); if (list == null || list.size() == 0) { return; } @@ -240,7 +238,7 @@ public final class GraphToTypedInstanceMapper { private void mapVertexToMapInstance(Vertex instanceVertex, ITypedInstance typedInstance, AttributeInfo attributeInfo, final String propertyName) throws AtlasException { LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name); - List<String> keys = instanceVertex.getProperty(propertyName); + List<String> keys = GraphHelper.getProperty(instanceVertex, propertyName); if (keys == null || keys.size() == 0) { return; } @@ -251,7 +249,7 @@ public final class GraphToTypedInstanceMapper { for (String key : keys) { final String keyPropertyName = propertyName + "." + key; final String edgeLabel = GraphHelper.EDGE_LABEL_PREFIX + keyPropertyName; - final Object keyValue = instanceVertex.getProperty(keyPropertyName); + final Object keyValue = GraphHelper.getProperty(instanceVertex, keyPropertyName); Object mapValue = mapVertexToCollectionEntry(instanceVertex, attributeInfo, valueType, keyValue, edgeLabel); if (mapValue != null) { values.put(key, mapValue); @@ -312,33 +310,33 @@ public final class GraphToTypedInstanceMapper { AttributeInfo attributeInfo) throws AtlasException { LOG.debug("Adding primitive {} from vertex {}", attributeInfo, instanceVertex); final String vertexPropertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo); - if (instanceVertex.getProperty(vertexPropertyName) == null) { + Object propertyValue = GraphHelper.getProperty(instanceVertex, vertexPropertyName); + if (propertyValue == null) { return; } if (attributeInfo.dataType() == DataTypes.STRING_TYPE) { - typedInstance.setString(attributeInfo.name, instanceVertex.<String>getProperty(vertexPropertyName)); + typedInstance.setString(attributeInfo.name, (String) propertyValue); } else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) { - typedInstance.setShort(attributeInfo.name, instanceVertex.<Short>getProperty(vertexPropertyName)); + typedInstance.setShort(attributeInfo.name, (Short) propertyValue); } else if (attributeInfo.dataType() == DataTypes.INT_TYPE) { - typedInstance.setInt(attributeInfo.name, instanceVertex.<Integer>getProperty(vertexPropertyName)); + typedInstance.setInt(attributeInfo.name, (Integer) propertyValue); } else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) { - typedInstance.setBigInt(attributeInfo.name, instanceVertex.<BigInteger>getProperty(vertexPropertyName)); + typedInstance.setBigInt(attributeInfo.name, (BigInteger) propertyValue); } else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) { - typedInstance.setBoolean(attributeInfo.name, instanceVertex.<Boolean>getProperty(vertexPropertyName)); + typedInstance.setBoolean(attributeInfo.name, (Boolean) propertyValue); } else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) { - typedInstance.setByte(attributeInfo.name, instanceVertex.<Byte>getProperty(vertexPropertyName)); + typedInstance.setByte(attributeInfo.name, (Byte) propertyValue); } else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) { - typedInstance.setLong(attributeInfo.name, instanceVertex.<Long>getProperty(vertexPropertyName)); + typedInstance.setLong(attributeInfo.name, (Long) propertyValue); } else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) { - typedInstance.setFloat(attributeInfo.name, instanceVertex.<Float>getProperty(vertexPropertyName)); + typedInstance.setFloat(attributeInfo.name, (Float) propertyValue); } else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) { - typedInstance.setDouble(attributeInfo.name, instanceVertex.<Double>getProperty(vertexPropertyName)); + typedInstance.setDouble(attributeInfo.name, (Double) propertyValue); } else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) { - typedInstance - .setBigDecimal(attributeInfo.name, instanceVertex.<BigDecimal>getProperty(vertexPropertyName)); + typedInstance.setBigDecimal(attributeInfo.name, (BigDecimal) propertyValue); } else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) { - final Long dateVal = instanceVertex.<Long>getProperty(vertexPropertyName); + final Long dateVal = (Long) propertyValue; typedInstance.setDate(attributeInfo.name, new Date(dateVal)); } } @@ -359,11 +357,7 @@ public final class GraphToTypedInstanceMapper { return instance; case CLASS: //TODO isComposite handling for class loads - final String guid = referredVertex.getProperty(Constants.GUID_PROPERTY_KEY); - Id referenceId = - new Id(guid, referredVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), - referredType.getName()); - return referenceId; + return GraphHelper.getIdFromVertex(referredType.getName(), referredVertex); default: throw new UnsupportedOperationException("Loading " + referredType.getTypeCategory() + " is not supported"); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d64112d6/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java index 4c1f559..f4f9ec1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java @@ -324,7 +324,7 @@ public final class TypedInstanceToGraphMapper { } String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo); - List<String> currentElements = instanceVertex.getProperty(propertyName); + List<String> currentElements = GraphHelper.getProperty(instanceVertex, propertyName); IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType(); List<Object> newElementsCreated = new ArrayList<>(); @@ -403,11 +403,11 @@ public final class TypedInstanceToGraphMapper { Map<String, String> currentMap = new HashMap<>(); Map<String, Object> newMap = new HashMap<>(); - List<String> currentKeys = instanceVertex.getProperty(propertyName); + List<String> currentKeys = GraphHelper.getProperty(instanceVertex, propertyName); if (currentKeys != null && !currentKeys.isEmpty()) { for (String key : currentKeys) { String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(propertyName, key); - String propertyValueForKey = instanceVertex.getProperty(propertyNameForKey).toString(); + String propertyValueForKey = GraphHelper.getProperty(instanceVertex, propertyNameForKey).toString(); currentMap.put(key, propertyValueForKey); } } @@ -562,7 +562,7 @@ public final class TypedInstanceToGraphMapper { // Update attributes final MessageDigest digester = MD5Utils.getDigester(); String newSignature = newAttributeValue.getSignatureHash(digester); - String curSignature = structInstanceVertex.getProperty(SIGNATURE_HASH_PROPERTY_KEY); + String curSignature = GraphHelper.getProperty(structInstanceVertex, SIGNATURE_HASH_PROPERTY_KEY); if (!newSignature.equals(curSignature)) { //Update struct vertex instance only if there is a change @@ -622,7 +622,7 @@ public final class TypedInstanceToGraphMapper { if (id.isUnassigned()) { Vertex classVertex = idToVertexMap.get(id); - String guid = classVertex.getProperty(Constants.GUID_PROPERTY_KEY); + String guid = GraphHelper.getIdFromVertex(classVertex); id = new Id(guid, 0, typedReference.getTypeName()); } return id; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d64112d6/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java index 4503899..58ef675 100755 --- a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java @@ -26,7 +26,6 @@ import com.thinkaurelius.titan.core.TitanGraph; import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Vertex; - import org.apache.atlas.AtlasException; import org.apache.atlas.GraphTransaction; import org.apache.atlas.repository.Constants; @@ -59,6 +58,8 @@ import java.util.Iterator; import java.util.List; import java.util.Set; +import static org.apache.atlas.repository.graph.GraphHelper.setProperty; + @Singleton public class GraphBackedTypeStore implements ITypeStore { public static final String VERTEX_TYPE = "typeSystem"; @@ -106,20 +107,15 @@ public class GraphBackedTypeStore implements ITypeStore { } } - private void addProperty(Vertex vertex, String propertyName, Object value) { - LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex); - vertex.setProperty(propertyName, value); - } - private void storeInGraph(EnumType dataType) { Vertex vertex = createVertex(dataType.getTypeCategory(), dataType.getName(), dataType.getDescription()); List<String> values = new ArrayList<>(dataType.values().size()); for (EnumValue enumValue : dataType.values()) { String key = getPropertyKey(dataType.getName(), enumValue.value); - addProperty(vertex, key, enumValue.ordinal); + setProperty(vertex, key, enumValue.ordinal); values.add(enumValue.value); } - addProperty(vertex, getPropertyKey(dataType.getName()), values); + setProperty(vertex, getPropertyKey(dataType.getName()), values); } private String getPropertyKey(String name) { @@ -142,7 +138,7 @@ public class GraphBackedTypeStore implements ITypeStore { for (AttributeInfo attribute : attributes) { String propertyKey = getPropertyKey(typeName, attribute.name); try { - addProperty(vertex, propertyKey, attribute.toJson()); + setProperty(vertex, propertyKey, attribute.toJson()); } catch (JSONException e) { throw new StorageException(typeName, e); } @@ -150,7 +146,7 @@ public class GraphBackedTypeStore implements ITypeStore { addReferencesForAttribute(typeSystem, vertex, attribute); } } - addProperty(vertex, getPropertyKey(typeName), attrNames); + setProperty(vertex, getPropertyKey(typeName), attrNames); //Add edges for hierarchy if (superTypes != null) { @@ -272,10 +268,10 @@ public class GraphBackedTypeStore implements ITypeStore { String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY); String typeDescription = vertex.getProperty(Constants.TYPEDESCRIPTION_PROPERTY_KEY); List<EnumValue> enumValues = new ArrayList<>(); - List<String> values = vertex.getProperty(getPropertyKey(typeName)); + List<String> values = graphHelper.getProperty(vertex, getPropertyKey(typeName)); for (String value : values) { String valueProperty = getPropertyKey(typeName, value); - enumValues.add(new EnumValue(value, vertex.<Integer>getProperty(valueProperty))); + enumValues.add(new EnumValue(value, (Integer) graphHelper.getProperty(vertex, valueProperty))); } return new EnumTypeDefinition(typeName, typeDescription, enumValues.toArray(new EnumValue[enumValues.size()])); } @@ -292,12 +288,12 @@ public class GraphBackedTypeStore implements ITypeStore { private AttributeDefinition[] getAttributes(Vertex vertex, String typeName) throws AtlasException { List<AttributeDefinition> attributes = new ArrayList<>(); - List<String> attrNames = vertex.getProperty(getPropertyKey(typeName)); + List<String> attrNames = graphHelper.getProperty(vertex, getPropertyKey(typeName)); if (attrNames != null) { for (String attrName : attrNames) { try { String propertyKey = getPropertyKey(typeName, attrName); - attributes.add(AttributeInfo.fromJson((String) vertex.getProperty(propertyKey))); + attributes.add(AttributeInfo.fromJson((String) graphHelper.getProperty(vertex, propertyKey))); } catch (JSONException e) { throw new AtlasException(e); } @@ -306,10 +302,6 @@ public class GraphBackedTypeStore implements ITypeStore { return attributes.toArray(new AttributeDefinition[attributes.size()]); } - private String toString(Vertex vertex) { - return PROPERTY_PREFIX + vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY); - } - /** * Find vertex for the given type category and name, else create new vertex * @param category @@ -333,14 +325,14 @@ public class GraphBackedTypeStore implements ITypeStore { if (vertex == null) { LOG.debug("Adding vertex {}{}", PROPERTY_PREFIX, typeName); vertex = titanGraph.addVertex(null); - addProperty(vertex, Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE); // Mark as type vertex - addProperty(vertex, Constants.TYPE_CATEGORY_PROPERTY_KEY, category); - addProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, typeName); + setProperty(vertex, Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE); // Mark as type vertex + setProperty(vertex, Constants.TYPE_CATEGORY_PROPERTY_KEY, category); + setProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, typeName); } if (typeDescription != null) { String oldDescription = getPropertyKey(Constants.TYPEDESCRIPTION_PROPERTY_KEY); if (!typeDescription.equals(oldDescription)) { - addProperty(vertex, Constants.TYPEDESCRIPTION_PROPERTY_KEY, typeDescription); + setProperty(vertex, Constants.TYPEDESCRIPTION_PROPERTY_KEY, typeDescription); } } else { LOG.debug(" type description is null "); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d64112d6/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala index 8add6c8..d76b173 100755 --- a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala +++ b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala @@ -18,15 +18,16 @@ package org.apache.atlas.query -import org.apache.atlas.query.TypeUtils.FieldInfo; +import org.apache.atlas.query.TypeUtils.FieldInfo import org.apache.atlas.query.Expressions._ -import org.apache.atlas.typesystem.types.{TypeSystem, DataTypes} +import org.apache.atlas.typesystem.types.{DataTypes, TypeSystem} import org.apache.atlas.typesystem.types.DataTypes.TypeCategory import org.joda.time.format.ISODateTimeFormat import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import org.apache.atlas.typesystem.types.IDataType +import org.apache.commons.lang.StringEscapeUtils trait IntSequence { def next: Int @@ -111,7 +112,8 @@ trait SelectExpressionHandling { /** * For each Output Column in the SelectExpression compute the ArrayList(Src) this maps to and the position within * this list. - * @param sel + * + * @param sel * @return */ def buildResultMapping(sel: SelectExpression): Map[String, (String, Int)] = { @@ -200,13 +202,19 @@ class GremlinTranslator(expr: Expression, } def typeTestExpression(typeName : String) : String = { - val stats = gPersistenceBehavior.typeTestExpression(typeName, counter) + val stats = gPersistenceBehavior.typeTestExpression(escape(typeName), counter) preStatements ++= stats.init stats.last } + def escape(str: String): String = { + if (str != null) { + return str.replace("\"", "\\\"").replace("$", "\\$"); + } + str + } - private def genQuery(expr: Expression, inSelect: Boolean): String = expr match { + private def genQuery(expr: Expression, inSelect: Boolean): String = expr match { case ClassExpression(clsName) => typeTestExpression(clsName) case TraitExpression(clsName) => @@ -239,9 +247,9 @@ class GremlinTranslator(expr: Expression, } } case c@ComparisonExpression(symb, f@FieldExpression(fieldName, fInfo, ch), l) => { - val qualifiedPropertyName = s"${gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo)}"; - val persistentExprValue = translateValueToPersistentForm(fInfo, l); - return generateAndPrependExpr(ch, inSelect, s"""has("${qualifiedPropertyName}", ${gPersistenceBehavior.gremlinCompOp(c)}, $persistentExprValue)"""); + val qualifiedPropertyName = s"${gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo)}" + val persistentExprValue = translateValueToPersistentForm(fInfo, l) + return generateAndPrependExpr(ch, inSelect, s"""has("${qualifiedPropertyName}", ${gPersistenceBehavior.gremlinCompOp(c)}, $persistentExprValue)""") } case fil@FilterExpression(child, condExpr) => { s"${genQuery(child, inSelect)}.${genQuery(condExpr, inSelect)}" @@ -329,9 +337,9 @@ class GremlinTranslator(expr: Expression, def translateValueToPersistentForm(fInfo: FieldInfo, l: Expression): Any = { val dataType = fInfo.attrInfo.dataType; + val QUOTE = "\""; if (dataType == DataTypes.DATE_TYPE) { - val QUOTE = "\""; try { //Accepts both date, datetime formats val dateStr = l.toString.stripPrefix(QUOTE).stripSuffix(QUOTE) @@ -360,9 +368,10 @@ class GremlinTranslator(expr: Expression, } else if(dataType == DataTypes.DOUBLE_TYPE) { return s"""${l}d""" - } - else { - return l + } else if(dataType == DataTypes.STRING_TYPE) { + return string(escape(l.toString.stripPrefix(QUOTE).stripSuffix(QUOTE))); + } else { + l } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d64112d6/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java new file mode 100644 index 0000000..8a0a6bc --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.graph; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; + +public class GraphHelperTest { + @DataProvider(name = "encodeDecodeTestData") + private Object[][] createTestData() { + return new Object[][]{ + {"hivedb$", "hivedb_d"}, + {"hivedb", "hivedb"}, + {"{hivedb}", "_ohivedb_c"}, + {"%hivedb}", "_phivedb_c"}, + {"\"hivedb\"", "_qhivedb_q"}, + {"\"$%{}", "_q_d_p_o_c"}, + {"", ""}, + {" ", " "}, + {"\n\r", "\n\r"}, + {null, null} + }; + } + + @Test(dataProvider = "encodeDecodeTestData") + public void testEncodeDecode(String str, String expectedEncodedStr) throws Exception { + String encodedStr = GraphHelper.encodePropertyKey(str); + assertEquals(encodedStr, expectedEncodedStr); + + String decodedStr = GraphHelper.decodePropertyKey(encodedStr); + assertEquals(decodedStr, str); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d64112d6/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java index 443dd4a..f2a5b50 100644 --- a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java @@ -24,12 +24,12 @@ import com.google.inject.Inject; import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.util.TitanCleanup; import org.apache.atlas.AtlasClient; -import org.apache.atlas.AtlasConstants; import org.apache.atlas.AtlasException; import org.apache.atlas.EntityAuditEvent; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.RequestContext; import org.apache.atlas.TestUtils; +import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; import org.apache.atlas.listener.EntityChangeListener; import org.apache.atlas.repository.audit.EntityAuditRepository; import org.apache.atlas.repository.audit.HBaseBasedAuditRepository; @@ -47,10 +47,12 @@ import org.apache.atlas.typesystem.exception.TypeNotFoundException; import org.apache.atlas.typesystem.json.InstanceSerialization; import org.apache.atlas.typesystem.json.TypesSerialization; import org.apache.atlas.typesystem.persistence.Id; +import org.apache.atlas.typesystem.types.AttributeDefinition; import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.DataTypes; import org.apache.atlas.typesystem.types.EnumValue; import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.ValueConversionException; import org.apache.atlas.typesystem.types.utils.TypesUtil; @@ -78,7 +80,11 @@ import static org.apache.atlas.TestUtils.PII; import static org.apache.atlas.TestUtils.TABLE_TYPE; import static org.apache.atlas.TestUtils.createColumnEntity; import static org.apache.atlas.TestUtils.createDBEntity; +import static org.apache.atlas.TestUtils.createInstance; import static org.apache.atlas.TestUtils.createTableEntity; +import static org.apache.atlas.TestUtils.randomString; +import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; +import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; @@ -96,6 +102,9 @@ public class DefaultMetadataServiceTest { @Inject private EntityAuditRepository auditRepository; + @Inject + private GraphBackedDiscoveryService discoveryService; + private Referenceable db = createDBEntity(); private Referenceable table; @@ -121,9 +130,9 @@ public class DefaultMetadataServiceTest { metadataService.createType(TypesSerialization.toJson(typesDef)); } - String dbGUid = createInstance(db); + String dbGUid = TestUtils.createInstance(metadataService, db); table = createTableEntity(dbGUid); - String tableGuid = createInstance(table); + String tableGuid = TestUtils.createInstance(metadataService, table); String tableDefinitionJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); table = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); @@ -151,19 +160,6 @@ public class DefaultMetadataServiceTest { } } - private String createInstance(Referenceable entity) throws Exception { - RequestContext.createContext(); - - String entityjson = InstanceSerialization.toJson(entity, true); - JSONArray entitiesJson = new JSONArray(); - entitiesJson.put(entityjson); - List<String> guids = metadataService.createEntities(entitiesJson.toString()); - if (guids != null && guids.size() > 0) { - return guids.get(guids.size() - 1); - } - return null; - } - private AtlasClient.EntityResult updateInstance(Referenceable entity) throws Exception { RequestContext.createContext(); ParamChecker.notNull(entity, "Entity"); @@ -180,7 +176,7 @@ public class DefaultMetadataServiceTest { String dbName = RandomStringUtils.randomAlphanumeric(10); entity.set(NAME, dbName); entity.set("description", "us db"); - createInstance(entity); + TestUtils.createInstance(metadataService, entity); Assert.fail(TypeNotFoundException.class.getSimpleName() + " was expected but none thrown."); } @@ -188,23 +184,79 @@ public class DefaultMetadataServiceTest { public void testCreateEntityWithUniqueAttribute() throws Exception { //name is the unique attribute Referenceable entity = createDBEntity(); - String id = createInstance(entity); + String id = TestUtils.createInstance(metadataService, entity); assertAuditEvents(id, EntityAuditEvent.EntityAuditAction.ENTITY_CREATE); //using the same name should succeed, but not create another entity - String newId = createInstance(entity); + String newId = TestUtils.createInstance(metadataService, entity); assertNull(newId); //Same entity, but different qualified name should succeed entity.set(NAME, TestUtils.randomString()); - newId = createInstance(entity); + newId = TestUtils.createInstance(metadataService, entity); Assert.assertNotEquals(newId, id); } @Test + //Titan doesn't allow some reserved chars in property keys. Verify that atlas encodes these + //See GraphHelper.encodePropertyKey() + public void testSpecialCharacters() throws Exception { + //Verify that type can be created with reserved characters in typename, attribute name + String strAttrName = randomStrWithReservedChars(); + String arrayAttrName = randomStrWithReservedChars(); + String mapAttrName = randomStrWithReservedChars(); + HierarchicalTypeDefinition<ClassType> typeDefinition = + createClassTypeDef(randomStrWithReservedChars(), ImmutableSet.<String>of(), + createOptionalAttrDef(strAttrName, DataTypes.STRING_TYPE), + new AttributeDefinition(arrayAttrName, DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()), + Multiplicity.OPTIONAL, false, null), + new AttributeDefinition(mapAttrName, + DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()), + Multiplicity.OPTIONAL, false, null)); + metadataService.createType(TypesSerialization.toJson(typeDefinition, false)); + + //verify that entity can be created with reserved characters in string value, array value and map key and value + Referenceable entity = new Referenceable(typeDefinition.typeName); + entity.set(strAttrName, randomStrWithReservedChars()); + entity.set(arrayAttrName, new String[]{randomStrWithReservedChars()}); + entity.set(mapAttrName, new HashMap<String, String>() {{ + put(randomStrWithReservedChars(), randomStrWithReservedChars()); + }}); + String id = createInstance(metadataService, entity); + + //Verify that get entity definition returns actual values with reserved characters + Referenceable instance = + InstanceSerialization.fromJsonReferenceable(metadataService.getEntityDefinition(id), true); + assertReferenceableEquals(instance, entity); + + //Verify that search with reserved characters works - for string attribute + String responseJson = discoveryService.searchByDSL( + String.format("`%s` where `%s` = '%s'", typeDefinition.typeName, strAttrName, entity.get(strAttrName))); + JSONObject response = new JSONObject(responseJson); + assertEquals(response.getJSONArray("rows").length(), 1); + } + + //equals excluding the id + private void assertReferenceableEquals(Referenceable actual, Referenceable expected) { + List<String> traits = actual.getTraits(); + Map<String, IStruct> traitsMap = new HashMap<>(); + for (String trait : traits) { + traitsMap.put(trait, actual.getTrait(trait)); + } + + Referenceable newActual = new Referenceable(expected.getId(), actual.getTypeName(), actual.getValuesMap(), + traits, traitsMap); + assertEquals(InstanceSerialization.toJson(newActual, true), InstanceSerialization.toJson(expected, true)); + } + + private String randomStrWithReservedChars() { + return randomString() + "\"${}%"; + } + + @Test public void testAddDeleteTrait() throws Exception { Referenceable entity = createDBEntity(); - String id = createInstance(entity); + String id = TestUtils.createInstance(metadataService, entity); //add trait Struct tag = new Struct(TestUtils.PII); @@ -218,7 +270,7 @@ public class DefaultMetadataServiceTest { String traitDefinition = metadataService.getTraitDefinition(id, PII); Struct traitResult = InstanceSerialization.fromJsonStruct(traitDefinition, true); Assert.assertNotNull(traitResult); - Assert.assertEquals(traitResult.getValuesMap().size(), 0); + assertEquals(traitResult.getValuesMap().size(), 0); //delete trait metadataService.deleteTrait(id, PII); @@ -237,7 +289,7 @@ public class DefaultMetadataServiceTest { public void testEntityAudit() throws Exception { //create entity Referenceable entity = createDBEntity(); - String id = createInstance(entity); + String id = TestUtils.createInstance(metadataService, entity); assertAuditEvents(id, EntityAuditEvent.EntityAuditAction.ENTITY_CREATE); Struct tag = new Struct(TestUtils.PII); @@ -279,7 +331,7 @@ public class DefaultMetadataServiceTest { @Test public void testCreateEntityWithUniqueAttributeWithReference() throws Exception { Referenceable db = createDBEntity(); - String dbId = createInstance(db); + String dbId = TestUtils.createInstance(metadataService, db); //Assert that there is just 1 audit events and thats for entity create assertAuditEvents(dbId, 1); @@ -292,14 +344,14 @@ public class DefaultMetadataServiceTest { table.set("tableType", "MANAGED"); table.set("database", new Id(dbId, 0, TestUtils.DATABASE_TYPE)); table.set("databaseComposite", db); - createInstance(table); + TestUtils.createInstance(metadataService, table); //table create should re-use the db instance created earlier String tableDefinitionJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); Referenceable actualDb = (Referenceable) tableDefinition.get("databaseComposite"); - Assert.assertEquals(actualDb.getId().id, dbId); + assertEquals(actualDb.getId().id, dbId); //Assert that as part table create, db is not created and audit event is not added to db assertAuditEvents(dbId, 1); @@ -318,7 +370,7 @@ public class DefaultMetadataServiceTest { metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); List<String> actualColumns = (List) tableDefinition.get("columnNames"); - Assert.assertEquals(actualColumns, colNameList); + assertEquals(actualColumns, colNameList); } @Test @@ -350,7 +402,7 @@ public class DefaultMetadataServiceTest { metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); - Assert.assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2); + assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2); Assert.assertTrue(partsMap.get("part1").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part1"))); //update map - remove a key and add another key @@ -366,7 +418,7 @@ public class DefaultMetadataServiceTest { metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); - Assert.assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2); + assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2); Assert.assertNull(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part0")); Assert.assertTrue(partsMap.get("part2").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part2"))); @@ -378,7 +430,7 @@ public class DefaultMetadataServiceTest { metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); - Assert.assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2); + assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2); Assert.assertNull(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part0")); Assert.assertTrue(partsMap.get("part2").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part2"))); @@ -471,7 +523,7 @@ public class DefaultMetadataServiceTest { metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); List<String> actualColumns = (List) tableDefinition.get("columnNames"); - Assert.assertEquals(actualColumns, colNameList); + assertEquals(actualColumns, colNameList); //update array of primitives final List<String> updatedColNameList = ImmutableList.of("col2", "col3"); @@ -484,7 +536,7 @@ public class DefaultMetadataServiceTest { metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); actualColumns = (List) tableDefinition.get("columnNames"); - Assert.assertEquals(actualColumns, updatedColNameList); + assertEquals(actualColumns, updatedColNameList); } private AtlasClient.EntityResult updateEntityPartial(String guid, Referenceable entity) throws AtlasException { @@ -625,7 +677,7 @@ public class DefaultMetadataServiceTest { table.set("serde1", serdeInstance); String newtableId = updateInstance(table).getUpdateEntities().get(0); - Assert.assertEquals(newtableId, tableId._getId()); + assertEquals(newtableId, tableId._getId()); String tableDefinitionJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); @@ -659,7 +711,7 @@ public class DefaultMetadataServiceTest { sdReferenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, TestUtils.randomString()); sdReferenceable.set("compressed", "false"); sdReferenceable.set("location", "hdfs://tmp/hive-user"); - String sdGuid = createInstance(sdReferenceable); + String sdGuid = TestUtils.createInstance(metadataService, sdReferenceable); Referenceable sdRef2 = new Referenceable(sdGuid, TestUtils.STORAGE_DESC_TYPE, null); @@ -669,7 +721,7 @@ public class DefaultMetadataServiceTest { partRef.set("table", table); partRef.set("sd", sdRef2); - String partGuid = createInstance(partRef); + String partGuid = TestUtils.createInstance(metadataService, partRef); Assert.assertNotNull(partGuid); } @@ -680,7 +732,7 @@ public class DefaultMetadataServiceTest { databaseInstance.set(NAME, TestUtils.randomString()); databaseInstance.set("description", "new database"); - String dbId = createInstance(databaseInstance); + String dbId = TestUtils.createInstance(metadataService, databaseInstance); /*Update reference property with Id */ metadataService.updateEntityAttributeByGuid(tableId._getId(), "database", dbId); @@ -689,7 +741,7 @@ public class DefaultMetadataServiceTest { metadataService.getEntityDefinition(tableId._getId()); Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); - Assert.assertEquals(dbId, (((Id) tableDefinition.get("database"))._getId())); + assertEquals(dbId, (((Id) tableDefinition.get("database"))._getId())); /* Update with referenceable - TODO - Fails . Need to fix this */ /*final String dbName = TestUtils.randomString(); @@ -729,7 +781,7 @@ public class DefaultMetadataServiceTest { table.set("partitions", partitions); String newtableId = updateInstance(table).getUpdateEntities().get(0); - Assert.assertEquals(newtableId, tableId._getId()); + assertEquals(newtableId, tableId._getId()); String tableDefinitionJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); @@ -745,7 +797,7 @@ public class DefaultMetadataServiceTest { partitions.add(partition3); table.set("partitions", partitions); newtableId = updateInstance(table).getUpdateEntities().get(0); - Assert.assertEquals(newtableId, tableId._getId()); + assertEquals(newtableId, tableId._getId()); tableDefinitionJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); @@ -759,7 +811,7 @@ public class DefaultMetadataServiceTest { partitions.remove(1); table.set("partitions", partitions); newtableId = updateInstance(table).getUpdateEntities().get(0); - Assert.assertEquals(newtableId, tableId._getId()); + assertEquals(newtableId, tableId._getId()); tableDefinitionJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); @@ -772,7 +824,7 @@ public class DefaultMetadataServiceTest { //Update struct value within array of struct partitions.get(0).set(NAME, "part4"); newtableId = updateInstance(table).getUpdateEntities().get(0); - Assert.assertEquals(newtableId, tableId._getId()); + assertEquals(newtableId, tableId._getId()); tableDefinitionJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); @@ -788,7 +840,7 @@ public class DefaultMetadataServiceTest { partitions.add(partition4); table.set("partitions", partitions); newtableId = updateInstance(table).getUpdateEntities().get(0); - Assert.assertEquals(newtableId, tableId._getId()); + assertEquals(newtableId, tableId._getId()); tableDefinitionJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); @@ -802,7 +854,7 @@ public class DefaultMetadataServiceTest { // Remove all elements. Should set array attribute to null partitions.clear(); newtableId = updateInstance(table).getUpdateEntities().get(0); - Assert.assertEquals(newtableId, tableId._getId()); + assertEquals(newtableId, tableId._getId()); tableDefinitionJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); @@ -826,7 +878,7 @@ public class DefaultMetadataServiceTest { metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); - Assert.assertEquals(tableDefinition.get("description"), "random table"); + assertEquals(tableDefinition.get("description"), "random table"); table.setNull("description"); updateInstance(table); @@ -845,7 +897,7 @@ public class DefaultMetadataServiceTest { table.setNull("created"); String newtableId = updateInstance(table).getUpdateEntities().get(0); - Assert.assertEquals(newtableId, tableId._getId()); + assertEquals(newtableId, tableId._getId()); tableDefinitionJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); @@ -860,20 +912,20 @@ public class DefaultMetadataServiceTest { Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); EnumValue tableType = (EnumValue) tableDefinition.get("tableType"); - Assert.assertEquals(tableType, new EnumValue("MANAGED", 1)); + assertEquals(tableType, new EnumValue("MANAGED", 1)); } @Test public void testGetEntityByUniqueAttribute() throws Exception { Referenceable entity = createDBEntity(); - createInstance(entity); + TestUtils.createInstance(metadataService, entity); //get entity by valid qualified name String entityJson = metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE, NAME, (String) entity.get(NAME)); Assert.assertNotNull(entityJson); Referenceable referenceable = InstanceSerialization.fromJsonReferenceable(entityJson, true); - Assert.assertEquals(referenceable.get(NAME), entity.get(NAME)); + assertEquals(referenceable.get(NAME), entity.get(NAME)); //get entity by invalid qualified name try { @@ -897,13 +949,13 @@ public class DefaultMetadataServiceTest { public void testDeleteEntities() throws Exception { // Create a table entity, with 3 composite column entities Referenceable dbEntity = createDBEntity(); - String dbGuid = createInstance(dbEntity); + String dbGuid = TestUtils.createInstance(metadataService, dbEntity); Referenceable table1Entity = createTableEntity(dbGuid); Referenceable col1 = createColumnEntity(); Referenceable col2 = createColumnEntity(); Referenceable col3 = createColumnEntity(); table1Entity.set(COLUMNS_ATTR_NAME, ImmutableList.of(col1, col2, col3)); - createInstance(table1Entity); + TestUtils.createInstance(metadataService, table1Entity); // Retrieve the table entities from the repository, // to get their guids and the composite column guids. @@ -946,7 +998,7 @@ public class DefaultMetadataServiceTest { // Verify that the listener was notified about the deleted entities. List<String> deletedEntitiesFromListener = listener.getDeletedEntities(); Assert.assertNotNull(deletedEntitiesFromListener); - Assert.assertEquals(deletedEntitiesFromListener.size(), entityResult.getDeletedEntities().size()); + assertEquals(deletedEntitiesFromListener.size(), entityResult.getDeletedEntities().size()); Assert.assertTrue(deletedEntitiesFromListener.containsAll(entityResult.getDeletedEntities())); } @@ -964,13 +1016,13 @@ public class DefaultMetadataServiceTest { public void testDeleteEntityByUniqueAttribute() throws Exception { // Create a table entity, with 3 composite column entities Referenceable dbEntity = createDBEntity(); - String dbGuid = createInstance(dbEntity); + String dbGuid = TestUtils.createInstance(metadataService, dbEntity); Referenceable table1Entity = createTableEntity(dbGuid); Referenceable col1 = createColumnEntity(); Referenceable col2 = createColumnEntity(); Referenceable col3 = createColumnEntity(); table1Entity.set(COLUMNS_ATTR_NAME, ImmutableList.of(col1, col2, col3)); - createInstance(table1Entity); + TestUtils.createInstance(metadataService, table1Entity); // to get their guids and the composite column guids. String entityJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, @@ -1005,12 +1057,12 @@ public class DefaultMetadataServiceTest { // Verify that the listener was notified about the deleted entities. List<String> deletedEntitiesFromListener = listener.getDeletedEntities(); Assert.assertNotNull(deletedEntitiesFromListener); - Assert.assertEquals(deletedEntitiesFromListener.size(), deletedGuids.size()); + assertEquals(deletedEntitiesFromListener.size(), deletedGuids.size()); Assert.assertTrue(deletedEntitiesFromListener.containsAll(deletedGuids)); } @Test - public void testTypeUpdateWithReservedAttributes() throws AtlasException, JSONException { + public void testTypeUpdateFailureShouldRollBack() throws AtlasException, JSONException { String typeName = "test_type_"+ RandomStringUtils.randomAlphanumeric(10); HierarchicalTypeDefinition<ClassType> typeDef = TypesUtil.createClassTypeDef( typeName, ImmutableSet.<String>of(), @@ -1022,17 +1074,20 @@ public class DefaultMetadataServiceTest { HierarchicalTypeDefinition<ClassType> updatedTypeDef = TypesUtil.createClassTypeDef( typeName, ImmutableSet.<String>of(), TypesUtil.createUniqueRequiredAttrDef("test_type_attribute", DataTypes.STRING_TYPE), - TypesUtil.createOptionalAttrDef("test_type_invalid_attribute$", DataTypes.STRING_TYPE)); + TypesUtil.createRequiredAttrDef("test_type_invalid_attribute$", DataTypes.STRING_TYPE)); TypesDef updatedTypesDef = new TypesDef(updatedTypeDef, false); try { metadataService.updateType(TypesSerialization.toJson(updatedTypesDef)); - Assert.fail("Should not be able to update type with reserved character"); - } catch (AtlasException ae) { - // pass.. expected + fail("Expected AtlasException"); + } catch (AtlasException e) { + //expected } + + //type definition should reflect old type String typeDefinition = metadataService.getTypeDefinition(typeName); - Assert.assertNotNull(typeDefinition); + typesDef = TypesSerialization.fromJson(typeDefinition); + assertEquals(typesDef.classTypes().head().attributeDefinitions.length, 1); } @Test
