fixed test failures - GraphBackedMetadataRepositoryTest#testFullTextSearch, DSLTest.test1
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/4b2a4cae Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/4b2a4cae Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/4b2a4cae Branch: refs/remotes/origin/master Commit: 4b2a4caed572fb358cddb37ce1b183072e08a0a6 Parents: 8e73ed2 Author: Shwetha GS <[email protected]> Authored: Mon May 4 14:59:11 2015 +0530 Committer: Shwetha GS <[email protected]> Committed: Mon May 4 14:59:22 2015 +0530 ---------------------------------------------------------------------- .../graph/GraphBackedMetadataRepository.java | 71 +++++++++----------- .../metadata/repository/graph/GraphHelper.java | 2 +- .../GraphBackedMetadataRepositoryTest.java | 14 ++-- .../hadoop/metadata/tools/dsl/package.scala | 2 +- .../hadoop/metadata/tools/dsl/DSLTest.scala | 2 +- .../metadata/typesystem/types/TypeSystem.java | 8 +-- 6 files changed, 44 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java index d6a3123..d273787 100755 --- a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java +++ b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java @@ -473,9 +473,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { Id id = typedInstance.getId(); Vertex instanceVertex = entityProcessor.idToVertexMap.get(id); String fullText = getFullTextForVertex(instanceVertex, true); - instanceVertex.setProperty(Constants.ENTITY_TEXT_PROPERTY_KEY, fullText); - LOG.debug("Adding {} for {} = {}", Constants.ENTITY_TEXT_PROPERTY_KEY, - instanceVertex, fullText); + addProperty(instanceVertex, Constants.ENTITY_TEXT_PROPERTY_KEY, fullText); } } @@ -658,8 +656,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository { private void mapInstanceToVertex(Id id, ITypedInstance typedInstance, Vertex instanceVertex, Map<String, AttributeInfo> fields, Map<Id, Vertex> idToVertexMap) throws MetadataException { - LOG.debug("Mapping instance {} to vertex {} for fields {}", - typedInstance.getTypeName(), instanceVertex, fields); + LOG.debug("Mapping instance {} of {} to vertex {}", + typedInstance, typedInstance.getTypeName(), instanceVertex); for (AttributeInfo attributeInfo : fields.values()) { final IDataType dataType = attributeInfo.dataType(); mapAttributesToVertex(id, typedInstance, instanceVertex, @@ -672,9 +670,10 @@ public class GraphBackedMetadataRepository implements MetadataRepository { Map<Id, Vertex> idToVertexMap, AttributeInfo attributeInfo, IDataType dataType) throws MetadataException { - LOG.debug("mapping attributeInfo {}", attributeInfo); + Object attrValue = typedInstance.get(attributeInfo.name); + LOG.debug("mapping attribute {} = {}", attributeInfo.name, attrValue); final String propertyName = getQualifiedName(typedInstance, attributeInfo); - if (typedInstance.get(attributeInfo.name) == null) { + if (attrValue == null) { return; } @@ -684,8 +683,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { break; case ENUM: - instanceVertex.setProperty(propertyName, - typedInstance.getInt(attributeInfo.name)); + addProperty(instanceVertex, propertyName, typedInstance.getInt(attributeInfo.name)); break; case ARRAY: @@ -749,7 +747,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { buffer.setLength(buffer.length() - 1); // for dereference on way out - instanceVertex.setProperty(propertyName, buffer.toString()); + addProperty(instanceVertex, propertyName, buffer.toString()); } private void mapMapCollectionToVertex(Id id, ITypedInstance typedInstance, @@ -778,7 +776,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { buffer.setLength(buffer.length() - 1); // for dereference on way out - instanceVertex.setProperty(propertyName, buffer.toString()); + addProperty(instanceVertex, propertyName, buffer.toString()); } private String mapCollectionEntryToVertex(Id id, Vertex instanceVertex, @@ -792,7 +790,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { switch (elementType.getTypeCategory()) { case PRIMITIVE: case ENUM: - instanceVertex.setProperty(propertyNameWithSuffix, value); + addProperty(instanceVertex, propertyNameWithSuffix, value); return propertyNameWithSuffix; case ARRAY: @@ -851,7 +849,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository { Vertex structInstanceVertex = GraphHelper.createVertexWithoutIdentity( titanGraph, structInstance.getTypeName(), id, Collections.<String>emptySet()); // no super types for struct type - LOG.debug("created vertex {} for struct {}", structInstanceVertex, attributeInfo.name); + LOG.debug("created vertex {} for struct {} value {}", structInstanceVertex, attributeInfo.name, + structInstance); // map all the attributes to this newly created vertex mapInstanceToVertex(id, structInstance, structInstanceVertex, @@ -895,47 +894,43 @@ public class GraphBackedMetadataRepository implements MetadataRepository { private void mapPrimitiveToVertex(ITypedInstance typedInstance, Vertex instanceVertex, AttributeInfo attributeInfo) throws MetadataException { - LOG.debug("Adding primitive {} to v {}", attributeInfo, instanceVertex); - if (typedInstance.get(attributeInfo.name) == null) { + Object attrValue = typedInstance.get(attributeInfo.name); + if (attrValue == null) { return; // add only if instance has this attribute } final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo); - + Object propertyValue = null; if (attributeInfo.dataType() == DataTypes.STRING_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getString(attributeInfo.name)); + propertyValue = typedInstance.getString(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getShort(attributeInfo.name)); + propertyValue = typedInstance.getShort(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.INT_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getInt(attributeInfo.name)); + propertyValue = typedInstance.getInt(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getBigInt(attributeInfo.name)); + propertyValue = typedInstance.getBigInt(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getBoolean(attributeInfo.name)); + propertyValue = typedInstance.getBoolean(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getByte(attributeInfo.name)); + propertyValue = typedInstance.getByte(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getLong(attributeInfo.name)); + propertyValue = typedInstance.getLong(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getFloat(attributeInfo.name)); + propertyValue = typedInstance.getFloat(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getDouble(attributeInfo.name)); + propertyValue = typedInstance.getDouble(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) { - instanceVertex.setProperty(vertexPropertyName, - typedInstance.getBigDecimal(attributeInfo.name)); + propertyValue = typedInstance.getBigDecimal(attributeInfo.name); } + addProperty(instanceVertex, vertexPropertyName, propertyValue); } } + private void addProperty(Vertex vertex, String propertyName, Object value) { + LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex); + vertex.setProperty(propertyName, value); + } + public final class GraphToTypedInstanceMapper { public ITypedReferenceableInstance mapGraphToTypedInstance(String guid, @@ -985,7 +980,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { public void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance, AttributeInfo attributeInfo) throws MetadataException { - LOG.debug("mapping attributeInfo = " + attributeInfo); + LOG.debug("mapping attributeInfo {}", attributeInfo.name); final IDataType dataType = attributeInfo.dataType(); final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo); @@ -1226,7 +1221,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ITypedStruct structInstance = structType.createInstance(); typedInstance.set(attributeInfo.name, structInstance); - String relationshipLabel = getQualifiedName(structType, attributeInfo.name); + String relationshipLabel = getQualifiedName(typedInstance, attributeInfo); LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel); for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) { final Vertex structInstanceVertex = edge.getVertex(Direction.IN); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java index 970bd81..ac9e2fa 100755 --- a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java @@ -85,7 +85,7 @@ public final class GraphHelper { public static Edge addEdge(TitanGraph titanGraph, Vertex fromVertex, Vertex toVertex, String edgeLabel) { - LOG.debug("Adding edge for {} -> struct label {} -> v{}", + LOG.debug("Adding edge for {} -> label {} -> {}", fromVertex, edgeLabel, toVertex); return titanGraph.addEdge(null, fromVertex, toVertex, edgeLabel); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java b/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java index 89609db..842677b 100755 --- a/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java +++ b/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java @@ -334,11 +334,6 @@ public class GraphBackedMetadataRepositoryTest { String response = discoveryService.searchByFullText("john"); Assert.assertNotNull(response); JSONArray results = new JSONArray(response); - System.out.println("Found the following results"); - for (int i = 0 ; i < results.length(); i++) { - JSONObject myrow = results.getJSONObject(i); - System.out.println(myrow.toString()); - } Assert.assertEquals(results.length(), 1); JSONObject row = (JSONObject) results.get(0); Assert.assertEquals(row.get("typeName"), "Person"); @@ -346,11 +341,10 @@ public class GraphBackedMetadataRepositoryTest { //person in hr department who lives in santa clara response = discoveryService.searchByFullText("Jane AND santa AND clara"); Assert.assertNotNull(response); - // todo: enable this - temporarily commented this as its failing -// results = new JSONArray(response); -// Assert.assertEquals(results.length(), 1); -// row = (JSONObject) results.get(0); -// Assert.assertEquals(row.get("typeName"), "Manager"); + results = new JSONArray(response); + Assert.assertEquals(results.length(), 1); + row = (JSONObject) results.get(0); + Assert.assertEquals(row.get("typeName"), "Manager"); //search for person in hr department whose name starts is john/jahn response = discoveryService.searchByFullText("hr AND (john OR jahn)"); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala ---------------------------------------------------------------------- diff --git a/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala b/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala index de62551..ba09342 100755 --- a/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala +++ b/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala @@ -36,7 +36,7 @@ import scala.collection.JavaConversions._ package object dsl { val defFormat = new DefaultFormats { - override protected def dateFormatter = new SimpleDateFormat("yyyy-MM-dd") + override protected def dateFormatter = TypeSystem.getInstance().getDateFormat; override val typeHints = NoTypeHints } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala ---------------------------------------------------------------------- diff --git a/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala b/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala index 47db8fe..1698d40 100755 --- a/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala +++ b/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala @@ -120,7 +120,7 @@ class DSLTest { Assert.assertEquals(s"${i.o.asInstanceOf[java.util.Map[_, _]].keySet}", "[b, a]") // 5. Serialize mytype instance to Json - Assert.assertEquals(s"${pretty(render(i))}", "{\n \"$typeName$\":\"mytype\",\n \"e\":1," + "\n \"n\":[1,1.100000000000000088817841970012523233890533447265625],\n \"h\":1.0,\n \"b\":true,\n \"k\":1,\n \"j\":1,\n \"d\":2,\n \"m\":[1,1],\n \"g\":1,\n \"a\":1,\n \"i\":1.0,\n \"c\":1,\n \"l\":\"2014-12-02\",\n \"f\":1,\n \"o\":{\n \"b\":2.0,\n \"a\":1.0\n }\n}") + Assert.assertEquals(s"${pretty(render(i))}", "{\n \"$typeName$\":\"mytype\",\n \"e\":1," + "\n \"n\":[1,1.100000000000000088817841970012523233890533447265625],\n \"h\":1.0,\n \"b\":true,\n \"k\":1,\n \"j\":1,\n \"d\":2,\n \"m\":[1,1],\n \"g\":1,\n \"a\":1,\n \"i\":1.0,\n \"c\":1,\n \"l\":\"2014-12-03\",\n \"f\":1,\n \"o\":{\n \"b\":2.0,\n \"a\":1.0\n }\n}") } @Test def test2 { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java index c04016e..61c60dc 100755 --- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java +++ b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java @@ -42,10 +42,10 @@ import java.util.concurrent.ConcurrentHashMap; @InterfaceAudience.Private public class TypeSystem { private static final TypeSystem INSTANCE = new TypeSystem(); - public static ThreadLocal<DateFormat> dateFormat = new ThreadLocal() { + private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal() { @Override - public DateFormat initialValue() { - DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + public SimpleDateFormat initialValue() { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); return dateFormat; } @@ -294,7 +294,7 @@ public class TypeSystem { return eT; } - public DateFormat getDateFormat() { + public SimpleDateFormat getDateFormat() { return dateFormat.get(); }
