ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/6b33bcf6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/6b33bcf6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/6b33bcf6 Branch: refs/heads/master Commit: 6b33bcf647264131cf5b37838485675d2ed9c017 Parents: 901aca9 Author: Shwetha GS <[email protected]> Authored: Mon Jun 22 16:52:15 2015 +0530 Committer: Shwetha GS <[email protected]> Committed: Mon Jun 22 16:52:15 2015 +0530 ---------------------------------------------------------------------- pom.xml | 16 +- release-log.txt | 1 + repository/pom.xml | 10 + .../graph/GraphBackedMetadataRepository.java | 7 +- .../graph/GraphBackedSearchIndexer.java | 8 +- .../org/apache/atlas/query/Expressions.scala | 7 +- .../query/GraphPersistenceStrategies.scala | 6 + .../org/apache/atlas/query/GremlinQuery.scala | 21 +- .../GraphBackedDiscoveryServiceTest.java | 5 +- .../graph/GraphRepoMapperScaleTest.java | 24 + .../org/apache/atlas/query/GremlinTest.scala | 397 +++++++++- .../apache/atlas/query/HiveTitanSample.scala | 27 +- .../apache/atlas/query/LineageQueryTest.scala | 779 ++++++++++--------- .../apache/atlas/query/QueryTestsUtils.scala | 10 +- .../atlas/typesystem/types/TypeSystem.java | 2 +- 15 files changed, 901 insertions(+), 419 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6b33bcf6/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index a45f59e..6e7c10c 100755 --- a/pom.xml +++ b/pom.xml @@ -338,6 +338,7 @@ <spray.version>1.3.1</spray.version> <guava.version>14.0</guava.version> <fastutil.version>6.5.16</fastutil.version> + <guice.version>4.0</guice.version> <PermGen>64m</PermGen> <MaxPermGen>512m</MaxPermGen> @@ -563,25 +564,25 @@ <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> - <version>4.0</version> + <version>${guice.version}</version> </dependency> <dependency> <groupId>com.google.inject.extensions</groupId> <artifactId>guice-throwingproviders</artifactId> - <version>4.0</version> + <version>${guice.version}</version> </dependency> <dependency> <groupId>com.google.inject.extensions</groupId> <artifactId>guice-multibindings</artifactId> - <version>4.0</version> + <version>${guice.version}</version> </dependency> <dependency> <groupId>com.google.inject.extensions</groupId> <artifactId>guice-servlet</artifactId> - <version>4.0</version> + <version>${guice.version}</version> </dependency> <dependency> @@ -591,6 +592,13 @@ </dependency> <dependency> + <groupId>org.skyscreamer</groupId> + <artifactId>jsonassert</artifactId> + <version>1.2.0</version> + <scope>test</scope> + </dependency> + + <dependency> <groupId>joda-time</groupId> <artifactId>joda-time</artifactId> <version>2.5</version> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6b33bcf6/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 1ccf094..f527845 100644 --- a/release-log.txt +++ b/release-log.txt @@ -4,6 +4,7 @@ Apache Atlas Release Notes --trunk - unreleased INCOMPATIBLE CHANGES: +ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags) ALL CHANGES: ATLAS-31 ATLAS build fails with clean repo (suma.shivaprasad via shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6b33bcf6/repository/pom.xml ---------------------------------------------------------------------- diff --git a/repository/pom.xml b/repository/pom.xml index e885f76..b23c289 100755 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -55,6 +55,11 @@ </dependency> <dependency> + <groupId>joda-time</groupId> + <artifactId>joda-time</artifactId> + </dependency> + + <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> </dependency> @@ -146,6 +151,11 @@ </dependency> <dependency> + <groupId>org.skyscreamer</groupId> + <artifactId>jsonassert</artifactId> + </dependency> + + <dependency> <groupId>org.testng</groupId> <artifactId>testng</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6b33bcf6/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java index dbee992..cc10ea3 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java @@ -857,7 +857,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository { } else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) { propertyValue = typedInstance.getBigDecimal(attributeInfo.name); } else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) { - propertyValue = typedInstance.getDate(attributeInfo.name); + final Date dateVal = typedInstance.getDate(attributeInfo.name); + //Convert Property value to Long while persisting + propertyValue = dateVal.getTime(); } addProperty(instanceVertex, vertexPropertyName, propertyValue); } @@ -1180,7 +1182,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository { typedInstance .setBigDecimal(attributeInfo.name, instanceVertex.<BigDecimal>getProperty(vertexPropertyName)); } else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) { - typedInstance.setDate(attributeInfo.name, instanceVertex.<Date>getProperty(vertexPropertyName)); + final Long dateVal = instanceVertex.<Long>getProperty(vertexPropertyName); + typedInstance.setDate(attributeInfo.name, new Date(dateVal)); } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6b33bcf6/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index 3e6edb6..c43c64f 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -65,7 +65,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer { this.titanGraph = graphProvider.get(); /* Create the transaction for indexing. - * Commit/rollback is expected to be called from the caller. */ management = titanGraph.getManagementSystem(); initialize(); @@ -257,7 +256,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer { } else if (dataType == DataTypes.BIGDECIMAL_TYPE) { return BigDecimal.class; } else if (dataType == DataTypes.DATE_TYPE) { - return Date.class; + //Indexing with date converted to long as of now since Titan is yet to add support for Date type with mixed indexes + return Long.class; } @@ -333,9 +333,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer { } private boolean checkIfMixedIndexApplicable(Class propertyClass) { - //TODO - Check why date types are failing in ES/Solr - if (propertyClass == Boolean.class || propertyClass == BigDecimal.class || propertyClass == BigInteger.class - || propertyClass == Date.class) { + if (propertyClass == Boolean.class || propertyClass == BigDecimal.class || propertyClass == BigInteger.class) { return false; } return true; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6b33bcf6/repository/src/main/scala/org/apache/atlas/query/Expressions.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/Expressions.scala b/repository/src/main/scala/org/apache/atlas/query/Expressions.scala index 7c5c2b5..24fa917 100755 --- a/repository/src/main/scala/org/apache/atlas/query/Expressions.scala +++ b/repository/src/main/scala/org/apache/atlas/query/Expressions.scala @@ -490,6 +490,8 @@ object Expressions { def string(rawValue: Any) = literal(DataTypes.STRING_TYPE, rawValue) + def date(rawValue: Any) = literal(DataTypes.DATE_TYPE, rawValue) + case class ArithmeticExpression(symbol: String, left: Expression, right: Expression) @@ -598,7 +600,10 @@ object Expressions { throw new UnresolvedException(this, s"datatype. Can not resolve due to unresolved children") } - if (left.dataType != DataTypes.STRING_TYPE || right.dataType != DataTypes.STRING_TYPE) { + + if(left.dataType == DataTypes.DATE_TYPE) { + DataTypes.DATE_TYPE + } else if (left.dataType != DataTypes.STRING_TYPE || right.dataType != DataTypes.STRING_TYPE) { TypeUtils.combinedType(left.dataType, right.dataType) } DataTypes.BOOLEAN_TYPE http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6b33bcf6/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala b/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala index b475407..2cec503 100755 --- a/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala +++ b/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala @@ -18,6 +18,8 @@ package org.apache.atlas.query +import java.util.Date + import com.thinkaurelius.titan.core.TitanVertex import com.tinkerpop.blueprints.Direction import org.apache.atlas.query.Expressions.{ComparisonExpression, ExpressionException} @@ -304,6 +306,10 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies { case x: FloatType => i.setFloat(aInfo.name, v.getProperty[java.lang.Float](fName)) case x: DoubleType => i.setDouble(aInfo.name, v.getProperty[java.lang.Double](fName)) case x: StringType => i.setString(aInfo.name, v.getProperty[java.lang.String](fName)) + case x: DateType => { + val dateVal = v.getProperty[java.lang.Long](fName) + i.setDate(aInfo.name, new Date(dateVal)) + } case _ => throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported") } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6b33bcf6/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala index 914ed7a..75fa916 100755 --- a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala +++ b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala @@ -19,7 +19,9 @@ package org.apache.atlas.query import org.apache.atlas.query.Expressions._ +import org.apache.atlas.typesystem.types.{TypeSystem, DataTypes} import org.apache.atlas.typesystem.types.DataTypes.TypeCategory +import org.joda.time.format.ISODateTimeFormat import scala.collection.mutable import scala.collection.mutable.ArrayBuffer @@ -227,12 +229,29 @@ class GremlinTranslator(expr: Expression, } } case c@ComparisonExpression(symb, f@FieldExpression(fieldName, fInfo, ch), l) => { + val QUOTE = "\""; val fieldGremlinExpr = s"${gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo)}" ch match { case Some(child) => { s"""${genQuery(child, inSelect)}.has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)}, $l)""" } - case None => s"""has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)}, $l)""" + case None => { + if (fInfo.attrInfo.dataType == DataTypes.DATE_TYPE) { + try { + //Accepts both date, datetime formats + val dateStr = l.toString.stripPrefix(QUOTE).stripSuffix(QUOTE) + val dateVal = ISODateTimeFormat.dateOptionalTimeParser().parseDateTime(dateStr).getMillis + s"""has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)},${dateVal})""" + } catch { + case pe: java.text.ParseException => + throw new GremlinTranslationException(c, + "Date format " + l + " not supported. Should be of the format " + TypeSystem.getInstance().getDateFormat.toPattern); + + } + } + else + s"""has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)}, $l)""" + } } } case fil@FilterExpression(child, condExpr) => { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6b33bcf6/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java index fc07010..67cdd78 100755 --- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java @@ -169,7 +169,8 @@ public class GraphBackedDiscoveryServiceTest { @DataProvider(name = "dslQueriesProvider") private Object[][] createDSLQueries() { - return new String[][]{{"from DB"}, {"DB"}, {"DB where DB.name=\"Reporting\""}, {"DB DB.name = \"Reporting\""}, + return new String[][]{ + {"from DB"}, {"DB"}, {"DB where DB.name=\"Reporting\""}, {"DB DB.name = \"Reporting\""}, {"DB where DB.name=\"Reporting\" select name, owner"}, {"DB has name"}, {"DB, Table"}, {"DB is JdbcAccess"}, /* @@ -186,6 +187,8 @@ public class GraphBackedDiscoveryServiceTest { {"DB where DB is JdbcAccess"}, {"DB where DB has name"}, {"DB Table"}, {"DB where DB has name"}, {"DB as db1 Table where (db1.name = \"Reporting\")"}, {"DB where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 "}, + {"Table where (name = \"sales_fact\" and created > \"2014-01-01\" ) select name as _col_0, created as _col_1 "}, + {"Table where (name = \"sales_fact\" and created > \"2014-12-11T02:35:58.440Z\" ) select name as _col_0, created as _col_1 "}, /* todo: does not work {"DB where (name = \"Reporting\") and ((createTime + 1) > 0)"}, http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6b33bcf6/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java index 01b7480..be99efe 100755 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java @@ -21,15 +21,20 @@ package org.apache.atlas.repository.graph; import com.google.common.collect.ImmutableList; import com.thinkaurelius.titan.core.TitanFactory; import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.TitanGraphQuery; import com.thinkaurelius.titan.core.TitanIndexQuery; +import com.thinkaurelius.titan.core.schema.TitanGraphIndex; import com.thinkaurelius.titan.diskstorage.BackendException; import com.thinkaurelius.titan.diskstorage.configuration.ReadConfiguration; import com.thinkaurelius.titan.diskstorage.configuration.backend.CommonsConfiguration; import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; import com.tinkerpop.blueprints.Compare; +import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.GraphQuery; +import com.tinkerpop.blueprints.Predicate; import com.tinkerpop.blueprints.Vertex; import org.apache.atlas.GraphTransaction; +import org.apache.atlas.repository.BaseTest; import org.apache.atlas.repository.Constants; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.Referenceable; @@ -55,6 +60,7 @@ import org.testng.annotations.Test; import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Date; import java.util.Map; import java.util.Random; @@ -157,6 +163,7 @@ public class GraphRepoMapperScaleTest { searchWithOutIndex("hive_table_type.name", "bar-999"); searchWithIndex("hive_table_type.name", "bar-999"); + searchWithIndex("hive_table_type.created", Compare.GREATER_THAN_EQUAL, BaseTest.TEST_DATE_IN_LONG); for (int index = 500; index < 600; index++) { searchWithIndex("hive_table_type.name", "bar-" + index); @@ -190,6 +197,21 @@ public class GraphRepoMapperScaleTest { } } finally { System.out.println("Search on [" + key + "=" + value + "] returned results: " + count + ", took " + ( + System.currentTimeMillis() - start) + " ms"); + } + } + + private void searchWithIndex(String key, Predicate searchPredicate, Object value) { + TitanGraph graph = graphProvider.get(); + long start = System.currentTimeMillis(); + int count = 0; + try { + GraphQuery query = graph.query().has(key, searchPredicate, value); + for (Vertex ignored : query.vertices()) { + count++; + } + } finally { + System.out.println("Search on [" + key + "=" + value + "] returned results: " + count + ", took " + ( System.currentTimeMillis() - start) + " ms"); } } @@ -222,6 +244,7 @@ public class GraphRepoMapperScaleTest { TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE), TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE), + TypesUtil.createOptionalAttrDef("created", DataTypes.DATE_TYPE), // enum new AttributeDefinition("tableType", "table_type", Multiplicity.REQUIRED, false, null), // array of strings @@ -262,6 +285,7 @@ public class GraphRepoMapperScaleTest { tableInstance.set("name", TABLE_NAME + "-" + uberIndex); tableInstance.set("description", "bar table" + "-" + uberIndex); tableInstance.set("type", "managed"); + tableInstance.set("created", new Date(BaseTest.TEST_DATE_IN_LONG)); tableInstance.set("tableType", 1); // enum // refer to an existing class http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6b33bcf6/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala ---------------------------------------------------------------------- diff --git a/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala b/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala index aa41313..957a566 100755 --- a/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala +++ b/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala @@ -43,7 +43,7 @@ class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest { test("testClass") { val r = QueryProcessor.evaluate(_class("DB"), g) - validateJson(r, "{\n \"query\":\"DB\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"John ETL\",\n \"name\":\"Sales\",\n \"createTime\":1000\n },\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}") + validateJson(r, "{\n \"query\":\"DB\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"John ETL\",\n \"name\":\"Sales\",\n \"createTime\":1000\n },\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7424\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}") } test("testName") { @@ -53,12 +53,12 @@ class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest { test("testFilter") { var r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting"))), g) - validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\")\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}") + validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\")\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7424\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}") } test("testFilter2") { var r = QueryProcessor.evaluate(_class("DB").where(id("DB").field("name").`=`(string("Reporting"))), g) - validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\")\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}") + validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\")\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7424\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}") } @@ -70,17 +70,400 @@ class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest { test("testIsTrait") { val r = QueryProcessor.evaluate(_class("Table").where(isTrait("Dimension")), g) - validateJson(r, "{\n \"query\":\"Table where Table is Dimension\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"Table\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"db\",\n \"dataTypeName\":\"DB\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"sd\",\n \"dataTypeName\ ":\"StorageDesc\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"3328\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"2304\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"product_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"4864\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"3840\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"time_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"6656\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"5376\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"customer_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n }\n ]\n}") + validateJson(r, """{ + | "query":"Table where Table is Dimension", + | "dataType":{ + | "superTypes":[ + | + | ], + | "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType", + | "typeName":"Table", + | "attributeDefinitions":[ + | { + | "name":"name", + | "dataTypeName":"string", + | "multiplicity":{ + | "lower":0, + | "upper":1, + | "isUnique":false + | }, + | "isComposite":false, + | "isUnique":false, + | "isIndexable":true, + | "reverseAttributeName":null + | }, + | { + | "name":"db", + | "dataTypeName":"DB", + | "multiplicity":{ + | "lower":1, + | "upper":1, + | "isUnique":false + | }, + | "isComposite":false, + | "isUnique":false, + | "isIndexable":true, + | "reverseAttributeName":null + | }, + | { + | "name":"sd", + | "dataTypeName":"StorageDesc", + | "multiplicity":{ + | "lower":1, + | "upper":1, + | "isUnique":false + | }, + | "isComposite":false, + | "isUnique":false, + | "isIndexable":true, + | "reverseAttributeName":null + | }, + | { + | "name":"created", + | "dataTypeName":"date", + | "multiplicity":{ + | "lower":0, + | "upper":1, + | "isUnique":false + | }, + | "isComposite":false, + | "isUnique":false, + | "isIndexable":true, + | "reverseAttributeName":null + | } + | ] + | }, + | "rows":[ + | { + | "$typeName$":"Table", + | "$id$":{ + | "$typeName$":"Table", + | "version":0 + | }, + | "created":"2014-12-11T02:35:58.440Z", + | "sd":{ + | "$typeName$":"StorageDesc", + | "version":0 + | }, + | "db":{ + | "$typeName$":"DB", + | "version":0 + | }, + | "name":"product_dim", + | "$traits$":{ + | "Dimension":{ + | "$typeName$":"Dimension" + | } + | } + | }, + | { + | "$typeName$":"Table", + | "$id$":{ + | "$typeName$":"Table", + | "version":0 + | }, + | "created":"2014-12-11T02:35:58.440Z", + | "sd":{ + | "$typeName$":"StorageDesc", + | "version":0 + | }, + | "db":{ + | "$typeName$":"DB", + | "version":0 + | }, + | "name":"time_dim", + | "$traits$":{ + | "Dimension":{ + | "$typeName$":"Dimension" + | } + | } + | }, + | { + | "$typeName$":"Table", + | "$id$":{ + | "$typeName$":"Table", + | "version":0 + | }, + | "created":"2014-12-11T02:35:58.440Z", + | "sd":{ + | "$typeName$":"StorageDesc", + | "version":0 + | }, + | "db":{ + | "$typeName$":"DB", + | "version":0 + | }, + | "name":"customer_dim", + | "$traits$":{ + | "Dimension":{ + | "$typeName$":"Dimension" + | } + | } + | } + | ] + |}""".stripMargin) } test("testhasField") { val r = QueryProcessor.evaluate(_class("DB").where(hasField("name")), g) - validateJson(r, "{\n \"query\":\"DB where DB has name\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeNam e\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"John ETL\",\n \"name\":\"Sales\",\n \"createTime\":1000\n },\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}") + validateJson(r, """{ + | "query":"DB where DB has name", + | "dataType":{ + | "superTypes":[ + | + | ], + | "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType", + | "typeName":"DB", + | "attributeDefinitions":[ + | { + | "name":"name", + | "dataTypeName":"string", + | "multiplicity":{ + | "lower":0, + | "upper":1, + | "isUnique":false + | }, + | "isComposite":false, + | "isUnique":false, + | "isIndexable":true, + | "reverseAttributeName":null + | }, + | { + | "name":"owner", + | "dataTypeName":"string", + | "multiplicity":{ + | "lower":0, + | "upper":1, + | "isUnique":false + | }, + | "isComposite":false, + | "isUnique":false, + | "isIndexable":true, + | "reverseAttributeName":null + | }, + | { + | "name":"createTime", + | "dataTypeName":"int", + | "multiplicity":{ + | "lower":0, + | "upper":1, + | "isUnique":false + | }, + | "isComposite":false, + | "isUnique":false, + | "isIndexable":true, + | "reverseAttributeName":null + | } + | ] + | }, + | "rows":[ + | { + | "$typeName$":"DB", + | "$id$":{ + | "$typeName$":"DB", + | "version":0 + | }, + | "owner":"John ETL", + | "name":"Sales", + | "createTime":1000 + | }, + | { + | "$typeName$":"DB", + | "$id$":{ + | "$typeName$":"DB", + | "version":0 + | }, + | "owner":"Jane BI", + | "name":"Reporting", + | "createTime":1500 + | } + | ] + |}""".stripMargin) } test("testFieldReference") { val r = QueryProcessor.evaluate(_class("DB").field("Table"), g) - validateJson(r, "{\n \"query\":\"DB Table\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"Table\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"db\",\n \"dataTypeName\":\"DB\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"sd\",\n \"dataTypeName\":\"StorageDesc\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"2048\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"512\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"3328\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"2304\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"2 56\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"product_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"4864\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"3840\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"time_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"6656\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"5376\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"customer_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"8960\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"7424\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"12800\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"11264\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"7168\" ,\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"sales_fact_monthly_mv\"\n }\n ]\n}") + validateJson(r, """{ + | "query":"DB Table", + | "dataType":{ + | "superTypes":[ ], + | "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType", + | "typeName":"Table", + | "attributeDefinitions":[ + | { + | "name":"name", + | "dataTypeName":"string", + | "multiplicity":{ + | "lower":0, + | "upper":1, + | "isUnique":false + | }, + | "isComposite":false, + | "isUnique":false, + | "isIndexable":true, + | "reverseAttributeName":null + | }, + | { + | "name":"db", + | "dataTypeName":"DB", + | "multiplicity":{ + | "lower":1, + | "upper":1, + | "isUnique":false + | }, + | "isComposite":false, + | "isUnique":false, + | "isIndexable":true, + | "reverseAttributeName":null + | }, + | { + | "name":"sd", + | "dataTypeName":"StorageDesc", + | "multiplicity":{ + | "lower":1, + | "upper":1, + | "isUnique":false + | }, + | "isComposite":false, + | "isUnique":false, + | "isIndexable":true, + | "reverseAttributeName":null + | }, + | { + | "name":"created", + | "dataTypeName":"date", + | "multiplicity":{ + | "lower":0, + | "upper":1, + | "isUnique":false + | }, + | "isComposite":false, + | "isUnique":false, + | "isIndexable":true, + | "reverseAttributeName":null + | } + | ] + | }, + | "rows":[ + | { + | "$typeName$":"Table", + | "$id$":{ + | "$typeName$":"Table", + | "version":0 + | }, + | "created":"2014-12-11T02:35:58.440Z", + | "sd":{ + | "$typeName$":"StorageDesc", + | "version":0 + | }, + | "db":{ + | "$typeName$":"DB", + | "version":0 + | }, + | "name":"sales_fact" + | }, + | { + | "$typeName$":"Table", + | "$id$":{ + | "$typeName$":"Table", + | "version":0 + | }, + | "created":"2014-12-11T02:35:58.440Z", + | "sd":{ + | "$typeName$":"StorageDesc", + | "version":0 + | }, + | "db":{ + | "$typeName$":"DB", + | "version":0 + | }, + | "name":"product_dim", + | "$traits$":{ + | "Dimension":{ + | "$typeName$":"Dimension" + | } + | } + | }, + | { + | "$typeName$":"Table", + | "$id$":{ + | "$typeName$":"Table", + | "version":0 + | }, + | "created":"2014-12-11T02:35:58.440Z", + | "sd":{ + | "$typeName$":"StorageDesc", + | "version":0 + | }, + | "db":{ + | "$typeName$":"DB", + | "version":0 + | }, + | "name":"time_dim", + | "$traits$":{ + | "Dimension":{ + | "$typeName$":"Dimension" + | } + | } + | }, + | { + | "$typeName$":"Table", + | "$id$":{ + | "$typeName$":"Table", + | "version":0 + | }, + | "created":"2014-12-11T02:35:58.440Z", + | "sd":{ + | "$typeName$":"StorageDesc", + | "version":0 + | }, + | "db":{ + | "$typeName$":"DB", + | "version":0 + | }, + | "name":"customer_dim", + | "$traits$":{ + | "Dimension":{ + | "$typeName$":"Dimension" + | } + | } + | }, + | { + | "$typeName$":"Table", + | "$id$":{ + | "$typeName$":"Table", + | "version":0 + | }, + | "created":"2014-12-11T02:35:58.440Z", + | "sd":{ + | "$typeName$":"StorageDesc", + | "version":0 + | }, + | "db":{ + | "$typeName$":"DB", + | "version":0 + | }, + | "name":"sales_fact_daily_mv" + | }, + | { + | "$typeName$":"Table", + | "$id$":{ + | "$typeName$":"Table", + | "version":0 + | }, + | "created":"2014-12-11T02:35:58.440Z", + | "sd":{ + | "$typeName$":"StorageDesc", + | "version":0 + | }, + | "db":{ + | "$typeName$":"DB", + | "version":0 + | }, + | "name":"sales_fact_monthly_mv" + | } + | ] + |}""".stripMargin); } test("testBackReference") { @@ -98,7 +481,7 @@ class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest { test("testComparisonLogical") { val r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting")). and(id("createTime") > int(0))), g) - validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\") and (createTime > 0)\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\" createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}") + validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\") and (createTime > 0)\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\" createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7424\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}") } test("testJoinAndSelect1") { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6b33bcf6/repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala ---------------------------------------------------------------------- diff --git a/repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala b/repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala index 71cbad3..eef13df 100755 --- a/repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala +++ b/repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala @@ -19,12 +19,14 @@ package org.apache.atlas.query import java.io.File -import java.util.UUID +import java.util.{Date, UUID} import java.util.concurrent.atomic.AtomicInteger import javax.script.{Bindings, ScriptEngine, ScriptEngineManager} import com.thinkaurelius.titan.core.TitanGraph import com.typesafe.config.ConfigFactory +import org.apache.atlas.repository.BaseTest +import org.apache.atlas.typesystem.types.TypeSystem import org.apache.commons.io.FileUtils import scala.collection.mutable.ArrayBuffer @@ -54,18 +56,19 @@ object HiveTitanSample { this.getClass.getDeclaredFields filter (_.getName != "traits") foreach { f => f.setAccessible(true) - var fV = f.get(this) - fV = fV match { + val fV = f.get(this) + val convertedVal = fV match { case _: String => s""""$fV"""" + case d: Date => d.getTime case _ => fV } - fV match { + convertedVal match { case x: Vertex => addEdge(x, s"${this.getClass.getSimpleName}.${f.getName}", edges) case l: List[_] => l.foreach(x => addEdge(x.asInstanceOf[Vertex], s"${this.getClass.getSimpleName}.${f.getName}", edges)) - case _ => sb.append( s""", "${f.getName}" : $fV""") - sb.append( s""", "${this.getClass.getSimpleName}.${f.getName}" : $fV""") + case _ => sb.append( s""", "${f.getName}" : $convertedVal""") + sb.append( s""", "${this.getClass.getSimpleName}.${f.getName}" : $convertedVal""") } } @@ -127,17 +130,23 @@ object HiveTitanSample { _id: String = "" + nextVertexId.incrementAndGet()) extends Instance case class Table(name: String, db: DB, sd: StorageDescriptor, + created: Date, traits: Option[List[Trait]] = None, _id: String = "" + nextVertexId.incrementAndGet()) extends Instance case class TableDef(name: String, db: DB, inputFormat: String, outputFormat: String, columns: List[(String, String, Option[List[Trait]])], - traits: Option[List[Trait]] = None) { + traits: Option[List[Trait]] = None, + created: Option[Date] = None) { + val createdDate : Date = created match { + case Some(x) => x + case None => new Date(BaseTest.TEST_DATE_IN_LONG) + } val sd = StorageDescriptor(inputFormat, outputFormat) val colDefs = columns map { c => Column(c._1, c._2, sd, c._3) } - val tablDef = Table(name, db, sd, traits) + val tablDef = Table(name, db, sd, createdDate, traits) def toGSon(vertices: ArrayBuffer[String], edges: ArrayBuffer[String]): Unit = { @@ -167,6 +176,7 @@ object HiveTitanSample { ("time_id", "int", None), ("product_id", "int", None), ("customer_id", "int", None), + ("created", "date", None), ("sales", "double", Some(List(Metric()))) )) val productDim = TableDef("product_dim", @@ -269,6 +279,7 @@ object HiveTitanSample { FileUtils.writeStringToFile(new File(fileName), toGSon()) } + val GremlinQueries = List( // 1. List all DBs """g.V.has("typeName", "DB")""",
