http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7660c9b2/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java index db51ae5..0e6913d 100644 --- a/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java @@ -18,32 +18,10 @@ package org.apache.atlas.discovery; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.thinkaurelius.titan.core.TitanGraph; -import com.thinkaurelius.titan.core.util.TitanCleanup; +import org.apache.atlas.BaseHiveRepositoryTest; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; import org.apache.atlas.repository.EntityNotFoundException; -import org.apache.atlas.repository.graph.GraphProvider; -import org.apache.atlas.services.DefaultMetadataService; -import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.TypesDef; -import org.apache.atlas.typesystem.json.InstanceSerialization; -import org.apache.atlas.typesystem.json.TypesSerialization; -import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.types.AttributeDefinition; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.EnumTypeDefinition; -import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; -import org.apache.atlas.typesystem.types.IDataType; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.StructTypeDefinition; -import org.apache.atlas.typesystem.types.TraitType; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.apache.atlas.typesystem.types.TypeUtils; -import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONObject; import org.testng.Assert; @@ -60,37 +38,22 @@ import java.util.List; * Unit tests for Hive LineageService. */ @Guice(modules = RepositoryMetadataModule.class) -public class HiveLineageServiceTest { +public class HiveLineageServiceTest extends BaseHiveRepositoryTest { @Inject private GraphBackedDiscoveryService discoveryService; @Inject - private DefaultMetadataService metadataService; - - @Inject - private GraphProvider<TitanGraph> graphProvider; - - @Inject private HiveLineageService hiveLineageService; - @BeforeClass public void setUp() throws Exception { - setUpTypes(); - setupInstances(); - // TestUtils.dumpGraph(graphProvider.get()); + super.setUp(); } @AfterClass public void tearDown() throws Exception { - TypeSystem.getInstance().reset(); - graphProvider.get().shutdown(); - try { - TitanCleanup.clear(graphProvider.get()); - } catch (Exception e) { - e.printStackTrace(); - } + super.tearDown(); } @DataProvider(name = "dslQueriesProvider") @@ -296,248 +259,4 @@ public class HiveLineageServiceTest { hiveLineageService.getSchema("blah"); Assert.fail(); } - - private void setUpTypes() throws Exception { - TypesDef typesDef = createTypeDefinitions(); - String typesAsJSON = TypesSerialization.toJson(typesDef); - metadataService.createType(typesAsJSON); - } - - private static final String DATABASE_TYPE = "hive_db"; - private static final String HIVE_TABLE_TYPE = "hive_table"; - private static final String COLUMN_TYPE = "hive_column"; - private static final String HIVE_PROCESS_TYPE = "hive_process"; - private static final String STORAGE_DESC_TYPE = "StorageDesc"; - private static final String VIEW_TYPE = "View"; - - private TypesDef createTypeDefinitions() { - HierarchicalTypeDefinition<ClassType> dbClsDef = TypesUtil - .createClassTypeDef(DATABASE_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), - attrDef("description", DataTypes.STRING_TYPE), attrDef("locationUri", DataTypes.STRING_TYPE), - attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE)); - - HierarchicalTypeDefinition<ClassType> storageDescClsDef = TypesUtil - .createClassTypeDef(STORAGE_DESC_TYPE, null, attrDef("location", DataTypes.STRING_TYPE), - attrDef("inputFormat", DataTypes.STRING_TYPE), attrDef("outputFormat", DataTypes.STRING_TYPE), - attrDef("compressed", DataTypes.STRING_TYPE, Multiplicity.REQUIRED, false, null)); - - HierarchicalTypeDefinition<ClassType> columnClsDef = TypesUtil - .createClassTypeDef(COLUMN_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), - attrDef("dataType", DataTypes.STRING_TYPE), attrDef("comment", DataTypes.STRING_TYPE)); - - HierarchicalTypeDefinition<ClassType> tblClsDef = TypesUtil - .createClassTypeDef(HIVE_TABLE_TYPE, ImmutableList.of("DataSet"), - attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE), - attrDef("lastAccessTime", DataTypes.LONG_TYPE), attrDef("tableType", DataTypes.STRING_TYPE), - attrDef("temporary", DataTypes.BOOLEAN_TYPE), - new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null), - // todo - uncomment this, something is broken - // new AttributeDefinition("sd", STORAGE_DESC_TYPE, - // Multiplicity.REQUIRED, true, null), - new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE), - Multiplicity.COLLECTION, true, null)); - - HierarchicalTypeDefinition<ClassType> loadProcessClsDef = TypesUtil - .createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableList.of("Process"), - attrDef("userName", DataTypes.STRING_TYPE), attrDef("startTime", DataTypes.LONG_TYPE), - attrDef("endTime", DataTypes.LONG_TYPE), - attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), - attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), - attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), - attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED)); - - HierarchicalTypeDefinition<ClassType> viewClsDef = TypesUtil - .createClassTypeDef(VIEW_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), - new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null), - new AttributeDefinition("inputTables", DataTypes.arrayTypeName(HIVE_TABLE_TYPE), - Multiplicity.COLLECTION, false, null)); - - HierarchicalTypeDefinition<TraitType> dimTraitDef = TypesUtil.createTraitTypeDef("Dimension", null); - - HierarchicalTypeDefinition<TraitType> factTraitDef = TypesUtil.createTraitTypeDef("Fact", null); - - HierarchicalTypeDefinition<TraitType> metricTraitDef = TypesUtil.createTraitTypeDef("Metric", null); - - HierarchicalTypeDefinition<TraitType> etlTraitDef = TypesUtil.createTraitTypeDef("ETL", null); - - HierarchicalTypeDefinition<TraitType> piiTraitDef = TypesUtil.createTraitTypeDef("PII", null); - - HierarchicalTypeDefinition<TraitType> jdbcTraitDef = TypesUtil.createTraitTypeDef("JdbcAccess", null); - - return TypeUtils.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), - ImmutableList.of(dimTraitDef, factTraitDef, piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef), - ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef, tblClsDef, loadProcessClsDef, viewClsDef)); - } - - AttributeDefinition attrDef(String name, IDataType dT) { - return attrDef(name, dT, Multiplicity.OPTIONAL, false, null); - } - - AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m) { - return attrDef(name, dT, m, false, null); - } - - AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m, boolean isComposite, - String reverseAttributeName) { - Preconditions.checkNotNull(name); - Preconditions.checkNotNull(dT); - return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName); - } - - private void setupInstances() throws Exception { - Id salesDB = database("Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales"); - - Referenceable sd = - storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true); - - List<Referenceable> salesFactColumns = ImmutableList - .of(column("time_id", "int", "time id"), column("product_id", "int", "product id"), - column("customer_id", "int", "customer id", "PII"), - column("sales", "double", "product id", "Metric")); - - Id salesFact = table("sales_fact", "sales fact table", salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact"); - - List<Referenceable> timeDimColumns = ImmutableList - .of(column("time_id", "int", "time id"), column("dayOfYear", "int", "day Of Year"), - column("weekDay", "int", "week Day")); - - Id timeDim = table("time_dim", "time dimension table", salesDB, sd, "John Doe", "External", timeDimColumns, - "Dimension"); - - Id reportingDB = - database("Reporting", "reporting database", "Jane BI", "hdfs://host:8000/apps/warehouse/reporting"); - - Id salesFactDaily = - table("sales_fact_daily_mv", "sales fact daily materialized view", reportingDB, sd, "Joe BI", "Managed", - salesFactColumns, "Metric"); - - loadProcess("loadSalesDaily", "hive query for daily summary", "John ETL", ImmutableList.of(salesFact, timeDim), - ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL"); - - List<Referenceable> productDimColumns = ImmutableList - .of(column("product_id", "int", "product id"), column("product_name", "string", "product name"), - column("brand_name", "int", "brand name")); - - Id productDim = - table("product_dim", "product dimension table", salesDB, sd, "John Doe", "Managed", productDimColumns, - "Dimension"); - - view("product_dim_view", reportingDB, ImmutableList.of(productDim), "Dimension", "JdbcAccess"); - - List<Referenceable> customerDimColumns = ImmutableList.of(column("customer_id", "int", "customer id", "PII"), - column("name", "string", "customer name", "PII"), - column("address", "string", "customer address", "PII")); - - Id customerDim = - table("customer_dim", "customer dimension table", salesDB, sd, "fetl", "External", customerDimColumns, - "Dimension"); - - view("customer_dim_view", reportingDB, ImmutableList.of(customerDim), "Dimension", "JdbcAccess"); - - Id salesFactMonthly = - table("sales_fact_monthly_mv", "sales fact monthly materialized view", reportingDB, sd, "Jane BI", - "Managed", salesFactColumns, "Metric"); - - loadProcess("loadSalesMonthly", "hive query for monthly summary", "John ETL", ImmutableList.of(salesFactDaily), - ImmutableList.of(salesFactMonthly), "create table as select ", "plan", "id", "graph", "ETL"); - } - - Id database(String name, String description, String owner, String locationUri, String... traitNames) - throws Exception { - Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames); - referenceable.set("name", name); - referenceable.set("description", description); - referenceable.set("owner", owner); - referenceable.set("locationUri", locationUri); - referenceable.set("createTime", System.currentTimeMillis()); - - return createInstance(referenceable); - } - - Referenceable storageDescriptor(String location, String inputFormat, String outputFormat, boolean compressed) - throws Exception { - Referenceable referenceable = new Referenceable(STORAGE_DESC_TYPE); - referenceable.set("location", location); - referenceable.set("inputFormat", inputFormat); - referenceable.set("outputFormat", outputFormat); - referenceable.set("compressed", compressed); - - return referenceable; - } - - Referenceable column(String name, String dataType, String comment, String... traitNames) throws Exception { - Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames); - referenceable.set("name", name); - referenceable.set("dataType", dataType); - referenceable.set("comment", comment); - - return referenceable; - } - - Id table(String name, String description, Id dbId, Referenceable sd, String owner, String tableType, - List<Referenceable> columns, String... traitNames) throws Exception { - Referenceable referenceable = new Referenceable(HIVE_TABLE_TYPE, traitNames); - referenceable.set("name", name); - referenceable.set("description", description); - referenceable.set("owner", owner); - referenceable.set("tableType", tableType); - referenceable.set("temporary", false); - referenceable.set("createTime", System.currentTimeMillis()); - referenceable.set("lastAccessTime", System.currentTimeMillis()); - referenceable.set("retention", System.currentTimeMillis()); - - referenceable.set("db", dbId); - // todo - uncomment this, something is broken - // referenceable.set("sd", sd); - referenceable.set("columns", columns); - - return createInstance(referenceable); - } - - Id loadProcess(String name, String description, String user, List<Id> inputTables, List<Id> outputTables, - String queryText, String queryPlan, String queryId, String queryGraph, String... traitNames) - throws Exception { - Referenceable referenceable = new Referenceable(HIVE_PROCESS_TYPE, traitNames); - referenceable.set("name", name); - referenceable.set("description", description); - referenceable.set("user", user); - referenceable.set("startTime", System.currentTimeMillis()); - referenceable.set("endTime", System.currentTimeMillis() + 10000); - - referenceable.set("inputs", inputTables); - referenceable.set("outputs", outputTables); - - referenceable.set("queryText", queryText); - referenceable.set("queryPlan", queryPlan); - referenceable.set("queryId", queryId); - referenceable.set("queryGraph", queryGraph); - - return createInstance(referenceable); - } - - Id view(String name, Id dbId, List<Id> inputTables, String... traitNames) throws Exception { - Referenceable referenceable = new Referenceable(VIEW_TYPE, traitNames); - referenceable.set("name", name); - referenceable.set("db", dbId); - - referenceable.set("inputTables", inputTables); - - return createInstance(referenceable); - } - - private Id createInstance(Referenceable referenceable) throws Exception { - String typeName = referenceable.getTypeName(); - System.out.println("creating instance of type " + typeName); - - String entityJSON = InstanceSerialization.toJson(referenceable, true); - System.out.println("Submitting new entity= " + entityJSON); - JSONArray jsonArray = new JSONArray(); - jsonArray.put(entityJSON); - String response = metadataService.createEntities(jsonArray.toString()); - String guid = new JSONArray(response).getString(0); - System.out.println("created instance for type " + typeName + ", guid: " + guid); - - // return the reference to created instance with guid - return new Id(guid, 0, referenceable.getTypeName()); - } }
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7660c9b2/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java index bec3067..39be47d 100755 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -101,12 +101,12 @@ public class GraphBackedMetadataRepositoryTest { try { //TODO - Fix failure during shutdown while using BDB graphProvider.get().shutdown(); - } catch(Exception e) { + } catch (Exception e) { e.printStackTrace(); } try { TitanCleanup.clear(graphProvider.get()); - } catch(Exception e) { + } catch (Exception e) { e.printStackTrace(); } } @@ -153,8 +153,8 @@ public class GraphBackedMetadataRepositoryTest { @Test(dependsOnMethods = "testSubmitEntity") public void testGetTraitLabel() throws Exception { Assert.assertEquals( - repositoryService.getTraitLabel(typeSystem.getDataType(ClassType.class, TestUtils.TABLE_TYPE), - TestUtils.CLASSIFICATION), TestUtils.TABLE_TYPE + "." + TestUtils.CLASSIFICATION); + repositoryService.getTraitLabel(typeSystem.getDataType(ClassType.class, TestUtils.TABLE_TYPE), + TestUtils.CLASSIFICATION), TestUtils.TABLE_TYPE + "." + TestUtils.CLASSIFICATION); } @Test @@ -264,8 +264,8 @@ public class GraphBackedMetadataRepositoryTest { final String traitName = "P_I_I"; HierarchicalTypeDefinition<TraitType> piiTrait = TypesUtil - .createTraitTypeDef(traitName, ImmutableList.<String>of(), - TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE)); + .createTraitTypeDef(traitName, ImmutableList.<String>of(), + TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE)); TraitType traitType = typeSystem.defineTraitType(piiTrait); ITypedStruct traitInstance = traitType.createInstance(); traitInstance.set("type", "SSN"); @@ -414,7 +414,7 @@ public class GraphBackedMetadataRepositoryTest { @Test(dependsOnMethods = "testCreateEntity") public void testBug37860() throws Exception { String dslQuery = "hive_table as t where name = 'bar' " - + "database where name = 'foo' and description = 'foo database' select t"; + + "database where name = 'foo' and description = 'foo database' select t"; System.out.println("Executing dslQuery = " + dslQuery); String jsonResults = discoveryService.searchByDSL(dslQuery); Assert.assertNotNull(jsonResults); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7660c9b2/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java index 20781db..85ab738 100755 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java @@ -40,6 +40,7 @@ import org.apache.atlas.typesystem.types.IDataType; import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.commons.io.FileUtils; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -147,7 +148,7 @@ public class GraphRepoMapperScaleTest { searchWithOutIndex("hive_table.name", "bar-999"); searchWithIndex("hive_table.name", "bar-999"); - searchWithIndex("hive_table.created", Compare.GREATER_THAN_EQUAL, TestUtils.TEST_DATE_IN_LONG); + searchWithIndex("hive_table.created", Compare.GREATER_THAN_EQUAL, TestUtils.TEST_DATE_IN_LONG, 1000); for (int index = 500; index < 600; index++) { searchWithIndex("hive_table.name", "bar-" + index); @@ -185,7 +186,7 @@ public class GraphRepoMapperScaleTest { } } - private void searchWithIndex(String key, Predicate searchPredicate, Object value) { + private void searchWithIndex(String key, Predicate searchPredicate, Object value, int expectedResults) { TitanGraph graph = graphProvider.get(); long start = System.currentTimeMillis(); int count = 0; @@ -197,6 +198,7 @@ public class GraphRepoMapperScaleTest { } finally { System.out.println("Search on [" + key + "=" + value + "] returned results: " + count + ", took " + ( System.currentTimeMillis() - start) + " ms"); + Assert.assertEquals(count, expectedResults); } }
