http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java b/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java deleted file mode 100644 index 7c6b60b..0000000 --- a/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java +++ /dev/null @@ -1,428 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import org.apache.atlas.repository.MetadataRepository; -import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; -import org.apache.atlas.services.MetadataService; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.typesystem.ITypedReferenceableInstance; -import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.TypesDef; -import org.apache.atlas.typesystem.json.TypesSerialization; -import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.types.*; -import org.apache.atlas.typesystem.types.utils.TypesUtil; -import org.testng.annotations.Guice; - -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_INPUTS; -import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS; - -/** - * Base Class to set up hive types and instances for tests - */ -@Guice(modules = TestModules.TestOnlyModule.class) -public class BaseRepositoryTest { - - @Inject - protected MetadataService metadataService; - - @Inject - protected MetadataRepository repository; - - - protected void setUp() throws Exception { - //force graph initialization / built in type registration - TestUtils.getGraph(); - setUpDefaultTypes(); - setUpTypes(); - TestUtils.getGraph().commit(); - new GraphBackedSearchIndexer(new AtlasTypeRegistry()); - TestUtils.resetRequestContext(); - setupInstances(); - TestUtils.getGraph().commit(); - TestUtils.dumpGraph(TestUtils.getGraph()); - } - - protected void tearDown() throws Exception { - TypeSystem.getInstance().reset(); - } - - private void setUpTypes() throws Exception { - TypesDef typesDef = createTypeDefinitions(); - String typesAsJSON = TypesSerialization.toJson(typesDef); - metadataService.createType(typesAsJSON); - } - - protected static final String DATABASE_TYPE = "hive_db"; - protected static final String HIVE_TABLE_TYPE = "hive_table"; - private static final String COLUMN_TYPE = "hive_column"; - private static final String HIVE_PROCESS_TYPE = "hive_process"; - private static final String STORAGE_DESC_TYPE = "StorageDesc"; - private static final String VIEW_TYPE = "View"; - private static final String PARTITION_TYPE = "hive_partition"; - protected static final String DATASET_SUBTYPE = "dataset_subtype"; - - TypesDef createTypeDefinitions() { - HierarchicalTypeDefinition<ClassType> dbClsDef = TypesUtil - .createClassTypeDef(DATABASE_TYPE, null, - TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), - attrDef("description", DataTypes.STRING_TYPE), attrDef("locationUri", DataTypes.STRING_TYPE), - attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE)); - - HierarchicalTypeDefinition<ClassType> columnClsDef = TypesUtil - .createClassTypeDef(COLUMN_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), - attrDef("dataType", DataTypes.STRING_TYPE), attrDef("comment", DataTypes.STRING_TYPE)); - - HierarchicalTypeDefinition<ClassType> storageDescClsDef = TypesUtil - .createClassTypeDef(STORAGE_DESC_TYPE, null, - attrDef("location", DataTypes.STRING_TYPE), - attrDef("inputFormat", DataTypes.STRING_TYPE), attrDef("outputFormat", DataTypes.STRING_TYPE), - attrDef("compressed", DataTypes.STRING_TYPE, Multiplicity.REQUIRED, false, null)); - - - HierarchicalTypeDefinition<ClassType> tblClsDef = TypesUtil - .createClassTypeDef(HIVE_TABLE_TYPE, ImmutableSet.of("DataSet"), - attrDef("owner", DataTypes.STRING_TYPE), - attrDef("createTime", DataTypes.DATE_TYPE), - attrDef("lastAccessTime", DataTypes.LONG_TYPE), attrDef("tableType", DataTypes.STRING_TYPE), - attrDef("temporary", DataTypes.BOOLEAN_TYPE), - new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null), - // todo - uncomment this, something is broken - new AttributeDefinition("sd", STORAGE_DESC_TYPE, Multiplicity.REQUIRED, true, null), - new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE), - Multiplicity.COLLECTION, true, null)); - - HierarchicalTypeDefinition<ClassType> loadProcessClsDef = TypesUtil - .createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableSet.of("Process"), - attrDef("userName", DataTypes.STRING_TYPE), attrDef("startTime", DataTypes.LONG_TYPE), - attrDef("endTime", DataTypes.LONG_TYPE), - attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), - attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), - attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), - attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED)); - - HierarchicalTypeDefinition<ClassType> viewClsDef = TypesUtil - .createClassTypeDef(VIEW_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), - new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null), - new AttributeDefinition("inputTables", DataTypes.arrayTypeName(HIVE_TABLE_TYPE), - Multiplicity.COLLECTION, false, null)); - - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("values", DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()), - Multiplicity.OPTIONAL, false, null), - new AttributeDefinition("table", HIVE_TABLE_TYPE, Multiplicity.REQUIRED, false, null), - }; - HierarchicalTypeDefinition<ClassType> partClsDef = - new HierarchicalTypeDefinition<>(ClassType.class, PARTITION_TYPE, null, null, - attributeDefinitions); - - HierarchicalTypeDefinition<ClassType> datasetSubTypeClsDef = TypesUtil - .createClassTypeDef(DATASET_SUBTYPE, ImmutableSet.of("DataSet")); - - HierarchicalTypeDefinition < TraitType > dimTraitDef = TypesUtil.createTraitTypeDef("Dimension", null); - - HierarchicalTypeDefinition<TraitType> factTraitDef = TypesUtil.createTraitTypeDef("Fact", null); - - HierarchicalTypeDefinition<TraitType> metricTraitDef = TypesUtil.createTraitTypeDef("Metric", null); - - HierarchicalTypeDefinition<TraitType> etlTraitDef = TypesUtil.createTraitTypeDef("ETL", null); - - HierarchicalTypeDefinition<TraitType> piiTraitDef = TypesUtil.createTraitTypeDef("PII", null); - - HierarchicalTypeDefinition<TraitType> jdbcTraitDef = TypesUtil.createTraitTypeDef("JdbcAccess", null); - - HierarchicalTypeDefinition<TraitType> logTraitDef = TypesUtil.createTraitTypeDef("Log Data", null); - - HierarchicalTypeDefinition<TraitType> isaKeywordTraitDef = TypesUtil.createTraitTypeDef("isa", null); - - return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), - ImmutableList.of(dimTraitDef, factTraitDef, piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef, logTraitDef, - isaKeywordTraitDef), - ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef, tblClsDef, loadProcessClsDef, viewClsDef, partClsDef, datasetSubTypeClsDef)); - } - - AttributeDefinition attrDef(String name, IDataType dT) { - return attrDef(name, dT, Multiplicity.OPTIONAL, false, null); - } - - AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m) { - return attrDef(name, dT, m, false, null); - } - - AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m, boolean isComposite, - String reverseAttributeName) { - Preconditions.checkNotNull(name); - Preconditions.checkNotNull(dT); - return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName); - } - - private void setupInstances() throws Exception { - Id salesDB = database("Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales"); - - Referenceable sd = - storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of( - column("time_id", "int", "time id"))); - - List<Referenceable> salesFactColumns = ImmutableList - .of(column("time_id", "int", "time id"), - column("product_id", "int", "product id"), - column("customer_id", "int", "customer id", "PII"), - column("sales", "double", "product id", "Metric")); - - Id salesFact = table("sales_fact", "sales fact table", salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact"); - - List<Referenceable> logFactColumns = ImmutableList - .of(column("time_id", "int", "time id"), column("app_id", "int", "app id"), - column("machine_id", "int", "machine id"), column("log", "string", "log data", "Log Data")); - - List<Referenceable> timeDimColumns = ImmutableList - .of(column("time_id", "int", "time id"), - column("dayOfYear", "int", "day Of Year"), - column("weekDay", "int", "week Day")); - - Id timeDim = table("time_dim", "time dimension table", salesDB, sd, "John Doe", "External", timeDimColumns, - "Dimension"); - - Id reportingDB = - database("Reporting", "reporting database", "Jane BI", "hdfs://host:8000/apps/warehouse/reporting"); - - Id salesFactDaily = - table("sales_fact_daily_mv", "sales fact daily materialized view", reportingDB, sd, "Joe BI", "Managed", - salesFactColumns, "Metric"); - - Id circularLineageTable1 = table("table1", "", reportingDB, sd, "Vimal", "Managed", salesFactColumns, "Metric"); - - Id circularLineageTable2 = table("table2", "", reportingDB, sd, "Vimal", "Managed", salesFactColumns, "Metric"); - - loadProcess("circularLineage1", "hive query for daily summary", "John ETL", ImmutableList.of(circularLineageTable1), - ImmutableList.of(circularLineageTable2), "create table as select ", "plan", "id", "graph", "ETL"); - - loadProcess("circularLineage2", "hive query for daily summary", "John ETL", ImmutableList.of(circularLineageTable2), - ImmutableList.of(circularLineageTable1), "create table as select ", "plan", "id", "graph", "ETL"); - - loadProcess("loadSalesDaily", "hive query for daily summary", "John ETL", ImmutableList.of(salesFact, timeDim), - ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL"); - - Id logDB = database("Logging", "logging database", "Tim ETL", "hdfs://host:8000/apps/warehouse/logging"); - - Id loggingFactDaily = - table("log_fact_daily_mv", "log fact daily materialized view", logDB, sd, "Tim ETL", "Managed", - logFactColumns, "Log Data"); - - List<Referenceable> productDimColumns = ImmutableList - .of(column("product_id", "int", "product id"), - column("product_name", "string", "product name"), - column("brand_name", "int", "brand name")); - - Id productDim = - table("product_dim", "product dimension table", salesDB, sd, "John Doe", "Managed", productDimColumns, - "Dimension"); - - view("product_dim_view", reportingDB, ImmutableList.of(productDim), "Dimension", "JdbcAccess"); - - List<Referenceable> customerDimColumns = ImmutableList.of( - column("customer_id", "int", "customer id", "PII"), - column("name", "string", "customer name", "PII"), - column("address", "string", "customer address", "PII")); - - Id customerDim = - table("customer_dim", "customer dimension table", salesDB, sd, "fetl", "External", customerDimColumns, - "Dimension"); - - view("customer_dim_view", reportingDB, ImmutableList.of(customerDim), "Dimension", "JdbcAccess"); - - Id salesFactMonthly = - table("sales_fact_monthly_mv", "sales fact monthly materialized view", reportingDB, sd, "Jane BI", - "Managed", salesFactColumns, "Metric"); - - loadProcess("loadSalesMonthly", "hive query for monthly summary", "John ETL", ImmutableList.of(salesFactDaily), - ImmutableList.of(salesFactMonthly), "create table as select ", "plan", "id", "graph", "ETL"); - - Id loggingFactMonthly = - table("logging_fact_monthly_mv", "logging fact monthly materialized view", logDB, sd, "Tim ETL", - "Managed", logFactColumns, "Log Data"); - - loadProcess("loadLogsMonthly", "hive query for monthly summary", "Tim ETL", ImmutableList.of(loggingFactDaily), - ImmutableList.of(loggingFactMonthly), "create table as select ", "plan", "id", "graph", "ETL"); - - partition(new ArrayList() {{ add("2015-01-01"); }}, salesFactDaily); - - datasetSubType("dataSetSubTypeInst1", "testOwner"); - } - - Id database(String name, String description, String owner, String locationUri, String... traitNames) - throws Exception { - Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames); - referenceable.set("name", name); - referenceable.set("description", description); - referenceable.set("owner", owner); - referenceable.set("locationUri", locationUri); - referenceable.set("createTime", System.currentTimeMillis()); - - ClassType clsType = TypeSystem.getInstance().getDataType(ClassType.class, DATABASE_TYPE); - return createInstance(referenceable, clsType); - } - - protected Referenceable storageDescriptor(String location, String inputFormat, String outputFormat, boolean compressed, List<Referenceable> columns) - throws Exception { - Referenceable referenceable = new Referenceable(STORAGE_DESC_TYPE); - referenceable.set("location", location); - referenceable.set("inputFormat", inputFormat); - referenceable.set("outputFormat", outputFormat); - referenceable.set("compressed", compressed); - referenceable.set("cols", columns); - - return referenceable; - } - - protected Referenceable column(String name, String dataType, String comment, String... traitNames) throws Exception { - Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames); - referenceable.set("name", name); - referenceable.set("dataType", dataType); - referenceable.set("comment", comment); - - return referenceable; - } - - protected Id table(String name, String description, Id dbId, Referenceable sd, String owner, String tableType, - List<Referenceable> columns, String... traitNames) throws Exception { - Referenceable referenceable = new Referenceable(HIVE_TABLE_TYPE, traitNames); - referenceable.set("name", name); - referenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, "qualified:" + name); - referenceable.set("description", description); - referenceable.set("owner", owner); - referenceable.set("tableType", tableType); - referenceable.set("temporary", false); - referenceable.set("createTime", new Date(System.currentTimeMillis())); - referenceable.set("lastAccessTime", System.currentTimeMillis()); - referenceable.set("retention", System.currentTimeMillis()); - - referenceable.set("db", dbId); - // todo - uncomment this, something is broken - referenceable.set("sd", sd); - referenceable.set("columns", columns); - - ClassType clsType = TypeSystem.getInstance().getDataType(ClassType.class, HIVE_TABLE_TYPE); - return createInstance(referenceable, clsType); - } - - protected Id loadProcess(String name, String description, String user, List<Id> inputTables, List<Id> outputTables, - String queryText, String queryPlan, String queryId, String queryGraph, String... traitNames) - throws Exception { - Referenceable referenceable = new Referenceable(HIVE_PROCESS_TYPE, traitNames); - referenceable.set("name", name); - referenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name); - referenceable.set("description", description); - referenceable.set("user", user); - referenceable.set("startTime", System.currentTimeMillis()); - referenceable.set("endTime", System.currentTimeMillis() + 10000); - - referenceable.set("inputs", inputTables); - referenceable.set("outputs", outputTables); - - referenceable.set("queryText", queryText); - referenceable.set("queryPlan", queryPlan); - referenceable.set("queryId", queryId); - referenceable.set("queryGraph", queryGraph); - - ClassType clsType = TypeSystem.getInstance().getDataType(ClassType.class, HIVE_PROCESS_TYPE); - return createInstance(referenceable, clsType); - } - - Id view(String name, Id dbId, List<Id> inputTables, String... traitNames) throws Exception { - Referenceable referenceable = new Referenceable(VIEW_TYPE, traitNames); - referenceable.set("name", name); - referenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name); - referenceable.set("db", dbId); - - referenceable.set("inputTables", inputTables); - ClassType clsType = TypeSystem.getInstance().getDataType(ClassType.class, VIEW_TYPE); - return createInstance(referenceable, clsType); - } - - Id partition(List<String> values, Id table, String... traitNames) throws Exception { - Referenceable referenceable = new Referenceable(PARTITION_TYPE, traitNames); - referenceable.set("values", values); - referenceable.set("table", table); - ClassType clsType = TypeSystem.getInstance().getDataType(ClassType.class, PARTITION_TYPE); - return createInstance(referenceable, clsType); - } - - Id datasetSubType(final String name, String owner) throws Exception { - Referenceable referenceable = new Referenceable(DATASET_SUBTYPE); - referenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name); - referenceable.set(AtlasClient.NAME, name); - referenceable.set("owner", owner); - ClassType clsType = TypeSystem.getInstance().getDataType(ClassType.class, DATASET_SUBTYPE); - return createInstance(referenceable, clsType); - } - private Id createInstance(Referenceable referenceable, ClassType clsType) throws Exception { - ITypedReferenceableInstance typedInstance = clsType.convert(referenceable, Multiplicity.REQUIRED); - List<String> guids = repository.createEntities(typedInstance).getCreatedEntities(); - - // return the reference to created instance with guid - return new Id(guids.get(guids.size() - 1), 0, referenceable.getTypeName()); - } - - private void setUpDefaultTypes() throws Exception { - TypesDef typesDef = createDefaultTypeDefinitions(); - String typesAsJSON = TypesSerialization.toJson(typesDef); - metadataService.createType(typesAsJSON); - } - - TypesDef createDefaultTypeDefinitions() { - HierarchicalTypeDefinition<ClassType> referenceableType = TypesUtil - .createClassTypeDef(AtlasClient.REFERENCEABLE_SUPER_TYPE, ImmutableSet.<String>of(), - new AttributeDefinition(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, true, true, null)); - - HierarchicalTypeDefinition<ClassType> assetType = TypesUtil - .createClassTypeDef(AtlasClient.ASSET_TYPE, ImmutableSet.<String>of(), - new AttributeDefinition(AtlasClient.NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null), - TypesUtil.createOptionalAttrDef(AtlasClient.DESCRIPTION, DataTypes.STRING_TYPE), - new AttributeDefinition(AtlasClient.OWNER, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null)); - - HierarchicalTypeDefinition<ClassType> infraType = TypesUtil - .createClassTypeDef(AtlasClient.INFRASTRUCTURE_SUPER_TYPE, - ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE)); - - HierarchicalTypeDefinition<ClassType> datasetType = TypesUtil - .createClassTypeDef(AtlasClient.DATA_SET_SUPER_TYPE, - ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE)); - - HierarchicalTypeDefinition<ClassType> processType = TypesUtil - .createClassTypeDef(AtlasClient.PROCESS_SUPER_TYPE, - ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE), - new AttributeDefinition(PROCESS_ATTRIBUTE_INPUTS, DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE), - Multiplicity.OPTIONAL, false, null), - new AttributeDefinition(PROCESS_ATTRIBUTE_OUTPUTS, DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE), - Multiplicity.OPTIONAL, false, null)); - - return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), - ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), - ImmutableList.of(referenceableType, assetType, infraType, datasetType, processType)); - } -}
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/DBSandboxer.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/DBSandboxer.java b/repository/src/test/java/org/apache/atlas/DBSandboxer.java deleted file mode 100644 index f4f099a..0000000 --- a/repository/src/test/java/org/apache/atlas/DBSandboxer.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas; - -import org.apache.atlas.graph.GraphSandboxUtil; -import org.apache.atlas.repository.graph.AtlasGraphProvider; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.testng.ITestContext; -import org.testng.TestListenerAdapter; -import org.testng.xml.XmlClass; - -import java.util.List; - -public class DBSandboxer extends TestListenerAdapter { - @Override - public void onStart(ITestContext context) { - // This will only work if each test is run individually (test suite has only one running test) - // If there are multiple tests the the sandbox folder name is not provided and the GraphSandboxUtil provisions - // a unique name - List<XmlClass> testClassesToRun = context.getCurrentXmlTest().getClasses(); - if (CollectionUtils.isNotEmpty(testClassesToRun) && 1 == testClassesToRun.size()) { - XmlClass currentTestClass = testClassesToRun.get(0); - if (null != currentTestClass && StringUtils.isNotEmpty(currentTestClass.getName())) { - GraphSandboxUtil.create(currentTestClass.getName()); - } else { - GraphSandboxUtil.create(); - } - } else { - GraphSandboxUtil.create(); - } - } - - @Override - public void onFinish(ITestContext context) { - AtlasGraphProvider.cleanup(); - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java b/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java deleted file mode 100755 index 506d11c..0000000 --- a/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas; - -import org.apache.atlas.repository.graph.AtlasGraphProvider; -import org.testng.Assert; -import org.testng.annotations.Guice; -import org.testng.annotations.Test; - -/** - * Unit test for Guice injector service loading - * - * Uses TestNG's Guice annotation to load the necessary modules and inject the - * objects from Guice - */ -@Guice(modules = TestModules.TestOnlyModule.class) -public class RepositoryServiceLoadingTest { - - @Test - public void testGetGraphService() throws Exception { - Assert.assertNotNull(AtlasGraphProvider.getGraphInstance()); - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/TestModules.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/TestModules.java b/repository/src/test/java/org/apache/atlas/TestModules.java index 144f618..5c9e60f 100644 --- a/repository/src/test/java/org/apache/atlas/TestModules.java +++ b/repository/src/test/java/org/apache/atlas/TestModules.java @@ -26,24 +26,14 @@ import com.google.inject.multibindings.Multibinder; import org.apache.atlas.annotation.GraphTransaction; import org.apache.atlas.discovery.AtlasDiscoveryService; import org.apache.atlas.discovery.AtlasLineageService; -import org.apache.atlas.discovery.DataSetLineageService; -import org.apache.atlas.discovery.DiscoveryService; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.discovery.EntityLineageService; -import org.apache.atlas.discovery.LineageService; -import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; import org.apache.atlas.graph.GraphSandboxUtil; import org.apache.atlas.listener.EntityChangeListener; import org.apache.atlas.listener.TypeDefChangeListener; -import org.apache.atlas.listener.TypesChangeListener; -import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.audit.EntityAuditListener; import org.apache.atlas.repository.audit.EntityAuditRepository; -import org.apache.atlas.repository.graph.DeleteHandler; -import org.apache.atlas.repository.graph.GraphBackedMetadataRepository; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; -import org.apache.atlas.repository.graph.HardDeleteHandler; -import org.apache.atlas.repository.graph.SoftDeleteHandler; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.impexp.ExportService; import org.apache.atlas.repository.store.graph.AtlasEntityStore; @@ -58,16 +48,9 @@ import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1; import org.apache.atlas.repository.store.graph.v1.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v1.HardDeleteHandlerV1; import org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1; -import org.apache.atlas.repository.typestore.GraphBackedTypeStore; -import org.apache.atlas.repository.typestore.ITypeStore; -import org.apache.atlas.repository.typestore.StoreBackedTypeCache; import org.apache.atlas.service.Service; -import org.apache.atlas.services.DefaultMetadataService; -import org.apache.atlas.services.MetadataService; import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.apache.atlas.typesystem.types.cache.TypeCache; import org.apache.atlas.util.AtlasRepositoryConfiguration; import org.apache.atlas.util.SearchTracker; import org.apache.commons.configuration.Configuration; @@ -89,13 +72,6 @@ public class TestModules { private static final Logger LOG = LoggerFactory.getLogger(TestOnlyModule.class); - static class TypeSystemProvider implements Provider<TypeSystem> { - @Override - public TypeSystem get() { - return TypeSystem.getInstance(); - } - } - static class AtlasConfigurationProvider implements Provider<Configuration> { @Override @@ -126,32 +102,19 @@ public class TestModules { bind(AtlasGraph.class).toProvider(AtlasGraphProvider.class); - // allow for dynamic binding of the metadata repo & graph service - // bind the MetadataRepositoryService interface to an implementation - bind(MetadataRepository.class).to(GraphBackedMetadataRepository.class).asEagerSingleton(); - - bind(TypeSystem.class).toProvider(TypeSystemProvider.class).in(Singleton.class); + // allow for dynamic binding of graph service bind(Configuration.class).toProvider(AtlasConfigurationProvider.class).in(Singleton.class); - // bind the ITypeStore interface to an implementation - bind(ITypeStore.class).to(GraphBackedTypeStore.class).asEagerSingleton(); + // bind the AtlasTypeDefStore interface to an implementation bind(AtlasTypeDefStore.class).to(AtlasTypeDefGraphStoreV1.class).asEagerSingleton(); bind(AtlasTypeRegistry.class).asEagerSingleton(); bind(EntityGraphMapper.class).asEagerSingleton(); bind(ExportService.class).asEagerSingleton(); - //GraphBackedSearchIndexer must be an eager singleton to force the search index creation to happen before - //we try to restore the type system (otherwise we'll end up running queries - //before we have any indices during the initial graph setup) - Multibinder<TypesChangeListener> typesChangeListenerBinder = - Multibinder.newSetBinder(binder(), TypesChangeListener.class); - typesChangeListenerBinder.addBinding().to(GraphBackedSearchIndexer.class).asEagerSingleton(); - // New typesdef/instance change listener should also be bound to the corresponding implementation Multibinder<TypeDefChangeListener> typeDefChangeListenerMultibinder = Multibinder.newSetBinder(binder(), TypeDefChangeListener.class); - typeDefChangeListenerMultibinder.addBinding().to(DefaultMetadataService.class); typeDefChangeListenerMultibinder.addBinding().to(GraphBackedSearchIndexer.class).asEagerSingleton(); bind(SearchTracker.class).asEagerSingleton(); @@ -159,19 +122,12 @@ public class TestModules { bind(AtlasEntityStore.class).to(AtlasEntityStoreV1.class); bind(AtlasRelationshipStore.class).to(AtlasRelationshipStoreV1.class); - // bind the MetadataService interface to an implementation - bind(MetadataService.class).to(DefaultMetadataService.class).asEagerSingleton(); - // bind the DiscoveryService interface to an implementation - bind(DiscoveryService.class).to(GraphBackedDiscoveryService.class).asEagerSingleton(); bind(AtlasDiscoveryService.class).to(EntityDiscoveryService.class).asEagerSingleton(); - bind(LineageService.class).to(DataSetLineageService.class).asEagerSingleton(); bind(AtlasLineageService.class).to(EntityLineageService.class).asEagerSingleton(); bind(BulkImporter.class).to(BulkImporterImpl.class).asEagerSingleton(); - bindTypeCache(); - //Add EntityAuditListener as EntityChangeListener Multibinder<EntityChangeListener> entityChangeListenerBinder = Multibinder.newSetBinder(binder(), EntityChangeListener.class); @@ -182,12 +138,7 @@ public class TestModules { bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), graphTransactionInterceptor); } - protected void bindTypeCache() { - bind(TypeCache.class).to(AtlasRepositoryConfiguration.getTypeCache()).asEagerSingleton(); - } - protected void bindDeleteHandler(Binder binder) { - binder.bind(DeleteHandler.class).to(AtlasRepositoryConfiguration.getDeleteHandlerImpl()).asEagerSingleton(); binder.bind(DeleteHandlerV1.class).to(AtlasRepositoryConfiguration.getDeleteHandlerV1Impl()).asEagerSingleton(); } @@ -210,7 +161,6 @@ public class TestModules { public static class SoftDeleteModule extends TestOnlyModule { @Override protected void bindDeleteHandler(Binder binder) { - bind(DeleteHandler.class).to(SoftDeleteHandler.class).asEagerSingleton(); bind(DeleteHandlerV1.class).to(SoftDeleteHandlerV1.class).asEagerSingleton(); bind(AtlasEntityChangeNotifier.class).toProvider(MockNotifier.class); } @@ -219,20 +169,8 @@ public class TestModules { public static class HardDeleteModule extends TestOnlyModule { @Override protected void bindDeleteHandler(Binder binder) { - bind(DeleteHandler.class).to(HardDeleteHandler.class).asEagerSingleton(); bind(DeleteHandlerV1.class).to(HardDeleteHandlerV1.class).asEagerSingleton(); bind(AtlasEntityChangeNotifier.class).toProvider(MockNotifier.class); } } - - /** - * Guice module which sets TypeCache implementation class configuration property to {@link StoreBackedTypeCache}. - * - */ - public static class StoreBackedTypeCacheTestModule extends TestOnlyModule { - @Override - protected void bindTypeCache() { - bind(TypeCache.class).to(StoreBackedTypeCache.class).asEagerSingleton(); - } - } } http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/TestUtils.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java deleted file mode 100755 index 56bfb82..0000000 --- a/repository/src/test/java/org/apache/atlas/TestUtils.java +++ /dev/null @@ -1,778 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import org.apache.atlas.annotation.GraphTransaction; -import org.apache.atlas.listener.EntityChangeListener; -import org.apache.atlas.listener.TypesChangeListener; -import org.apache.atlas.repository.MetadataRepository; -import org.apache.atlas.repository.audit.InMemoryEntityAuditRepository; -import org.apache.atlas.repository.graph.AtlasGraphProvider; -import org.apache.atlas.repository.graph.GraphBackedMetadataRepository; -import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; -import org.apache.atlas.repository.graph.GraphHelper; -import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.repository.graphdb.GremlinVersion; -import org.apache.atlas.repository.typestore.GraphBackedTypeStore; -import org.apache.atlas.repository.typestore.ITypeStore; -import org.apache.atlas.services.DefaultMetadataService; -import org.apache.atlas.services.MetadataService; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.typesystem.IInstance; -import org.apache.atlas.typesystem.ITypedReferenceableInstance; -import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.TypesDef; -import org.apache.atlas.typesystem.json.InstanceSerialization; -import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.types.*; -import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; -import org.apache.atlas.typesystem.types.cache.DefaultTypeCache; -import org.apache.atlas.typesystem.types.cache.TypeCache; -import org.apache.atlas.typesystem.types.utils.TypesUtil; -import org.apache.atlas.util.AtlasRepositoryConfiguration; -import org.apache.commons.configuration.Configuration; -import org.apache.commons.lang.RandomStringUtils; -import org.codehaus.jettison.json.JSONArray; -import org.testng.Assert; -import org.testng.SkipException; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.lang.reflect.InvocationHandler; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.Proxy; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.apache.atlas.typesystem.types.utils.TypesUtil.*; -import static org.testng.Assert.assertEquals; - -/** - * Test utility class. - */ -public final class TestUtils { - - public static final long TEST_DATE_IN_LONG = 1418265358440L; - - - public static final String EMPLOYEES_ATTR = "employees"; - public static final String DEPARTMENT_ATTR = "department"; - public static final String ASSETS_ATTR = "assets"; - - public static final String POSITIONS_ATTR = "positions"; - public static final String ASSET_TYPE = "TestAsset"; - - public static final String DATABASE_TYPE = "hive_database"; - public static final String DATABASE_NAME = "foo"; - public static final String TABLE_TYPE = "hive_table"; - public static final String PROCESS_TYPE = "hive_process"; - public static final String COLUMN_TYPE = "column_type"; - public static final String TABLE_NAME = "bar"; - public static final String CLASSIFICATION = "classification"; - public static final String PII = "PII"; - public static final String SUPER_TYPE_NAME = "Base"; - public static final String STORAGE_DESC_TYPE = "hive_storagedesc"; - public static final String PARTITION_STRUCT_TYPE = "partition_struct_type"; - public static final String PARTITION_CLASS_TYPE = "partition_class_type"; - public static final String SERDE_TYPE = "serdeType"; - public static final String COLUMNS_MAP = "columnsMap"; - public static final String COLUMNS_ATTR_NAME = "columns"; - - public static final String NAME = "name"; - - private TestUtils() { - } - - /** - * Dumps the graph in GSON format in the path returned. - * - * @param graph handle to graph - * @return path to the dump file - * @throws Exception - */ - public static String dumpGraph(AtlasGraph<?,?> graph) throws Exception { - File tempFile = File.createTempFile("graph", ".gson"); - System.out.println("tempFile.getPath() = " + tempFile.getPath()); - GraphHelper.dumpToLog(graph); - FileOutputStream os = null; - try { - os = new FileOutputStream(tempFile); - graph.exportToGson(os); - } - finally { - if(os != null) { - try { - os.close(); - } - catch(IOException e) { - e.printStackTrace(); - } - } - } - - return tempFile.getPath(); - } - - /** - * Class Hierarchy is: - * Department(name : String, employees : Array[Person]) - * Person(name : String, department : Department, manager : Manager) - * Manager(subordinates : Array[Person]) extends Person - * <p/> - * Persons can have SecurityClearance(level : Int) clearance. - */ - public static void defineDeptEmployeeTypes(TypeSystem ts) throws AtlasException { - - String _description = "_description"; - EnumTypeDefinition orgLevelEnum = - new EnumTypeDefinition("OrgLevel", "OrgLevel"+_description, new EnumValue("L1", 1), new EnumValue("L2", 2)); - - StructTypeDefinition addressDetails = - createStructTypeDef("Address", "Address"+_description, createRequiredAttrDef("street", DataTypes.STRING_TYPE), - createRequiredAttrDef("city", DataTypes.STRING_TYPE)); - - HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, ImmutableSet.<String>of(), - createRequiredAttrDef(NAME, DataTypes.STRING_TYPE), - new AttributeDefinition(EMPLOYEES_ATTR, String.format("array<%s>", "Person"), Multiplicity.OPTIONAL, - true, DEPARTMENT_ATTR), - new AttributeDefinition(POSITIONS_ATTR, String.format("map<%s,%s>", DataTypes.STRING_TYPE.getName(), "Person"), Multiplicity.OPTIONAL, - false, null) - ); - - HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", "Person"+_description, ImmutableSet.<String>of(), - createRequiredAttrDef(NAME, DataTypes.STRING_TYPE), - createOptionalAttrDef("orgLevel", "OrgLevel"), - createOptionalAttrDef("address", "Address"), - new AttributeDefinition(DEPARTMENT_ATTR, "Department", Multiplicity.REQUIRED, false, EMPLOYEES_ATTR), - new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates"), - new AttributeDefinition("mentor", "Person", Multiplicity.OPTIONAL, false, null), - new AttributeDefinition(ASSETS_ATTR, String.format("array<%s>", ASSET_TYPE) , Multiplicity.OPTIONAL, false, null), - createOptionalAttrDef("birthday", DataTypes.DATE_TYPE), - createOptionalAttrDef("hasPets", DataTypes.BOOLEAN_TYPE), - createOptionalAttrDef("numberOfCars", DataTypes.BYTE_TYPE), - createOptionalAttrDef("houseNumber", DataTypes.SHORT_TYPE), - createOptionalAttrDef("carMileage", DataTypes.INT_TYPE), - createOptionalAttrDef("shares", DataTypes.LONG_TYPE), - createOptionalAttrDef("salary", DataTypes.DOUBLE_TYPE), - createOptionalAttrDef("age", DataTypes.FLOAT_TYPE), - createOptionalAttrDef("numberOfStarsEstimate", DataTypes.BIGINTEGER_TYPE), - createOptionalAttrDef("approximationOfPi", DataTypes.BIGDECIMAL_TYPE), - createOptionalAttrDef("isOrganDonor", DataTypes.BOOLEAN_TYPE) - ); - - - HierarchicalTypeDefinition<ClassType> assetTypeDef = createClassTypeDef(ASSET_TYPE, "Asset"+_description, ImmutableSet.<String>of(), - createRequiredAttrDef(NAME, DataTypes.STRING_TYPE), - new AttributeDefinition("childAssets", String.format("array<%s>", ASSET_TYPE) , Multiplicity.OPTIONAL, false, null) - ); - - HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager", "Manager"+_description, ImmutableSet.of("Person"), - new AttributeDefinition("subordinates", String.format("array<%s>", "Person"), Multiplicity.COLLECTION, - false, "manager")); - - HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = - createTraitTypeDef("SecurityClearance", "SecurityClearance"+_description, ImmutableSet.<String>of(), - createRequiredAttrDef("level", DataTypes.INT_TYPE)); - - ts.defineTypes(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails), - ImmutableList.of(securityClearanceTypeDef), - ImmutableList.of(deptTypeDef, personTypeDef, managerTypeDef, assetTypeDef)); - } - - public static final String DEPARTMENT_TYPE = "Department"; - public static final String PERSON_TYPE = "Person"; - - public static ITypedReferenceableInstance createDeptEg1(TypeSystem ts) throws AtlasException { - Referenceable hrDept = new Referenceable(DEPARTMENT_TYPE); - Referenceable john = new Referenceable(PERSON_TYPE); - - Referenceable jane = new Referenceable("Manager", "SecurityClearance"); - Referenceable johnAddr = new Referenceable("Address"); - Referenceable janeAddr = new Referenceable("Address"); - Referenceable julius = new Referenceable("Manager"); - Referenceable juliusAddr = new Referenceable("Address"); - Referenceable max = new Referenceable("Person"); - Referenceable maxAddr = new Referenceable("Address"); - - hrDept.set(NAME, "hr"); - john.set(NAME, "John"); - john.set(DEPARTMENT_ATTR, hrDept); - johnAddr.set("street", "Stewart Drive"); - johnAddr.set("city", "Sunnyvale"); - john.set("address", johnAddr); - - john.set("birthday",new Date(1950, 5, 15)); - john.set("isOrganDonor", true); - john.set("hasPets", true); - john.set("numberOfCars", 1); - john.set("houseNumber", 153); - john.set("carMileage", 13364); - john.set("shares", 15000); - john.set("salary", 123345.678); - john.set("age", 50); - john.set("numberOfStarsEstimate", new BigInteger("1000000000000000000000")); - john.set("approximationOfPi", new BigDecimal("3.141592653589793238462643383279502884197169399375105820974944592307816406286")); - - jane.set(NAME, "Jane"); - jane.set(DEPARTMENT_ATTR, hrDept); - janeAddr.set("street", "Great America Parkway"); - janeAddr.set("city", "Santa Clara"); - jane.set("address", janeAddr); - janeAddr.set("street", "Great America Parkway"); - - julius.set(NAME, "Julius"); - julius.set(DEPARTMENT_ATTR, hrDept); - juliusAddr.set("street", "Madison Ave"); - juliusAddr.set("city", "Newtonville"); - julius.set("address", juliusAddr); - julius.set("subordinates", ImmutableList.<Referenceable>of()); - - max.set(NAME, "Max"); - max.set(DEPARTMENT_ATTR, hrDept); - maxAddr.set("street", "Ripley St"); - maxAddr.set("city", "Newton"); - max.set("address", maxAddr); - max.set("manager", jane); - max.set("mentor", julius); - max.set("birthday",new Date(1979, 3, 15)); - max.set("isOrganDonor", true); - max.set("hasPets", true); - max.set("age", 36); - max.set("numberOfCars", 2); - max.set("houseNumber", 17); - max.set("carMileage", 13); - max.set("shares", Long.MAX_VALUE); - max.set("salary", Double.MAX_VALUE); - max.set("numberOfStarsEstimate", new BigInteger("1000000000000000000000000000000")); - max.set("approximationOfPi", new BigDecimal("3.1415926535897932")); - - john.set("manager", jane); - john.set("mentor", max); - hrDept.set(EMPLOYEES_ATTR, ImmutableList.of(john, jane, julius, max)); - - jane.set("subordinates", ImmutableList.of(john, max)); - - jane.getTrait("SecurityClearance").set("level", 1); - - ClassType deptType = ts.getDataType(ClassType.class, "Department"); - ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED); - Assert.assertNotNull(hrDept2); - - return hrDept2; - } - - - - public static TypesDef simpleType(){ - HierarchicalTypeDefinition<ClassType> superTypeDefinition = - createClassTypeDef("h_type", ImmutableSet.<String>of(), - createOptionalAttrDef("attr", DataTypes.STRING_TYPE)); - - StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType", - new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE)}); - - HierarchicalTypeDefinition<TraitType> traitTypeDefinition = - createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of()); - - EnumValue values[] = {new EnumValue("ONE", 1),}; - - EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("e_type", "enumType", values); - return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition), - ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition)); - } - - public static TypesDef simpleTypeUpdated(){ - HierarchicalTypeDefinition<ClassType> superTypeDefinition = - createClassTypeDef("h_type", ImmutableSet.<String>of(), - createOptionalAttrDef("attr", DataTypes.STRING_TYPE)); - - HierarchicalTypeDefinition<ClassType> newSuperTypeDefinition = - createClassTypeDef("new_h_type", ImmutableSet.<String>of(), - createOptionalAttrDef("attr", DataTypes.STRING_TYPE)); - - StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType", - new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE)}); - - HierarchicalTypeDefinition<TraitType> traitTypeDefinition = - createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of()); - - EnumValue values[] = {new EnumValue("ONE", 1),}; - - EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("e_type", "enumType", values); - return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition), - ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition, newSuperTypeDefinition)); - } - - public static TypesDef simpleTypeUpdatedDiff() { - HierarchicalTypeDefinition<ClassType> newSuperTypeDefinition = - createClassTypeDef("new_h_type", ImmutableSet.<String>of(), - createOptionalAttrDef("attr", DataTypes.STRING_TYPE)); - - return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), - ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), ImmutableList.of(newSuperTypeDefinition)); - } - - public static TypesDef defineHiveTypes() { - String _description = "_description"; - HierarchicalTypeDefinition<ClassType> superTypeDefinition = - createClassTypeDef(SUPER_TYPE_NAME, ImmutableSet.<String>of(), - createOptionalAttrDef("namespace", DataTypes.STRING_TYPE), - createOptionalAttrDef("cluster", DataTypes.STRING_TYPE), - createOptionalAttrDef("colo", DataTypes.STRING_TYPE)); - HierarchicalTypeDefinition<ClassType> databaseTypeDefinition = - createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description,ImmutableSet.of(SUPER_TYPE_NAME), - TypesUtil.createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE), - createOptionalAttrDef("created", DataTypes.DATE_TYPE), - createOptionalAttrDef("isReplicated", DataTypes.BOOLEAN_TYPE), - new AttributeDefinition("parameters", new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.OPTIONAL, false, null), - createRequiredAttrDef("description", DataTypes.STRING_TYPE)); - - - StructTypeDefinition structTypeDefinition = new StructTypeDefinition("serdeType", "serdeType" + _description, - new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE), - createRequiredAttrDef("serde", DataTypes.STRING_TYPE), - createOptionalAttrDef("description", DataTypes.STRING_TYPE)}); - - EnumValue values[] = {new EnumValue("MANAGED", 1), new EnumValue("EXTERNAL", 2),}; - - EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", "tableType" + _description, values); - - HierarchicalTypeDefinition<ClassType> columnsDefinition = - createClassTypeDef(COLUMN_TYPE, ImmutableSet.<String>of(), - createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE), - createRequiredAttrDef("type", DataTypes.STRING_TYPE)); - - StructTypeDefinition partitionDefinition = new StructTypeDefinition("partition_struct_type", "partition_struct_type" + _description, - new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE),}); - - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("location", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, - null), - new AttributeDefinition("inputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, - null), - new AttributeDefinition("outputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, - null), - new AttributeDefinition("compressed", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.REQUIRED, false, - null), - new AttributeDefinition("numBuckets", DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, - null), - }; - - HierarchicalTypeDefinition<ClassType> storageDescClsDef = - new HierarchicalTypeDefinition<>(ClassType.class, STORAGE_DESC_TYPE, STORAGE_DESC_TYPE + _description, - ImmutableSet.of(SUPER_TYPE_NAME), attributeDefinitions); - - AttributeDefinition[] partClsAttributes = new AttributeDefinition[]{ - new AttributeDefinition("values", DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()), - Multiplicity.OPTIONAL, false, null), - new AttributeDefinition("table", TABLE_TYPE, Multiplicity.REQUIRED, false, null), - new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(), Multiplicity.OPTIONAL, false, - null), - new AttributeDefinition("lastAccessTime", DataTypes.LONG_TYPE.getName(), Multiplicity.OPTIONAL, false, - null), - new AttributeDefinition("sd", STORAGE_DESC_TYPE, Multiplicity.REQUIRED, true, - null), - new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE), - Multiplicity.OPTIONAL, true, null), - new AttributeDefinition("parameters", new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.OPTIONAL, false, null),}; - - HierarchicalTypeDefinition<ClassType> partClsDef = - new HierarchicalTypeDefinition<>(ClassType.class, "partition_class_type", "partition_class_type" + _description, - ImmutableSet.of(SUPER_TYPE_NAME), partClsAttributes); - - HierarchicalTypeDefinition<ClassType> processClsType = - new HierarchicalTypeDefinition<>(ClassType.class, PROCESS_TYPE, PROCESS_TYPE + _description, - ImmutableSet.<String>of(), new AttributeDefinition[]{ - new AttributeDefinition("outputs", "array<" + TABLE_TYPE + ">", Multiplicity.OPTIONAL, false, null) - }); - - HierarchicalTypeDefinition<ClassType> tableTypeDefinition = - createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, ImmutableSet.of(SUPER_TYPE_NAME), - TypesUtil.createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE), - createRequiredAttrDef("description", DataTypes.STRING_TYPE), - createRequiredAttrDef("type", DataTypes.STRING_TYPE), - createOptionalAttrDef("created", DataTypes.DATE_TYPE), - // enum - new AttributeDefinition("tableType", "tableType", Multiplicity.REQUIRED, false, null), - // array of strings - new AttributeDefinition("columnNames", - String.format("array<%s>", DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL, - false, null), - // array of classes - new AttributeDefinition("columns", String.format("array<%s>", COLUMN_TYPE), - Multiplicity.OPTIONAL, true, null), - // array of structs - new AttributeDefinition("partitions", String.format("array<%s>", "partition_struct_type"), - Multiplicity.OPTIONAL, true, null), - // map of primitives - new AttributeDefinition("parametersMap", - DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()), - Multiplicity.OPTIONAL, true, null), - //map of classes - - new AttributeDefinition(COLUMNS_MAP, - DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), - COLUMN_TYPE), - Multiplicity.OPTIONAL, true, null), - //map of structs - new AttributeDefinition("partitionsMap", - DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), - "partition_struct_type"), - Multiplicity.OPTIONAL, true, null), - // struct reference - new AttributeDefinition("serde1", "serdeType", Multiplicity.OPTIONAL, false, null), - new AttributeDefinition("serde2", "serdeType", Multiplicity.OPTIONAL, false, null), - // class reference - new AttributeDefinition("database", DATABASE_TYPE, Multiplicity.REQUIRED, false, null), - //class reference as composite - new AttributeDefinition("databaseComposite", DATABASE_TYPE, Multiplicity.OPTIONAL, true, null)); - - HierarchicalTypeDefinition<TraitType> piiTypeDefinition = - createTraitTypeDef(PII, PII + _description, ImmutableSet.<String>of()); - - HierarchicalTypeDefinition<TraitType> classificationTypeDefinition = - createTraitTypeDef(CLASSIFICATION, CLASSIFICATION + _description, ImmutableSet.<String>of(), - createRequiredAttrDef("tag", DataTypes.STRING_TYPE)); - - HierarchicalTypeDefinition<TraitType> fetlClassificationTypeDefinition = - createTraitTypeDef("fetl" + CLASSIFICATION, "fetl" + CLASSIFICATION + _description, ImmutableSet.of(CLASSIFICATION), - createRequiredAttrDef("tag", DataTypes.STRING_TYPE)); - - return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), - ImmutableList.of(structTypeDefinition, partitionDefinition), - ImmutableList.of(classificationTypeDefinition, fetlClassificationTypeDefinition, piiTypeDefinition), - ImmutableList.of(superTypeDefinition, databaseTypeDefinition, columnsDefinition, tableTypeDefinition, - storageDescClsDef, partClsDef, processClsType)); - } - - public static Collection<IDataType> createHiveTypes(TypeSystem typeSystem) throws Exception { - if (!typeSystem.isRegistered(TABLE_TYPE)) { - TypesDef typesDef = defineHiveTypes(); - return typeSystem.defineTypes(typesDef).values(); - } - return new ArrayList<>(); - } - - public static final String randomString() { - return randomString(10); - } - - public static final String randomString(int count) { - final String prefix = "r"; - - return prefix + RandomStringUtils.randomAlphanumeric(count - prefix.length()); // ensure that the string starts with a letter - } - - public static Referenceable createDBEntity() { - Referenceable entity = new Referenceable(DATABASE_TYPE); - String dbName = RandomStringUtils.randomAlphanumeric(10); - entity.set(NAME, dbName); - entity.set("description", "us db"); - return entity; - } - - public static Referenceable createTableEntity(String dbId) { - Referenceable entity = new Referenceable(TABLE_TYPE); - String tableName = RandomStringUtils.randomAlphanumeric(10); - entity.set(NAME, tableName); - entity.set("description", "random table"); - entity.set("type", "type"); - entity.set("tableType", "MANAGED"); - entity.set("database", new Id(dbId, 0, DATABASE_TYPE)); - entity.set("created", new Date()); - return entity; - } - - public static Referenceable createColumnEntity() { - Referenceable entity = new Referenceable(COLUMN_TYPE); - entity.set(NAME, RandomStringUtils.randomAlphanumeric(10)); - entity.set("type", "VARCHAR(32)"); - return entity; - } - - /** - * Creates an entity in the graph and does basic validation - * of the GuidMapping that was created in the process. - * - */ - public static String createInstance(MetadataService metadataService, Referenceable entity) throws Exception { - RequestContext.createContext(); - - String entityjson = InstanceSerialization.toJson(entity, true); - JSONArray entitiesJson = new JSONArray(); - entitiesJson.put(entityjson); - CreateUpdateEntitiesResult creationResult = metadataService.createEntities(entitiesJson.toString()); - Map<String,String> guidMap = creationResult.getGuidMapping().getGuidAssignments(); - Map<Id, Referenceable> referencedObjects = findReferencedObjects(entity); - - for(Map.Entry<Id,Referenceable> entry : referencedObjects.entrySet()) { - Id foundId = entry.getKey(); - if(foundId.isUnassigned()) { - String guid = guidMap.get(entry.getKey()._getId()); - Referenceable obj = entry.getValue(); - loadAndDoSimpleValidation(guid,obj, metadataService); - } - } - List<String> guids = creationResult.getCreatedEntities(); - if (guids != null && guids.size() > 0) { - return guids.get(guids.size() - 1); - } - return null; - } - - private static Map<Id,Referenceable> findReferencedObjects(Referenceable ref) { - Map<Id, Referenceable> result = new HashMap<>(); - findReferencedObjects(ref, result); - return result; - } - - private static void findReferencedObjects(Referenceable ref, Map<Id, Referenceable> seen) { - - Id guid = ref.getId(); - if(seen.containsKey(guid)) { - return; - } - seen.put(guid, ref); - for(Map.Entry<String, Object> attr : ref.getValuesMap().entrySet()) { - Object value = attr.getValue(); - if(value instanceof Referenceable) { - findReferencedObjects((Referenceable)value, seen); - } - else if(value instanceof List) { - for(Object o : (List)value) { - if(o instanceof Referenceable) { - findReferencedObjects((Referenceable)o, seen); - } - } - } - else if(value instanceof Map) { - for(Object o : ((Map)value).values()) { - if(o instanceof Referenceable) { - findReferencedObjects((Referenceable)o, seen); - } - } - } - } - } - - /** - * Clears the state in the request context. - * - */ - public static void resetRequestContext() { - //reset the context while preserving the user - String user = RequestContext.get().getUser(); - RequestContext.createContext(); - RequestContext.get().setUser(user); - } - - /** - * Triggers the Atlas initialization process using the specified MetadataRepository. - * This causes the built-in types and their indices to be created. - */ - public static void setupGraphProvider(MetadataRepository repo) throws AtlasException { - TypeCache typeCache = null; - try { - typeCache = AtlasRepositoryConfiguration.getTypeCache().newInstance(); - } - catch(Throwable t) { - typeCache = new DefaultTypeCache(); - } - final GraphBackedSearchIndexer indexer = new GraphBackedSearchIndexer(new AtlasTypeRegistry()); - - Configuration config = ApplicationProperties.get(); - ITypeStore typeStore = new GraphBackedTypeStore(AtlasGraphProvider.getGraphInstance()); - DefaultMetadataService defaultMetadataService = new DefaultMetadataService(repo, - typeStore, - new HashSet<TypesChangeListener>() {{ add(indexer); }}, - new HashSet<EntityChangeListener>(), - TypeSystem.getInstance(), - config, - typeCache, - // Fixme: Can we work with Noop - new InMemoryEntityAuditRepository()); - - //commit the created types - getGraph().commit(); - - } - - public static AtlasGraph getGraph() { - - return AtlasGraphProvider.getGraphInstance(); - - } - - /** - * Adds a proxy wrapper around the specified MetadataService that automatically - * resets the request context before every call. - * - * @param delegate - * @return - */ - public static MetadataService addSessionCleanupWrapper(final MetadataService delegate) { - - return (MetadataService)Proxy.newProxyInstance(Thread.currentThread().getContextClassLoader(), - new Class[]{MetadataService.class}, new InvocationHandler() { - - @Override - public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { - - try { - resetRequestContext(); - Object result = method.invoke(delegate, args); - - return result; - } - catch(InvocationTargetException e) { - e.getCause().printStackTrace(); - throw e.getCause(); - } - catch(Throwable t) { - t.printStackTrace(); - throw t; - } - } - - }); - } - - /** - * Adds a proxy wrapper around the specified MetadataRepository that automatically - * resets the request context before every call and either commits or rolls - * back the graph transaction after every call. - * - * @param delegate - * @return - */ - public static MetadataRepository addTransactionWrapper(final MetadataRepository delegate) { - return (MetadataRepository)Proxy.newProxyInstance(Thread.currentThread().getContextClassLoader(), - new Class[]{MetadataRepository.class}, new InvocationHandler() { - - @Override - public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { - boolean useTransaction = GraphBackedMetadataRepository.class.getMethod( - method.getName(), method.getParameterTypes()) - .isAnnotationPresent(GraphTransaction.class); - try { - resetRequestContext(); - Object result = method.invoke(delegate, args); - if(useTransaction) { - System.out.println("Committing changes"); - getGraph().commit(); - System.out.println("Commit succeeded."); - } - return result; - } - catch(InvocationTargetException e) { - e.getCause().printStackTrace(); - if(useTransaction) { - System.out.println("Rolling back changes due to exception."); - getGraph().rollback(); - } - throw e.getCause(); - } - catch(Throwable t) { - t.printStackTrace(); - if(useTransaction) { - System.out.println("Rolling back changes due to exception."); - getGraph().rollback(); - } - throw t; - } - } - - }); - } - - /** - * Loads the entity and does sanity testing of the GuidMapping that was - * created during the operation. - * - */ - public static ITypedReferenceableInstance loadAndDoSimpleValidation(String guid, Referenceable original, MetadataRepository repositoryService) throws AtlasException { - ITypedReferenceableInstance loaded = repositoryService.getEntityDefinition(guid); - doSimpleValidation(original, loaded); - return loaded; - } - - /** - * Loads the entity and does sanity testing of the GuidMapping that was - * created during the operation. - * - */ - public static ITypedReferenceableInstance loadAndDoSimpleValidation(String guid, Referenceable original, MetadataService repositoryService) throws AtlasException { - ITypedReferenceableInstance loaded = repositoryService.getEntityDefinition(guid); - doSimpleValidation(original, loaded); - return loaded; - - } - - private static void doSimpleValidation(Referenceable original, IInstance loaded) throws AtlasException { - - assertEquals(loaded.getTypeName(), original.getTypeName()); - ClassType ct = TypeSystem.getInstance().getDataType(ClassType.class, loaded.getTypeName()); - - //compare primitive fields - for(AttributeInfo field : ct.fieldMapping.fields.values()) { - if(field.dataType().getTypeCategory() == TypeCategory.PRIMITIVE) { - if(original.get(field.name) != null) { - Object rawLoadedValue = loaded.get(field.name); - Object rawProvidedValue = original.get(field.name); - Object convertedLoadedValue = field.dataType().convert(rawLoadedValue, Multiplicity.REQUIRED); - Object convertedProvidedValue = field.dataType().convert(rawProvidedValue, Multiplicity.REQUIRED); - - assertEquals(convertedLoadedValue, convertedProvidedValue); - } - } - } - } - - /** - * Validates that the two String Collections contain the same items, without - * regard to order. - * - */ - public static void assertContentsSame(Collection<String> actual, Collection<String> expected) { - assertEquals(actual.size(), expected.size()); - Set<String> checker = new HashSet<>(); - checker.addAll(expected); - checker.removeAll(actual); - assertEquals(checker.size(), 0); - } - - public static void skipForGremlin3EnabledGraphDb() throws SkipException { - //ATLAS-1579 Currently, some tests are skipped for titan1 backened. As these tests are hard coded to use Gremlin2. See ATLAS-1579, ATLAS-1591 once it is fixed, please remove it. - if (TestUtils.getGraph().getSupportedGremlinVersion() == GremlinVersion.THREE) { - throw new SkipException ("This test requires Gremlin2. Skipping test "); - } - } - -}