ATLAS-2460: migration-import implementation (#2 - UT refactoring) Signed-off-by: Madhan Neethiraj <mad...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/9368c8a0 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/9368c8a0 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/9368c8a0 Branch: refs/heads/master Commit: 9368c8a05f34d9bbe4a34cd8aa531bfb36a70a44 Parents: 3ded3ee Author: Ashutosh Mestry <ames...@hortonworks.com> Authored: Wed Apr 4 22:15:16 2018 -0700 Committer: Madhan Neethiraj <mad...@apache.org> Committed: Thu Apr 5 00:03:36 2018 -0700 ---------------------------------------------------------------------- .../repository/migration/HiveParititionIT.java | 103 - .../migration/HiveParititionTest.java | 65 + .../repository/migration/HiveStocksIT.java | 102 - .../repository/migration/HiveStocksTest.java | 63 + .../migration/MigrationBaseAsserts.java | 48 + .../stocks_db/atlas-migration-data.json | 5156 ++++++++++++++++++ .../stocks_db/atlas-migration-typesdef.json | 2014 +++++++ 7 files changed, 7346 insertions(+), 205 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/9368c8a0/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionIT.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionIT.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionIT.java deleted file mode 100644 index bd1fb4c..0000000 --- a/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionIT.java +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.repository.migration; - -import com.google.inject.Inject; -import org.apache.atlas.TestModules; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.repository.graph.AtlasGraphProvider; -import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; -import org.apache.atlas.repository.graph.GraphHelper; -import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; -import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; -import org.apache.atlas.runner.LocalSolrRunner; -import org.apache.atlas.store.AtlasTypeDefStore; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.utils.TestResourceFileUtils; -import org.testng.annotations.AfterClass; -import org.testng.annotations.Guice; -import org.testng.annotations.Test; - -import java.io.IOException; -import java.util.List; - -import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr; -import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson; -import static org.testng.Assert.assertEquals; - -@Guice(modules = TestModules.TestOnlyModule.class) -public class HiveParititionIT extends MigrationBaseAsserts { - @Inject - private AtlasTypeDefStore typeDefStore; - - @Inject - private AtlasTypeRegistry typeRegistry; - - @Inject - private AtlasTypeDefStoreInitializer storeInitializer; - - @Inject - private GraphBackedSearchIndexer indexer; - - @Inject - public HiveParititionIT(AtlasGraph graph) { - super(graph); - } - - @AfterClass - public void clear() throws Exception { - AtlasGraphProvider.cleanup(); - - if (useLocalSolr()) { - LocalSolrRunner.stop(); - } - } - - @Test - public void fileImporterTest() throws IOException, AtlasBaseException { - loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry); - loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry); - - String directoryName = TestResourceFileUtils.getDirectory("parts_db"); - DataMigrationService.FileImporter fi = new DataMigrationService.FileImporter(typeDefStore, typeRegistry, - storeInitializer, directoryName, indexer); - - fi.run(); - - - assertPartitionKeyProperty(getVertex("hive_table", "t1"), 1); - assertPartitionKeyProperty(getVertex("hive_table", "tv1"), 1); - assertHiveVertices(1, 2, 7); - - assertTypeCountNameGuid("hive_db", 1, "parts_db", "ae30d78b-51b4-42ab-9436-8d60c8f68b95"); - assertTypeCountNameGuid("hive_process", 1, "", ""); - assertEdges("hive_db", "parts_db", AtlasEdgeDirection.IN,1, 1, ""); - assertEdges("hive_table", "t1", AtlasEdgeDirection.OUT, 1, 1, "hive_db_tables"); - assertEdges("hive_table", "tv1", AtlasEdgeDirection.OUT, 1, 1, "hive_db_tables"); - - assertMigrationStatus(136); - } - - private void assertPartitionKeyProperty(AtlasVertex vertex, int expectedCount) { - List<String> keys = GraphHelper.getListProperty(vertex, "hive_table.partitionKeys"); - assertEquals(keys.size(), expectedCount); - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/9368c8a0/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java new file mode 100644 index 0000000..382c288 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.migration; + +import com.google.inject.Inject; +import org.apache.atlas.TestModules; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.repository.graph.GraphHelper; +import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.util.List; + +import static org.testng.Assert.assertEquals; + +@Guice(modules = TestModules.TestOnlyModule.class) +public class HiveParititionTest extends MigrationBaseAsserts { + + @Inject + public HiveParititionTest(AtlasGraph graph) { + super(graph); + } + + @Test + public void fileImporterTest() throws IOException, AtlasBaseException { + runFileImporter("parts_db"); + + assertPartitionKeyProperty(getVertex("hive_table", "t1"), 1); + assertPartitionKeyProperty(getVertex("hive_table", "tv1"), 1); + assertHiveVertices(1, 2, 7); + + assertTypeCountNameGuid("hive_db", 1, "parts_db", "ae30d78b-51b4-42ab-9436-8d60c8f68b95"); + assertTypeCountNameGuid("hive_process", 1, "", ""); + assertEdges("hive_db", "parts_db", AtlasEdgeDirection.IN,1, 1, ""); + assertEdges("hive_table", "t1", AtlasEdgeDirection.OUT, 1, 1, "hive_db_tables"); + assertEdges("hive_table", "tv1", AtlasEdgeDirection.OUT, 1, 1, "hive_db_tables"); + + assertMigrationStatus(136); + } + + private void assertPartitionKeyProperty(AtlasVertex vertex, int expectedCount) { + List<String> keys = GraphHelper.getListProperty(vertex, "hive_table.partitionKeys"); + assertEquals(keys.size(), expectedCount); + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/9368c8a0/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksIT.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksIT.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksIT.java deleted file mode 100644 index 7fc8f2a..0000000 --- a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksIT.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.repository.migration; - -import com.google.inject.Inject; -import org.apache.atlas.RequestContextV1; -import org.apache.atlas.TestModules; -import org.apache.atlas.TestUtilsV2; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.repository.graph.AtlasGraphProvider; -import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; -import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.runner.LocalSolrRunner; -import org.apache.atlas.store.AtlasTypeDefStore; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.utils.TestResourceFileUtils; -import org.testng.ITestContext; -import org.testng.annotations.*; - -import java.io.FileInputStream; -import java.io.IOException; - -import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr; -import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson; - -@Guice(modules = TestModules.TestOnlyModule.class) -public class HiveStocksIT extends MigrationBaseAsserts { - @Inject - private AtlasTypeDefStore typeDefStore; - - @Inject - private AtlasTypeRegistry typeRegistry; - - @Inject - public HiveStocksIT(AtlasGraph graph) { - super(graph); - } - - @BeforeTest - public void setupTest() { - RequestContextV1.clear(); - RequestContextV1.get().setUser(TestUtilsV2.TEST_USER, null); - } - - @AfterClass - public void clear() throws Exception { - AtlasGraphProvider.cleanup(); - - if (useLocalSolr()) { - LocalSolrRunner.stop(); - } - } - - @DataProvider(name = "stocks-2-branch08-tag") - public static Object[][] getStocksTag(ITestContext context) throws IOException { - return new Object[][]{{ TestResourceFileUtils.getFileInputStream("stocks-2-0.8-extended-tag.json") }}; - } - - @Test(dataProvider = "stocks-2-branch08-tag") - public void migrateFromEarlierVersionWithTag(FileInputStream fs) throws AtlasBaseException, IOException { - loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry); - loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry); - - typeDefStore.loadLegacyData(RelationshipCacheGenerator.get(typeRegistry), fs); - - assertHiveVertices(1, 1, 7); - assertTypeCountNameGuid("hive_db", 1, "stocks", "4e13b36b-9c54-4616-9001-1058221165d0"); - assertTypeCountNameGuid("hive_table", 1, "stocks_daily", "5cfc2540-9947-40e0-8905-367e07481774"); - assertTypeAttribute("hive_table", 7, "stocks_daily", "5cfc2540-9947-40e0-8905-367e07481774", "hive_table.columns"); - assertTypeCountNameGuid("hive_column", 1, "high", "d72ce4fb-6f17-4e68-aa85-967366c9e891"); - assertTypeCountNameGuid("hive_column", 1, "open", "788ba8fe-b7d8-41ba-84ef-c929732924ec"); - assertTypeCountNameGuid("hive_column", 1, "dt", "643a0a71-0d97-477d-a43b-7ca433f85160"); - assertTypeCountNameGuid("hive_column", 1, "low", "38caeaf7-49e6-4d6d-8727-231406a46821"); - assertTypeCountNameGuid("hive_column", 1, "close", "3bae9b76-f812-4745-b4d2-2a72d2773d07"); - assertTypeCountNameGuid("hive_column", 1, "volume", "bee376a4-3d8d-4943-b7e8-9bce042c2657"); - assertTypeCountNameGuid("hive_column", 1, "adj_close", "fcba2002-cb38-4c2e-b853-68d421d66703"); - assertTypeCountNameGuid("hive_process", 0, "", ""); - assertTypeCountNameGuid("hive_storagedesc", 1, "", "294290d8-4498-4677-973c-c266d594b039"); - assertTypeCountNameGuid("Tag1", 1, "", ""); - - assertEdges(getVertex("hive_db", "stocks").getEdges(AtlasEdgeDirection.IN).iterator(),1, 1, ""); - assertEdges(getVertex("hive_table", "stocks_daily").getEdges(AtlasEdgeDirection.OUT).iterator(), 1, 1, "hive_db_tables"); - assertEdges(getVertex("hive_column", "high").getEdges(AtlasEdgeDirection.OUT).iterator(), 1,1, "hive_table_columns"); - - assertMigrationStatus(164); - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/9368c8a0/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java new file mode 100644 index 0000000..25c72a4 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.migration; + +import com.google.inject.Inject; +import org.apache.atlas.TestModules; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import java.io.IOException; + +@Guice(modules = TestModules.TestOnlyModule.class) +public class HiveStocksTest extends MigrationBaseAsserts { + + @Inject + public HiveStocksTest(AtlasGraph graph) { + super(graph); + } + + @Test + public void migrateStocks() throws AtlasBaseException, IOException { + runFileImporter("stocks_db"); + + assertHiveVertices(1, 1, 7); + assertTypeCountNameGuid("hive_db", 1, "stocks", "4e13b36b-9c54-4616-9001-1058221165d0"); + assertTypeCountNameGuid("hive_table", 1, "stocks_daily", "5cfc2540-9947-40e0-8905-367e07481774"); + assertTypeAttribute("hive_table", 7, "stocks_daily", "5cfc2540-9947-40e0-8905-367e07481774", "hive_table.columns"); + assertTypeCountNameGuid("hive_column", 1, "high", "d72ce4fb-6f17-4e68-aa85-967366c9e891"); + assertTypeCountNameGuid("hive_column", 1, "open", "788ba8fe-b7d8-41ba-84ef-c929732924ec"); + assertTypeCountNameGuid("hive_column", 1, "dt", "643a0a71-0d97-477d-a43b-7ca433f85160"); + assertTypeCountNameGuid("hive_column", 1, "low", "38caeaf7-49e6-4d6d-8727-231406a46821"); + assertTypeCountNameGuid("hive_column", 1, "close", "3bae9b76-f812-4745-b4d2-2a72d2773d07"); + assertTypeCountNameGuid("hive_column", 1, "volume", "bee376a4-3d8d-4943-b7e8-9bce042c2657"); + assertTypeCountNameGuid("hive_column", 1, "adj_close", "fcba2002-cb38-4c2e-b853-68d421d66703"); + assertTypeCountNameGuid("hive_process", 0, "", ""); + assertTypeCountNameGuid("hive_storagedesc", 1, "", "294290d8-4498-4677-973c-c266d594b039"); + assertTypeCountNameGuid("Tag1", 1, "", ""); + + assertEdges(getVertex("hive_db", "stocks").getEdges(AtlasEdgeDirection.IN).iterator(),1, 1, ""); + assertEdges(getVertex("hive_table", "stocks_daily").getEdges(AtlasEdgeDirection.OUT).iterator(), 1, 1, "hive_db_tables"); + assertEdges(getVertex("hive_column", "high").getEdges(AtlasEdgeDirection.OUT).iterator(), 1,1, "hive_table_columns"); + + assertMigrationStatus(187); + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/9368c8a0/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java b/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java index ed2793f..5639b43 100644 --- a/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java +++ b/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java @@ -18,13 +18,26 @@ package org.apache.atlas.repository.migration; +import com.google.inject.Inject; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.repository.graph.AtlasGraphProvider; +import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.*; +import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; +import org.apache.atlas.runner.LocalSolrRunner; +import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.TestResourceFileUtils; import org.apache.commons.lang.StringUtils; +import org.testng.annotations.AfterClass; +import java.io.IOException; import java.util.Iterator; import java.util.List; +import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr; +import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.AssertJUnit.assertTrue; @@ -34,12 +47,47 @@ public class MigrationBaseAsserts { private final String TYPE_NAME_PROPERTY = "__typeName"; private final String R_GUID_PROPERTY_NAME = "_r__guid"; + @Inject + private AtlasTypeDefStore typeDefStore; + + @Inject + private AtlasTypeRegistry typeRegistry; + + @Inject + private AtlasTypeDefStoreInitializer storeInitializer; + + @Inject + private GraphBackedSearchIndexer indexer; + protected AtlasGraph graph; protected MigrationBaseAsserts(AtlasGraph graph) { this.graph = graph; } + @AfterClass + public void clear() throws Exception { + AtlasGraphProvider.cleanup(); + + if (useLocalSolr()) { + LocalSolrRunner.stop(); + } + } + + private void loadTypesFromJson() throws IOException, AtlasBaseException { + loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry); + loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry); + } + + protected void runFileImporter(String directoryToImport) throws IOException, AtlasBaseException { + loadTypesFromJson(); + String directoryName = TestResourceFileUtils.getDirectory(directoryToImport); + DataMigrationService.FileImporter fi = new DataMigrationService.FileImporter(typeDefStore, typeRegistry, + storeInitializer, directoryName, indexer); + + fi.run(); + } + protected void assertHiveVertices(int dbCount, int tableCount, int columnCount) { int i = 0;
