This is an automated email from the ASF dual-hosted git repository. vinoth pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push: new 3dedc7e [HUDI-265] Failed to delete tmp dirs created in unit tests (#928) 3dedc7e is described below commit 3dedc7e5fdd5f885915e81e47e110b845a905dbf Author: leesf <490081...@qq.com> AuthorDate: Fri Oct 4 00:48:13 2019 +0800 [HUDI-265] Failed to delete tmp dirs created in unit tests (#928) --- .../org/apache/hudi/HoodieClientTestHarness.java | 48 +----------- .../org/apache/hudi/TestCompactionAdminClient.java | 5 +- .../java/org/apache/hudi/TestConsistencyGuard.java | 6 +- .../apache/hudi/func/TestUpdateMapFunction.java | 5 +- .../hudi/index/TestHBaseQPSResourceAllocator.java | 3 +- .../java/org/apache/hudi/index/TestHbaseIndex.java | 3 +- .../org/apache/hudi/index/TestHoodieIndex.java | 5 +- .../hudi/index/bloom/TestHoodieBloomIndex.java | 3 +- .../index/bloom/TestHoodieGlobalBloomIndex.java | 5 +- .../apache/hudi/io/TestHoodieCommitArchiveLog.java | 3 +- .../org/apache/hudi/io/TestHoodieCompactor.java | 3 +- .../org/apache/hudi/io/TestHoodieMergeHandle.java | 3 +- .../apache/hudi/table/TestCopyOnWriteTable.java | 3 +- .../apache/hudi/table/TestMergeOnReadTable.java | 4 +- .../hudi/common/HoodieCommonTestHarness.java | 89 ++++++++++++++++++++++ .../common/table/HoodieTableMetaClientTest.java | 14 +--- .../hudi/common/table/log/HoodieLogFormatTest.java | 11 ++- .../table/string/HoodieActiveTimelineTest.java | 13 ++-- .../table/view/HoodieTableFileSystemViewTest.java | 38 ++++----- .../table/view/IncrementalFSViewSyncTest.java | 68 ++++++----------- .../RocksDBBasedIncrementalFSViewSyncTest.java | 4 +- .../table/view/RocksDbBasedFileSystemViewTest.java | 2 +- ...SpillableMapBasedIncrementalFSViewSyncTest.java | 2 +- .../hudi/common/util/TestCompactionUtils.java | 20 ++--- .../org/apache/hudi/common/util/TestFSUtils.java | 14 ++-- .../apache/hudi/common/util/TestFileIOUtils.java | 7 +- .../apache/hudi/common/util/TestParquetUtils.java | 15 +--- .../hudi/common/util/TestRocksDBManager.java | 12 ++- .../common/util/collection/TestDiskBasedMap.java | 24 +++--- .../util/collection/TestExternalSpillableMap.java | 32 ++++---- .../util/collection/TestRocksDbBasedMap.java | 11 ++- .../hudi/hadoop/TestHoodieROTablePathFilter.java | 13 ++-- .../hudi/utilities/TestHoodieSnapshotCopier.java | 32 ++++---- 33 files changed, 248 insertions(+), 272 deletions(-) diff --git a/hudi-client/src/test/java/org/apache/hudi/HoodieClientTestHarness.java b/hudi-client/src/test/java/org/apache/hudi/HoodieClientTestHarness.java index 10fb0bc..80cb70f 100644 --- a/hudi-client/src/test/java/org/apache/hudi/HoodieClientTestHarness.java +++ b/hudi-client/src/test/java/org/apache/hudi/HoodieClientTestHarness.java @@ -17,7 +17,6 @@ package org.apache.hudi; -import java.io.File; import java.io.IOException; import java.io.Serializable; import java.util.concurrent.ExecutorService; @@ -29,30 +28,27 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hudi.common.HoodieClientTestUtils; +import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.minicluster.HdfsTestService; -import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.FSUtils; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SQLContext; -import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * The test harness for resource initialization and cleanup. */ -public abstract class HoodieClientTestHarness implements Serializable { +public abstract class HoodieClientTestHarness extends HoodieCommonTestHarness implements Serializable { private static final Logger logger = LoggerFactory.getLogger(HoodieClientTestHarness.class); protected transient JavaSparkContext jsc = null; protected transient SQLContext sqlContext; protected transient FileSystem fs; - protected String basePath = null; - protected TemporaryFolder folder = null; protected transient HoodieTestDataGenerator dataGen = null; protected transient ExecutorService executorService; protected transient HoodieTableMetaClient metaClient; @@ -69,7 +65,7 @@ public abstract class HoodieClientTestHarness implements Serializable { * @throws IOException */ public void initResources() throws IOException { - initTempFolderAndPath(); + initPath(); initSparkContexts(); initTestDataGenerator(); initFileSystem(); @@ -85,7 +81,6 @@ public abstract class HoodieClientTestHarness implements Serializable { cleanupSparkContexts(); cleanupTestDataGenerator(); cleanupFileSystem(); - cleanupTempFolderAndPath(); } /** @@ -130,33 +125,6 @@ public abstract class HoodieClientTestHarness implements Serializable { } /** - * Initializes a temporary folder and base path. - * - * @throws IOException - */ - protected void initTempFolderAndPath() throws IOException { - folder = new TemporaryFolder(); - folder.create(); - basePath = folder.getRoot().getAbsolutePath(); - } - - /** - * Cleanups the temporary folder and base path. - * - * @throws IOException - */ - protected void cleanupTempFolderAndPath() throws IOException { - if (basePath != null) { - new File(basePath).delete(); - } - - if (folder != null) { - logger.info("Explicitly removing workspace used in previously run test-case"); - folder.delete(); - } - } - - /** * Initializes a file system with the hadoop configuration of Spark context. */ protected void initFileSystem() { @@ -230,16 +198,6 @@ public abstract class HoodieClientTestHarness implements Serializable { } /** - * Gets a default {@link HoodieTableType#COPY_ON_WRITE} table type. - * Sub-classes can override this method to specify a new table type. - * - * @return an instance of Hoodie table type. - */ - protected HoodieTableType getTableType() { - return HoodieTableType.COPY_ON_WRITE; - } - - /** * Initializes a distributed file system and base directory. * * @throws IOException diff --git a/hudi-client/src/test/java/org/apache/hudi/TestCompactionAdminClient.java b/hudi-client/src/test/java/org/apache/hudi/TestCompactionAdminClient.java index 3b6652a..dc2b236 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestCompactionAdminClient.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestCompactionAdminClient.java @@ -51,18 +51,17 @@ public class TestCompactionAdminClient extends TestHoodieClientBase { @Before public void setUp() throws Exception { - initTempFolderAndPath(); + initPath(); initSparkContexts(); metaClient = HoodieTestUtils.init(HoodieTestUtils.getDefaultHadoopConf(), basePath, MERGE_ON_READ); client = new CompactionAdminClient(jsc, basePath); } @After - public void tearDown() throws Exception { + public void tearDown() { client.close(); metaClient = null; cleanupSparkContexts(); - cleanupTempFolderAndPath(); } @Test diff --git a/hudi-client/src/test/java/org/apache/hudi/TestConsistencyGuard.java b/hudi-client/src/test/java/org/apache/hudi/TestConsistencyGuard.java index d671e5a..c6b6d11 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestConsistencyGuard.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestConsistencyGuard.java @@ -18,7 +18,6 @@ package org.apache.hudi; -import java.io.IOException; import java.util.Arrays; import java.util.concurrent.TimeoutException; import org.apache.hadoop.fs.Path; @@ -33,15 +32,14 @@ import org.junit.Test; public class TestConsistencyGuard extends HoodieClientTestHarness { @Before - public void setup() throws IOException { - initTempFolderAndPath(); + public void setup() { + initPath(); initFileSystemWithDefaultConfiguration(); } @After public void tearDown() throws Exception { cleanupFileSystem(); - cleanupTempFolderAndPath(); } @Test diff --git a/hudi-client/src/test/java/org/apache/hudi/func/TestUpdateMapFunction.java b/hudi-client/src/test/java/org/apache/hudi/func/TestUpdateMapFunction.java index db986de..5837628 100644 --- a/hudi-client/src/test/java/org/apache/hudi/func/TestUpdateMapFunction.java +++ b/hudi-client/src/test/java/org/apache/hudi/func/TestUpdateMapFunction.java @@ -53,14 +53,13 @@ public class TestUpdateMapFunction extends HoodieClientTestHarness { @Before public void setUp() throws Exception { - initTempFolderAndPath(); + initPath(); HoodieTestUtils.init(HoodieTestUtils.getDefaultHadoopConf(), basePath); initSparkContexts("TestUpdateMapFunction"); } @After - public void tearDown() throws Exception { - cleanupTempFolderAndPath(); + public void tearDown() { cleanupSparkContexts(); } diff --git a/hudi-client/src/test/java/org/apache/hudi/index/TestHBaseQPSResourceAllocator.java b/hudi-client/src/test/java/org/apache/hudi/index/TestHBaseQPSResourceAllocator.java index 9efe708..520b93b 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/TestHBaseQPSResourceAllocator.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/TestHBaseQPSResourceAllocator.java @@ -50,7 +50,7 @@ public class TestHBaseQPSResourceAllocator extends HoodieClientTestHarness { hbaseConfig = utility.getConnection().getConfiguration(); initSparkContexts("TestQPSResourceAllocator"); - initTempFolderAndPath(); + initPath(); basePath = folder.getRoot().getAbsolutePath() + QPS_TEST_SUFFIX_PATH; // Initialize table initMetaClient(); @@ -59,7 +59,6 @@ public class TestHBaseQPSResourceAllocator extends HoodieClientTestHarness { @After public void tearDown() throws Exception { cleanupSparkContexts(); - cleanupTempFolderAndPath(); cleanupMetaClient(); if (utility != null) { utility.shutdownMiniCluster(); diff --git a/hudi-client/src/test/java/org/apache/hudi/index/TestHbaseIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/TestHbaseIndex.java index 6c2fc0f..8817ad3 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/TestHbaseIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/TestHbaseIndex.java @@ -102,7 +102,7 @@ public class TestHbaseIndex extends HoodieClientTestHarness { jsc.hadoopConfiguration().addResource(utility.getConfiguration()); // Create a temp folder as the base path - initTempFolderAndPath(); + initPath(); initTestDataGenerator(); initMetaClient(); } @@ -110,7 +110,6 @@ public class TestHbaseIndex extends HoodieClientTestHarness { @After public void tearDown() throws Exception { cleanupSparkContexts(); - cleanupTempFolderAndPath(); cleanupTestDataGenerator(); cleanupMetaClient(); } diff --git a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java index 4354216..44451e1 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java @@ -35,14 +35,13 @@ public class TestHoodieIndex extends HoodieClientTestHarness { @Before public void setUp() throws Exception { initSparkContexts("TestHoodieIndex"); - initTempFolderAndPath(); + initPath(); initMetaClient(); } @After - public void tearDown() throws Exception { + public void tearDown() { cleanupSparkContexts(); - cleanupTempFolderAndPath(); cleanupMetaClient(); } diff --git a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java index 69d0cfa..a9234b9 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java @@ -89,7 +89,7 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness { @Before public void setUp() throws Exception { initSparkContexts("TestHoodieBloomIndex"); - initTempFolderAndPath(); + initPath(); initFileSystem(); // We have some records to be tagged (two different partitions) schemaStr = FileIOUtils.readAsUTFString(getClass().getResourceAsStream("/exampleSchema.txt")); @@ -101,7 +101,6 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness { public void tearDown() throws Exception { cleanupSparkContexts(); cleanupFileSystem(); - cleanupTempFolderAndPath(); cleanupMetaClient(); } diff --git a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java index 11669a3..cdf4441 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java @@ -64,7 +64,7 @@ public class TestHoodieGlobalBloomIndex extends HoodieClientTestHarness { @Before public void setUp() throws Exception { initSparkContexts("TestHoodieGlobalBloomIndex"); - initTempFolderAndPath(); + initPath(); // We have some records to be tagged (two different partitions) schemaStr = FileIOUtils.readAsUTFString(getClass().getResourceAsStream("/exampleSchema.txt")); schema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(schemaStr)); @@ -72,9 +72,8 @@ public class TestHoodieGlobalBloomIndex extends HoodieClientTestHarness { } @After - public void tearDown() throws Exception { + public void tearDown() { cleanupSparkContexts(); - cleanupTempFolderAndPath(); cleanupMetaClient(); } diff --git a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java index 9a9ddc0..8ea0ae4 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java @@ -59,7 +59,7 @@ public class TestHoodieCommitArchiveLog extends HoodieClientTestHarness { @Before public void init() throws Exception { initDFS(); - initTempFolderAndPath(); + initPath(); initSparkContexts("TestHoodieCommitArchiveLog"); hadoopConf = dfs.getConf(); jsc.hadoopConfiguration().addResource(dfs.getConf()); @@ -70,7 +70,6 @@ public class TestHoodieCommitArchiveLog extends HoodieClientTestHarness { @After public void clean() throws IOException { cleanupDFS(); - cleanupTempFolderAndPath(); cleanupSparkContexts(); } diff --git a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCompactor.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCompactor.java index 0a02b70..150f6e6 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCompactor.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCompactor.java @@ -60,7 +60,7 @@ public class TestHoodieCompactor extends HoodieClientTestHarness { initSparkContexts("TestHoodieCompactor"); // Create a temp folder as the base path - initTempFolderAndPath(); + initPath(); hadoopConf = HoodieTestUtils.getDefaultHadoopConf(); fs = FSUtils.getFs(basePath, hadoopConf); metaClient = HoodieTestUtils.init(hadoopConf, basePath, HoodieTableType.MERGE_ON_READ); @@ -71,7 +71,6 @@ public class TestHoodieCompactor extends HoodieClientTestHarness { public void tearDown() throws Exception { cleanupFileSystem(); cleanupTestDataGenerator(); - cleanupTempFolderAndPath(); cleanupSparkContexts(); } diff --git a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java index 68db6ca..5fdde99 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java @@ -56,7 +56,7 @@ public class TestHoodieMergeHandle extends HoodieClientTestHarness { @Before public void setUp() throws Exception { initSparkContexts("TestHoodieMergeHandle"); - initTempFolderAndPath(); + initPath(); initFileSystem(); initTestDataGenerator(); initMetaClient(); @@ -66,7 +66,6 @@ public class TestHoodieMergeHandle extends HoodieClientTestHarness { public void tearDown() throws Exception { cleanupFileSystem(); cleanupTestDataGenerator(); - cleanupTempFolderAndPath(); cleanupSparkContexts(); cleanupMetaClient(); } diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java b/hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java index 2c46fa9..6c0c0ca 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java @@ -72,7 +72,7 @@ public class TestCopyOnWriteTable extends HoodieClientTestHarness { @Before public void setUp() throws Exception { initSparkContexts("TestCopyOnWriteTable"); - initTempFolderAndPath(); + initPath(); initMetaClient(); initTestDataGenerator(); initFileSystem(); @@ -81,7 +81,6 @@ public class TestCopyOnWriteTable extends HoodieClientTestHarness { @After public void tearDown() throws Exception { cleanupSparkContexts(); - cleanupTempFolderAndPath(); cleanupMetaClient(); cleanupFileSystem(); cleanupTestDataGenerator(); diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java b/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java index 788c783..316e171 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java @@ -83,7 +83,7 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness { initDFS(); initSparkContexts("TestHoodieMergeOnReadTable"); jsc.hadoopConfiguration().addResource(dfs.getConf()); - initTempFolderAndPath(); + initPath(); dfs.mkdirs(new Path(basePath)); HoodieTestUtils.init(jsc.hadoopConfiguration(), basePath, HoodieTableType.MERGE_ON_READ); initTestDataGenerator(); @@ -92,7 +92,6 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness { @After public void clean() throws IOException { cleanupDFS(); - cleanupTempFolderAndPath(); cleanupSparkContexts(); cleanupTestDataGenerator(); } @@ -968,6 +967,7 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness { Thread.sleep(1000); // Rollback again to pretend the first rollback failed partially. This should not error our writeClient.rollback(newCommitTime); + folder.delete(); } } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/HoodieCommonTestHarness.java b/hudi-common/src/test/java/org/apache/hudi/common/HoodieCommonTestHarness.java new file mode 100644 index 0000000..03aa5b7 --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/common/HoodieCommonTestHarness.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common; + +import java.io.IOException; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; + +public class HoodieCommonTestHarness { + + protected String basePath = null; + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + protected transient HoodieTableMetaClient metaClient; + + /** + * Initializes basePath. + */ + protected void initPath() { + this.basePath = folder.getRoot().getAbsolutePath(); + } + + /** + * Initializes an instance of {@link HoodieTableMetaClient} with a special table type + * specified by {@code getTableType()}. + * + * @throws IOException + */ + protected void initMetaClient() throws IOException { + metaClient = HoodieTestUtils.init(folder.getRoot().getAbsolutePath(), getTableType()); + basePath = metaClient.getBasePath(); + } + + protected void refreshFsView() throws IOException { + metaClient = new HoodieTableMetaClient(metaClient.getHadoopConf(), basePath, true); + } + + protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) throws IOException { + return getFileSystemView(timeline, false); + } + + protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline, boolean enableIncrementalTimelineSync) { + return new HoodieTableFileSystemView(metaClient, timeline, enableIncrementalTimelineSync); + } + + protected SyncableFileSystemView getFileSystemView(HoodieTableMetaClient metaClient) + throws IOException { + return getFileSystemView(metaClient, metaClient.getActiveTimeline().filterCompletedAndCompactionInstants()); + } + + protected SyncableFileSystemView getFileSystemView(HoodieTableMetaClient metaClient, HoodieTimeline timeline) + throws IOException { + return getFileSystemView(timeline, true); + } + + /** + * Gets a default {@link HoodieTableType#COPY_ON_WRITE} table type. + * Sub-classes can override this method to specify a new table type. + * + * @return an instance of Hoodie table type. + */ + protected HoodieTableType getTableType() { + return HoodieTableType.COPY_ON_WRITE; + } +} diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/HoodieTableMetaClientTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/HoodieTableMetaClientTest.java index df72a7c..62f8bf3 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/HoodieTableMetaClientTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/HoodieTableMetaClientTest.java @@ -31,28 +31,20 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; +import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieArchivedTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.Option; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; -public class HoodieTableMetaClientTest { +public class HoodieTableMetaClientTest extends HoodieCommonTestHarness { - private HoodieTableMetaClient metaClient; - private String basePath; - - @Rule - public TemporaryFolder tmpFolder = new TemporaryFolder(); - @Before public void init() throws IOException { - metaClient = HoodieTestUtils.init(tmpFolder.getRoot().getAbsolutePath()); - basePath = metaClient.getBasePath(); + initMetaClient(); } @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/log/HoodieLogFormatTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/log/HoodieLogFormatTest.java index 8b6b597..6ce5850 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/log/HoodieLogFormatTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/log/HoodieLogFormatTest.java @@ -45,6 +45,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.minicluster.MiniClusterUtil; import org.apache.hudi.common.model.HoodieArchivedLogFile; import org.apache.hudi.common.model.HoodieKey; @@ -65,21 +66,20 @@ import org.apache.hudi.common.util.FSUtils; import org.apache.hudi.common.util.HoodieAvroUtils; import org.apache.hudi.common.util.SchemaTestUtil; import org.apache.hudi.exception.CorruptedLogFileException; + import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @SuppressWarnings("Duplicates") @RunWith(Parameterized.class) -public class HoodieLogFormatTest { +public class HoodieLogFormatTest extends HoodieCommonTestHarness { - private static final String BASE_OUTPUT_PATH = "/tmp/"; - private static String basePath; + private static String BASE_OUTPUT_PATH = "/tmp/"; private FileSystem fs; private Path partitionPath; private int bufferSize = 4096; @@ -108,8 +108,7 @@ public class HoodieLogFormatTest { @Before public void setUp() throws IOException, InterruptedException { this.fs = MiniClusterUtil.fileSystem; - TemporaryFolder folder = new TemporaryFolder(); - folder.create(); + assertTrue(fs.mkdirs(new Path(folder.getRoot().getPath()))); this.partitionPath = new Path(folder.getRoot().getPath()); this.basePath = folder.getRoot().getParent(); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/string/HoodieActiveTimelineTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/string/HoodieActiveTimelineTest.java index cb26079..81933ab 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/string/HoodieActiveTimelineTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/string/HoodieActiveTimelineTest.java @@ -24,8 +24,9 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.stream.Stream; + +import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.model.HoodieTestUtils; -import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; @@ -34,20 +35,16 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; -public class HoodieActiveTimelineTest { +public class HoodieActiveTimelineTest extends HoodieCommonTestHarness { private HoodieActiveTimeline timeline; - private HoodieTableMetaClient metaClient; @Rule public final ExpectedException exception = ExpectedException.none(); - @Rule - public TemporaryFolder tmpFolder = new TemporaryFolder(); - + @Before public void setUp() throws Exception { - this.metaClient = HoodieTestUtils.init(tmpFolder.getRoot().getAbsolutePath()); + initMetaClient(); } @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/HoodieTableFileSystemViewTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/HoodieTableFileSystemViewTest.java index 35ddd92..85af25e 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/HoodieTableFileSystemViewTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/HoodieTableFileSystemViewTest.java @@ -38,6 +38,7 @@ import java.util.stream.Stream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.model.CompactionOperation; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieDataFile; @@ -45,8 +46,6 @@ import org.apache.hudi.common.model.HoodieFileGroup; import org.apache.hudi.common.model.HoodieFileGroupId; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.model.HoodieTestUtils; -import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.common.table.SyncableFileSystemView; import org.apache.hudi.common.table.TableFileSystemView; @@ -62,50 +61,38 @@ import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.junit.Assert; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; @SuppressWarnings("ResultOfMethodCallIgnored") -public class HoodieTableFileSystemViewTest { +public class HoodieTableFileSystemViewTest extends HoodieCommonTestHarness { private static final transient Logger log = LogManager.getLogger(HoodieTableFileSystemViewTest.class); private static String TEST_WRITE_TOKEN = "1-0-1"; - protected HoodieTableMetaClient metaClient; - protected String basePath; protected SyncableFileSystemView fsView; protected TableFileSystemView.ReadOptimizedView roView; protected TableFileSystemView.RealtimeView rtView; - @Rule - public TemporaryFolder tmpFolder = new TemporaryFolder(); - @Before public void init() throws IOException { - initializeMetaClient(); + initMetaClient(); refreshFsView(); } - protected void initializeMetaClient() throws IOException { - metaClient = HoodieTestUtils.init(tmpFolder.getRoot().getAbsolutePath(), HoodieTableType.MERGE_ON_READ); - basePath = metaClient.getBasePath(); - } - - protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) throws IOException { - return new HoodieTableFileSystemView(metaClient, timeline); + protected void refreshFsView() throws IOException { + super.refreshFsView(); + closeFsView(); + fsView = getFileSystemView(metaClient.getActiveTimeline().filterCompletedAndCompactionInstants()); + roView = fsView; + rtView = fsView; } - protected void refreshFsView() throws IOException { - metaClient = new HoodieTableMetaClient(metaClient.getHadoopConf(), basePath, true); + private void closeFsView() { if (null != fsView) { fsView.close(); fsView = null; } - fsView = getFileSystemView(metaClient.getActiveTimeline().filterCompletedAndCompactionInstants()); - roView = (TableFileSystemView.ReadOptimizedView) fsView; - rtView = (TableFileSystemView.RealtimeView) fsView; } /** @@ -1185,4 +1172,9 @@ public class HoodieTableFileSystemViewTest { Assert.assertEquals(1, fileIdsInCompaction.size()); Assert.assertTrue(fileIdsInCompaction.contains(fileId)); } + + @Override + protected HoodieTableType getTableType() { + return HoodieTableType.MERGE_ON_READ; + } } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/IncrementalFSViewSyncTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/IncrementalFSViewSyncTest.java index e90e2f1..7135630 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/IncrementalFSViewSyncTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/IncrementalFSViewSyncTest.java @@ -41,6 +41,7 @@ import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.avro.model.HoodieRestoreMetadata; import org.apache.hudi.avro.model.HoodieRollbackMetadata; import org.apache.hudi.common.HoodieCleanStat; +import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.HoodieRollbackStat; import org.apache.hudi.common.model.CompactionOperation; import org.apache.hudi.common.model.FileSlice; @@ -50,7 +51,6 @@ import org.apache.hudi.common.model.HoodieDataFile; import org.apache.hudi.common.model.HoodieFileGroup; import org.apache.hudi.common.model.HoodieFileGroupId; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTimeline; @@ -67,62 +67,36 @@ import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.junit.Assert; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; -public class IncrementalFSViewSyncTest { +public class IncrementalFSViewSyncTest extends HoodieCommonTestHarness { private static final transient Logger log = LogManager.getLogger(IncrementalFSViewSyncTest.class); private static String TEST_WRITE_TOKEN = "1-0-1"; - protected HoodieTableMetaClient metaClient; - protected String basePath; - private final List<String> partitions = Arrays.asList("2018/01/01", "2018/01/02", "2019/03/01"); private final List<String> fileIdsPerPartition = IntStream.range(0, 10).mapToObj(x -> UUID.randomUUID().toString()).collect(Collectors.toList()); - @Rule - public TemporaryFolder tmpFolder = new TemporaryFolder(); - @Before public void init() throws IOException { - initializeMetaClient(); - refreshFsView(); - } - - protected void initializeMetaClient() throws IOException { - metaClient = HoodieTestUtils.init(tmpFolder.getRoot().getAbsolutePath(), HoodieTableType.MERGE_ON_READ); - basePath = metaClient.getBasePath(); + initMetaClient(); partitions.forEach(p -> new File(basePath + "/" + p).mkdirs()); - } - - protected void refreshFsView() throws IOException { - metaClient = new HoodieTableMetaClient(metaClient.getHadoopConf(), basePath, true); - } - - protected SyncableFileSystemView getNewFileSystemView(HoodieTableMetaClient metaClient) throws IOException { - return getNewFileSystemView(metaClient, metaClient.getActiveTimeline().filterCompletedAndCompactionInstants()); - } - - protected SyncableFileSystemView getNewFileSystemView(HoodieTableMetaClient metaClient, HoodieTimeline timeline) - throws IOException { - return new HoodieTableFileSystemView(metaClient, timeline, true); + refreshFsView(); } @Test public void testEmptyPartitionsAndTimeline() throws IOException { - SyncableFileSystemView view = getNewFileSystemView(metaClient); + SyncableFileSystemView view = getFileSystemView(metaClient); Assert.assertFalse(view.getLastInstant().isPresent()); partitions.forEach(p -> Assert.assertEquals(0, view.getLatestFileSlices(p).count())); } @Test public void testAsyncCompaction() throws IOException { - SyncableFileSystemView view = getNewFileSystemView(metaClient); + SyncableFileSystemView view = getFileSystemView(metaClient); view.sync(); // Run 3 ingestion on MOR table (3 delta commits) @@ -181,7 +155,7 @@ public class IncrementalFSViewSyncTest { @Test public void testIngestion() throws IOException { - SyncableFileSystemView view = getNewFileSystemView(metaClient); + SyncableFileSystemView view = getFileSystemView(metaClient); // Add an empty ingestion String firstEmptyInstantTs = "11"; @@ -198,7 +172,7 @@ public class IncrementalFSViewSyncTest { partitions.forEach(p -> Assert.assertEquals(0, view.getLatestFileSlices(p).count())); metaClient.reloadActiveTimeline(); - SyncableFileSystemView newView = getNewFileSystemView(metaClient); + SyncableFileSystemView newView = getFileSystemView(metaClient); for (String partition : partitions) { newView.getAllFileGroups(partition).count(); } @@ -225,7 +199,7 @@ public class IncrementalFSViewSyncTest { @Test public void testMultipleTransitions() throws IOException { - SyncableFileSystemView view1 = getNewFileSystemView(metaClient); + SyncableFileSystemView view1 = getFileSystemView(metaClient); view1.sync(); Map<String, List<String>> instantsToFiles = null; @@ -237,7 +211,7 @@ public class IncrementalFSViewSyncTest { testMultipleWriteSteps(view1, Arrays.asList("11"), true, "11"); SyncableFileSystemView view2 = - getNewFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath())); + getFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath())); // Run 2 more ingestion on MOR table. View1 is not yet synced but View2 is instantsToFiles.putAll(testMultipleWriteSteps(view2, Arrays.asList("12", "13"), true, "11")); @@ -247,7 +221,7 @@ public class IncrementalFSViewSyncTest { view2.sync(); SyncableFileSystemView view3 = - getNewFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath())); + getFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath())); partitions.stream().forEach(p -> view3.getLatestFileSlices(p).count()); view3.sync(); areViewsConsistent(view1, view2, partitions.size() * fileIdsPerPartition.size()); @@ -260,7 +234,7 @@ public class IncrementalFSViewSyncTest { view1.sync(); areViewsConsistent(view1, view2, partitions.size() * fileIdsPerPartition.size()); SyncableFileSystemView view4 = - getNewFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath())); + getFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath())); partitions.stream().forEach(p -> view4.getLatestFileSlices(p).count()); view4.sync(); @@ -275,7 +249,7 @@ public class IncrementalFSViewSyncTest { view1.sync(); areViewsConsistent(view1, view2, partitions.size() * fileIdsPerPartition.size() * 2); SyncableFileSystemView view5 = - getNewFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath())); + getFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath())); partitions.stream().forEach(p -> view5.getLatestFileSlices(p).count()); view5.sync(); @@ -296,7 +270,7 @@ public class IncrementalFSViewSyncTest { view1.sync(); areViewsConsistent(view1, view2, partitions.size() * fileIdsPerPartition.size() * 2); SyncableFileSystemView view6 = - getNewFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath())); + getFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath())); partitions.stream().forEach(p -> view5.getLatestFileSlices(p).count()); view6.sync(); @@ -375,7 +349,7 @@ public class IncrementalFSViewSyncTest { partitions.forEach(p -> Assert.assertEquals(expTotalFileSlicesPerPartition, view.getAllFileSlices(p).count())); metaClient.reloadActiveTimeline(); - SyncableFileSystemView newView = getNewFileSystemView(metaClient); + SyncableFileSystemView newView = getFileSystemView(metaClient); for (String partition : partitions) { newView.getAllFileGroups(partition).count(); } @@ -427,7 +401,7 @@ public class IncrementalFSViewSyncTest { partitions.forEach(p -> Assert.assertEquals(expTotalFileSlicesPerPartition, view.getAllFileSlices(p).count())); metaClient.reloadActiveTimeline(); - SyncableFileSystemView newView = getNewFileSystemView(metaClient); + SyncableFileSystemView newView = getFileSystemView(metaClient); for (String partition : partitions) { newView.getAllFileGroups(partition).count(); } @@ -559,7 +533,7 @@ public class IncrementalFSViewSyncTest { }); metaClient.reloadActiveTimeline(); - SyncableFileSystemView newView = getNewFileSystemView(metaClient); + SyncableFileSystemView newView = getFileSystemView(metaClient); partitions.forEach(p -> newView.getLatestFileSlices(p).count()); areViewsConsistent(view, newView, initialExpTotalFileSlices + partitions.size() * fileIdsPerPartition.size()); } @@ -681,7 +655,7 @@ public class IncrementalFSViewSyncTest { } metaClient.reloadActiveTimeline(); - SyncableFileSystemView newView = getNewFileSystemView(metaClient); + SyncableFileSystemView newView = getFileSystemView(metaClient); for (String partition : partitions) { newView.getAllFileGroups(partition).count(); } @@ -788,4 +762,10 @@ public class IncrementalFSViewSyncTest { new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, instant).getFileName())); return writeStats.stream().map(e -> e.getValue().getPath()).collect(Collectors.toList()); } + + @Override + protected HoodieTableType getTableType() { + return HoodieTableType.MERGE_ON_READ; + } + } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/RocksDBBasedIncrementalFSViewSyncTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/RocksDBBasedIncrementalFSViewSyncTest.java index a78a308..1f3c600 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/RocksDBBasedIncrementalFSViewSyncTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/RocksDBBasedIncrementalFSViewSyncTest.java @@ -26,10 +26,10 @@ import org.apache.hudi.common.table.SyncableFileSystemView; public class RocksDBBasedIncrementalFSViewSyncTest extends IncrementalFSViewSyncTest { @Override - protected SyncableFileSystemView getNewFileSystemView(HoodieTableMetaClient metaClient, HoodieTimeline timeline) + protected SyncableFileSystemView getFileSystemView(HoodieTableMetaClient metaClient, HoodieTimeline timeline) throws IOException { return new RocksDbBasedFileSystemView(metaClient, timeline, - FileSystemViewStorageConfig.newBuilder().withRocksDBPath(tmpFolder.newFolder().getAbsolutePath()) + FileSystemViewStorageConfig.newBuilder().withRocksDBPath(folder.newFolder().getAbsolutePath()) .withIncrementalTimelineSync(true).build()); } } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemViewTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemViewTest.java index 479a5cb..68c8dee 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemViewTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemViewTest.java @@ -26,7 +26,7 @@ public class RocksDbBasedFileSystemViewTest extends HoodieTableFileSystemViewTes protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) throws IOException { return new RocksDbBasedFileSystemView(metaClient, timeline, - FileSystemViewStorageConfig.newBuilder().withRocksDBPath(tmpFolder.newFolder().getAbsolutePath()) + FileSystemViewStorageConfig.newBuilder().withRocksDBPath(folder.newFolder().getAbsolutePath()) .build()); } } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/SpillableMapBasedIncrementalFSViewSyncTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/SpillableMapBasedIncrementalFSViewSyncTest.java index ce3cbdd..645c171 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/SpillableMapBasedIncrementalFSViewSyncTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/SpillableMapBasedIncrementalFSViewSyncTest.java @@ -25,7 +25,7 @@ import org.apache.hudi.common.table.SyncableFileSystemView; public class SpillableMapBasedIncrementalFSViewSyncTest extends IncrementalFSViewSyncTest { @Override - protected SyncableFileSystemView getNewFileSystemView(HoodieTableMetaClient metaClient, HoodieTimeline timeline) { + protected SyncableFileSystemView getFileSystemView(HoodieTableMetaClient metaClient, HoodieTimeline timeline) { return new SpillableMapBasedFileSystemView(metaClient, timeline, FileSystemViewStorageConfig.newBuilder().withMaxMemoryForView(0L).withIncrementalTimelineSync(true).build()); } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java index 11e0e7b..fa4f4af 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java @@ -19,7 +19,6 @@ package org.apache.hudi.common.util; import static org.apache.hudi.common.model.HoodieTestUtils.DEFAULT_PARTITION_PATHS; -import static org.apache.hudi.common.model.HoodieTestUtils.getDefaultHadoopConf; import static org.apache.hudi.common.util.CompactionTestUtils.createCompactionPlan; import static org.apache.hudi.common.util.CompactionTestUtils.scheduleCompaction; import static org.apache.hudi.common.util.CompactionTestUtils.setupAndValidateCompactionOperations; @@ -35,21 +34,19 @@ import java.util.stream.IntStream; import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieFileGroupId; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.CompactionTestUtils.TestHoodieDataFile; import org.apache.hudi.common.util.collection.Pair; import org.junit.Assert; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; -public class TestCompactionUtils { +public class TestCompactionUtils extends HoodieCommonTestHarness { private static String TEST_WRITE_TOKEN = "1-0-1"; @@ -57,17 +54,11 @@ public class TestCompactionUtils { new ImmutableMap.Builder<String, Double>() .put("key1", 1.0) .put("key2", 3.0).build(); - @Rule - public TemporaryFolder tmpFolder = new TemporaryFolder(); - private HoodieTableMetaClient metaClient; - private String basePath; private Function<Pair<String, FileSlice>, Map<String, Double>> metricsCaptureFn = (partitionFileSlice) -> metrics; @Before public void init() throws IOException { - metaClient = HoodieTestUtils.init(getDefaultHadoopConf(), - tmpFolder.getRoot().getAbsolutePath(), HoodieTableType.MERGE_ON_READ); - basePath = metaClient.getBasePath(); + initMetaClient(); } @Test @@ -237,4 +228,9 @@ public class TestCompactionUtils { }); Assert.assertEquals("Metrics set", metrics, op.getMetrics()); } + + @Override + protected HoodieTableType getTableType() { + return HoodieTableType.MERGE_ON_READ; + } } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java index 5a315e6..17efd3c 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java @@ -32,23 +32,29 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.exception.HoodieException; import org.junit.Assert; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.contrib.java.lang.system.EnvironmentVariables; -import org.junit.rules.TemporaryFolder; -public class TestFSUtils { +public class TestFSUtils extends HoodieCommonTestHarness { private static String TEST_WRITE_TOKEN = "1-0-1"; @Rule public final EnvironmentVariables environmentVariables = new EnvironmentVariables(); + @Before + public void setUp() throws IOException { + initMetaClient(); + } + @Test public void testMakeDataFileName() { String commitTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); @@ -75,13 +81,9 @@ public class TestFSUtils { * This code tests the fix by ensuring ".hoodie" and their subfolders are never processed. */ public void testProcessFiles() throws Exception { - TemporaryFolder tmpFolder = new TemporaryFolder(); - tmpFolder.create(); // All directories including marker dirs. List<String> folders = Arrays.asList("2016/04/15", "2016/05/16", ".hoodie/.temp/2/2016/04/15", ".hoodie/.temp/2/2016/05/16"); - HoodieTableMetaClient metaClient = HoodieTestUtils.init(tmpFolder.getRoot().getAbsolutePath()); - String basePath = metaClient.getBasePath(); folders.stream().forEach(f -> { try { metaClient.getFs().mkdirs(new Path(new Path(basePath), f)); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestFileIOUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestFileIOUtils.java index 222c7aa..2712136 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestFileIOUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestFileIOUtils.java @@ -27,15 +27,14 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; + +import org.apache.hudi.common.HoodieCommonTestHarness; import org.junit.Test; -import org.junit.rules.TemporaryFolder; -public class TestFileIOUtils { +public class TestFileIOUtils extends HoodieCommonTestHarness { @Test public void testMkdirAndDelete() throws IOException { - TemporaryFolder folder = new TemporaryFolder(); - folder.create(); try { FileIOUtils.mkdir(folder.getRoot()); } catch (IOException e) { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java index a63f912..eee49bc 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java @@ -21,7 +21,6 @@ package org.apache.hudi.common.util; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -34,6 +33,7 @@ import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.HoodieAvroWriteSupport; import org.apache.hudi.common.BloomFilter; +import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.parquet.avro.AvroSchemaConverter; @@ -41,19 +41,12 @@ import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.metadata.CompressionCodecName; import org.junit.Before; import org.junit.Test; -import org.junit.rules.TemporaryFolder; -public class TestParquetUtils { - - - private String basePath; +public class TestParquetUtils extends HoodieCommonTestHarness { @Before - public void setup() throws IOException { - // Create a temp folder as the base path - TemporaryFolder folder = new TemporaryFolder(); - folder.create(); - basePath = folder.getRoot().getAbsolutePath(); + public void setup() { + initPath(); } @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestRocksDBManager.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestRocksDBManager.java index 83519a3..40cb06b 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestRocksDBManager.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestRocksDBManager.java @@ -19,7 +19,6 @@ package org.apache.hudi.common.util; import java.io.File; -import java.io.IOException; import java.io.Serializable; import java.util.Arrays; import java.util.HashMap; @@ -33,14 +32,21 @@ import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; import org.apache.hudi.common.util.collection.Pair; import org.junit.AfterClass; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; public class TestRocksDBManager { private static RocksDBDAO dbManager; + @BeforeClass + public static void setUpClass() { + dbManager = new RocksDBDAO("/dummy/path", + FileSystemViewStorageConfig.newBuilder().build().newBuilder().build().getRocksdbBasePath()); + } + @AfterClass - public static void drop() throws IOException { + public static void tearDownClass() { if (dbManager != null) { dbManager.close(); dbManager = null; @@ -66,8 +72,6 @@ public class TestRocksDBManager { return new Payload(prefix, key, val, family); }).collect(Collectors.toList()); - dbManager = new RocksDBDAO("/dummy/path", - FileSystemViewStorageConfig.newBuilder().build().newBuilder().build().getRocksdbBasePath()); colFamilies.stream().forEach(family -> dbManager.dropColumnFamily(family)); colFamilies.stream().forEach(family -> dbManager.addColumnFamily(family)); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java index ab49307..091a9dc 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java @@ -35,8 +35,8 @@ import java.util.stream.Collectors; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.model.AvroBinaryTestPayload; -import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; @@ -47,19 +47,20 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.SchemaTestUtil; import org.apache.hudi.common.util.SpillableMapTestUtils; import org.apache.hudi.common.util.SpillableMapUtils; +import org.junit.Before; import org.junit.Ignore; import org.junit.Test; -public class TestDiskBasedMap { +public class TestDiskBasedMap extends HoodieCommonTestHarness { - private static final String BASE_OUTPUT_PATH = "/tmp/"; + @Before + public void setup() { + initPath(); + } @Test public void testSimpleInsert() throws IOException, URISyntaxException { - Schema schema = HoodieAvroUtils.addMetadataFields(getSimpleSchema()); - String payloadClazz = HoodieAvroPayload.class.getName(); - - DiskBasedMap records = new DiskBasedMap<>(BASE_OUTPUT_PATH); + DiskBasedMap records = new DiskBasedMap<>(basePath); List<IndexedRecord> iRecords = SchemaTestUtil.generateHoodieTestRecords(0, 100); ((GenericRecord) iRecords.get(0)).get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString(); List<String> recordKeys = SpillableMapTestUtils.upsertRecords(iRecords, records); @@ -77,10 +78,7 @@ public class TestDiskBasedMap { @Test public void testSimpleInsertWithoutHoodieMetadata() throws IOException, URISyntaxException { - Schema schema = getSimpleSchema(); - String payloadClazz = HoodieAvroPayload.class.getName(); - - DiskBasedMap records = new DiskBasedMap<>(BASE_OUTPUT_PATH); + DiskBasedMap records = new DiskBasedMap<>(basePath); List<HoodieRecord> hoodieRecords = SchemaTestUtil .generateHoodieTestRecordsWithoutHoodieMetadata(0, 1000); Set<String> recordKeys = new HashSet<>(); @@ -102,11 +100,9 @@ public class TestDiskBasedMap { @Test public void testSimpleUpsert() throws IOException, URISyntaxException { - Schema schema = HoodieAvroUtils.addMetadataFields(getSimpleSchema()); - String payloadClazz = HoodieAvroPayload.class.getName(); - DiskBasedMap records = new DiskBasedMap<>(BASE_OUTPUT_PATH); + DiskBasedMap records = new DiskBasedMap<>(basePath); List<IndexedRecord> iRecords = SchemaTestUtil.generateHoodieTestRecords(0, 100); // perform some inserts diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java index 6b61b9e..1f5d74a 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java @@ -22,7 +22,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; import java.net.URISyntaxException; @@ -32,6 +31,7 @@ import java.util.List; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; @@ -43,23 +43,20 @@ import org.apache.hudi.common.util.HoodieRecordSizeEstimator; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.SchemaTestUtil; import org.apache.hudi.common.util.SpillableMapTestUtils; -import org.junit.BeforeClass; +import org.junit.Before; import org.junit.FixMethodOrder; import org.junit.Test; import org.junit.runners.MethodSorters; @FixMethodOrder(MethodSorters.NAME_ASCENDING) -public class TestExternalSpillableMap { +public class TestExternalSpillableMap extends HoodieCommonTestHarness { - private static final String FAILURE_OUTPUT_PATH = "/tmp/test_fail"; - private static final String BASE_OUTPUT_PATH = "/tmp/"; + private static String failureOutputPath; - @BeforeClass - public static void cleanUp() { - File file = new File(BASE_OUTPUT_PATH); - file.delete(); - file = new File(FAILURE_OUTPUT_PATH); - file.delete(); + @Before + public void setUp() { + initPath(); + failureOutputPath = basePath + "/test_fail"; } @Test @@ -67,7 +64,7 @@ public class TestExternalSpillableMap { Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getSimpleSchema()); String payloadClazz = HoodieAvroPayload.class.getName(); ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records = - new ExternalSpillableMap<>(16L, BASE_OUTPUT_PATH, + new ExternalSpillableMap<>(16L, basePath, new DefaultSizeEstimator(), new HoodieRecordSizeEstimator(schema)); //16B List<IndexedRecord> iRecords = SchemaTestUtil.generateHoodieTestRecords(0, 100); @@ -89,7 +86,7 @@ public class TestExternalSpillableMap { String payloadClazz = HoodieAvroPayload.class.getName(); ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records = - new ExternalSpillableMap<>(16L, BASE_OUTPUT_PATH, + new ExternalSpillableMap<>(16L, basePath, new DefaultSizeEstimator(), new HoodieRecordSizeEstimator(schema)); //16B List<IndexedRecord> iRecords = SchemaTestUtil.generateHoodieTestRecords(0, 100); @@ -127,7 +124,7 @@ public class TestExternalSpillableMap { String payloadClazz = HoodieAvroPayload.class.getName(); ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records = - new ExternalSpillableMap<>(16L, BASE_OUTPUT_PATH, + new ExternalSpillableMap<>(16L, basePath, new DefaultSizeEstimator(), new HoodieRecordSizeEstimator(schema)); //16B List<IndexedRecord> iRecords = SchemaTestUtil.generateHoodieTestRecords(0, 100); @@ -180,10 +177,9 @@ public class TestExternalSpillableMap { @Test(expected = IOException.class) public void simpleTestWithException() throws IOException, URISyntaxException { Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getSimpleSchema()); - String payloadClazz = HoodieAvroPayload.class.getName(); ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records = - new ExternalSpillableMap<>(16L, FAILURE_OUTPUT_PATH, + new ExternalSpillableMap<>(16L, failureOutputPath, new DefaultSizeEstimator(), new HoodieRecordSizeEstimator(schema)); //16B List<IndexedRecord> iRecords = SchemaTestUtil.generateHoodieTestRecords(0, 100); @@ -202,7 +198,7 @@ public class TestExternalSpillableMap { String payloadClazz = HoodieAvroPayload.class.getName(); ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records = - new ExternalSpillableMap<>(16L, BASE_OUTPUT_PATH, + new ExternalSpillableMap<>(16L, basePath, new DefaultSizeEstimator(), new HoodieRecordSizeEstimator(schema)); //16B List<String> recordKeys = new ArrayList<>(); @@ -255,7 +251,7 @@ public class TestExternalSpillableMap { String payloadClazz = HoodieAvroPayload.class.getName(); ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records = - new ExternalSpillableMap<>(16L, BASE_OUTPUT_PATH, + new ExternalSpillableMap<>(16L, basePath, new DefaultSizeEstimator(), new HoodieRecordSizeEstimator(schema)); //16B List<String> recordKeys = new ArrayList<>(); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java index 5216a9e..1723831 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java @@ -25,20 +25,25 @@ import java.util.Iterator; import java.util.List; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.util.SchemaTestUtil; import org.apache.hudi.common.util.SpillableMapTestUtils; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; -public class TestRocksDbBasedMap { +public class TestRocksDbBasedMap extends HoodieCommonTestHarness { - private static final String BASE_OUTPUT_PATH = "/tmp/"; + @Before + public void setUp() { + initPath(); + } @Test public void testSimple() throws IOException, URISyntaxException { - RocksDBBasedMap records = new RocksDBBasedMap(BASE_OUTPUT_PATH); + RocksDBBasedMap records = new RocksDBBasedMap(basePath); List<IndexedRecord> iRecords = SchemaTestUtil.generateHoodieTestRecords(0, 100); ((GenericRecord) iRecords.get(0)).get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString(); List<String> recordKeys = SpillableMapTestUtils.upsertRecords(iRecords, records); diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java index e085c63..4eff607 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java @@ -25,26 +25,21 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; /** */ -public class TestHoodieROTablePathFilter { - - @Rule - public TemporaryFolder tmpFolder = new TemporaryFolder(); - - private HoodieTableMetaClient metaClient; +public class TestHoodieROTablePathFilter extends HoodieCommonTestHarness { @Before public void setUp() throws Exception { - this.metaClient = HoodieTestUtils.init(tmpFolder.getRoot().getAbsolutePath()); + initMetaClient(); } @Test @@ -105,5 +100,7 @@ public class TestHoodieROTablePathFilter { path = basePath + File.separator + "nonhoodiefolder/somefile"; new File(path).createNewFile(); assertTrue(pathFilter.accept(new Path("file:///" + path))); + + folder.delete(); } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieSnapshotCopier.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieSnapshotCopier.java index 00102ca..1ab1f04 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieSnapshotCopier.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieSnapshotCopier.java @@ -27,6 +27,7 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.util.FSUtils; @@ -35,9 +36,8 @@ import org.apache.spark.api.java.JavaSparkContext; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.junit.rules.TemporaryFolder; -public class TestHoodieSnapshotCopier { +public class TestHoodieSnapshotCopier extends HoodieCommonTestHarness { private static String TEST_WRITE_TOKEN = "1-0-1"; @@ -49,23 +49,17 @@ public class TestHoodieSnapshotCopier { @Before public void init() throws IOException { - try { - // Prepare directories - TemporaryFolder folder = new TemporaryFolder(); - folder.create(); - rootPath = "file://" + folder.getRoot().getAbsolutePath(); - basePath = rootPath + "/" + HoodieTestUtils.RAW_TRIPS_TEST_NAME; - outputPath = rootPath + "/output"; - - final Configuration hadoopConf = HoodieTestUtils.getDefaultHadoopConf(); - fs = FSUtils.getFs(basePath, hadoopConf); - HoodieTestUtils.init(hadoopConf, basePath); - // Start a local Spark job - SparkConf conf = new SparkConf().setAppName("snapshot-test-job").setMaster("local[2]"); - jsc = new JavaSparkContext(conf); - } catch (Exception e) { - e.printStackTrace(); - } + // Prepare directories + rootPath = "file://" + folder.getRoot().getAbsolutePath(); + basePath = rootPath + "/" + HoodieTestUtils.RAW_TRIPS_TEST_NAME; + outputPath = rootPath + "/output"; + + final Configuration hadoopConf = HoodieTestUtils.getDefaultHadoopConf(); + fs = FSUtils.getFs(basePath, hadoopConf); + HoodieTestUtils.init(hadoopConf, basePath); + // Start a local Spark job + SparkConf conf = new SparkConf().setAppName("snapshot-test-job").setMaster("local[2]"); + jsc = new JavaSparkContext(conf); } @Test