This is an automated email from the ASF dual-hosted git repository. vinoth pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
commit 5f04241fce99f47abfb7ef9f03292472b3ee23fb Author: yanghua <yanghua1...@gmail.com> AuthorDate: Tue Sep 10 15:42:54 2019 +0800 refactor code: add docs and init/cleanup resource group for hoodie client test base --- .../org/apache/hudi/HoodieClientTestHarness.java | 140 ++++++++++++++++++--- .../java/org/apache/hudi/TestAsyncCompaction.java | 18 --- .../src/test/java/org/apache/hudi/TestCleaner.java | 20 --- .../java/org/apache/hudi/TestClientRollback.java | 20 --- .../java/org/apache/hudi/TestHoodieClientBase.java | 12 ++ .../hudi/TestHoodieClientOnCopyOnWriteStorage.java | 20 --- .../java/org/apache/hudi/TestHoodieReadClient.java | 20 --- 7 files changed, 132 insertions(+), 118 deletions(-) diff --git a/hudi-client/src/test/java/org/apache/hudi/HoodieClientTestHarness.java b/hudi-client/src/test/java/org/apache/hudi/HoodieClientTestHarness.java index c8298be..04fcd4b 100644 --- a/hudi-client/src/test/java/org/apache/hudi/HoodieClientTestHarness.java +++ b/hudi-client/src/test/java/org/apache/hudi/HoodieClientTestHarness.java @@ -33,6 +33,7 @@ import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.minicluster.HdfsTestService; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.FSUtils; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SQLContext; @@ -61,6 +62,37 @@ public abstract class HoodieClientTestHarness implements Serializable { protected transient MiniDFSCluster dfsCluster; protected transient DistributedFileSystem dfs; + /** + * Initializes resource group for the subclasses of {@link TestHoodieClientBase}. + * + * @throws IOException + */ + public void initResourceGroupForHoodieClientTests() throws IOException { + initTempFolderAndPath(); + initSparkContexts(); + initTestDataGenerator(); + initFileSystem(); + initTableType(); + } + + /** + * Cleanups resource group for the subclasses of {@link TestHoodieClientBase}. + * @throws IOException + */ + public void cleanupResourceGroupForHoodieClientTests() throws IOException { + cleanupTableType(); + cleanupSparkContexts(); + cleanupTestDataGenerator(); + cleanupFileSystem(); + cleanupTempFolderAndPath(); + } + + /** + * Initializes the Spark contexts ({@link JavaSparkContext} and {@link SQLContext}) + * with the given application name. + * + * @param appName The specified application name. + */ protected void initSparkContexts(String appName) { // Initialize a local spark env jsc = new JavaSparkContext(HoodieClientTestUtils.getSparkConfForTest(appName)); @@ -70,10 +102,17 @@ public abstract class HoodieClientTestHarness implements Serializable { sqlContext = new SQLContext(jsc); } + /** + * Initializes the Spark contexts ({@link JavaSparkContext} and {@link SQLContext}) + * with a default name <b>TestHoodieClient</b>. + */ protected void initSparkContexts() { initSparkContexts("TestHoodieClient"); } + /** + * Cleanups Spark contexts ({@link JavaSparkContext} and {@link SQLContext}). + */ protected void cleanupSparkContexts() { if (sqlContext != null) { logger.info("Clearing sql context cache of spark-session used in previous test-case"); @@ -89,12 +128,22 @@ public abstract class HoodieClientTestHarness implements Serializable { } } + /** + * Initializes a temporary folder and base path. + * + * @throws IOException + */ protected void initTempFolderAndPath() throws IOException { folder = new TemporaryFolder(); folder.create(); basePath = folder.getRoot().getAbsolutePath(); } + /** + * Cleanups the temporary folder and base path. + * + * @throws IOException + */ protected void cleanupTempFolderAndPath() throws IOException { if (basePath != null) { new File(basePath).delete(); @@ -106,36 +155,29 @@ public abstract class HoodieClientTestHarness implements Serializable { } } + /** + * Initializes a file system with the hadoop configuration of Spark context. + */ protected void initFileSystem() { - if (basePath == null) { - throw new IllegalStateException("The base path has not been initialized."); - } - if (jsc == null) { throw new IllegalStateException("The Spark context has not been initialized."); } - fs = FSUtils.getFs(basePath, jsc.hadoopConfiguration()); - if (fs instanceof LocalFileSystem) { - LocalFileSystem lfs = (LocalFileSystem) fs; - // With LocalFileSystem, with checksum disabled, fs.open() returns an inputStream which is FSInputStream - // This causes ClassCastExceptions in LogRecordScanner (and potentially other places) calling fs.open - // So, for the tests, we enforce checksum verification to circumvent the problem - lfs.setVerifyChecksum(true); - } + initFileSystemWithConfiguration(jsc.hadoopConfiguration()); } + /** + * Initializes file system with a default empty configuration. + */ protected void initFileSystemWithDefaultConfiguration() { - fs = FSUtils.getFs(basePath, new Configuration()); - if (fs instanceof LocalFileSystem) { - LocalFileSystem lfs = (LocalFileSystem) fs; - // With LocalFileSystem, with checksum disabled, fs.open() returns an inputStream which is FSInputStream - // This causes ClassCastExceptions in LogRecordScanner (and potentially other places) calling fs.open - // So, for the tests, we enforce checksum verification to circumvent the problem - lfs.setVerifyChecksum(true); - } + initFileSystemWithConfiguration(new Configuration()); } + /** + * Cleanups file system. + * + * @throws IOException + */ protected void cleanupFileSystem() throws IOException { if (fs != null) { logger.warn("Closing file-system instance used in previous test-run"); @@ -143,6 +185,12 @@ public abstract class HoodieClientTestHarness implements Serializable { } } + /** + * Initializes an instance of {@link HoodieTableMetaClient} with a special table type + * specified by {@code getTableType()}. + * + * @throws IOException + */ protected void initTableType() throws IOException { if (basePath == null) { throw new IllegalStateException("The base path has not been initialized."); @@ -155,22 +203,46 @@ public abstract class HoodieClientTestHarness implements Serializable { HoodieTestUtils.initTableType(jsc.hadoopConfiguration(), basePath, getTableType()); } + /** + * Cleanups table type. + */ protected void cleanupTableType() { } + /** + * Initializes a test data generator which used to generate test datas. + * + * @throws IOException + */ protected void initTestDataGenerator() throws IOException { dataGen = new HoodieTestDataGenerator(); } + /** + * Cleanups test data generator. + * + * @throws IOException + */ protected void cleanupTestDataGenerator() throws IOException { dataGen = null; } + /** + * Gets a default {@link HoodieTableType#COPY_ON_WRITE} table type. + * Sub-classes can override this method to specify a new table type. + * + * @return an instance of Hoodie table type. + */ protected HoodieTableType getTableType() { return HoodieTableType.COPY_ON_WRITE; } + /** + * Initializes a distributed file system and base directory. + * + * @throws IOException + */ protected void initDFS() throws IOException { FileSystem.closeAll(); hdfsTestService = new HdfsTestService(); @@ -182,6 +254,11 @@ public abstract class HoodieClientTestHarness implements Serializable { dfs.mkdirs(new Path(dfsBasePath)); } + /** + * Cleanups the distributed file system. + * + * @throws IOException + */ protected void cleanupDFS() throws IOException { if (hdfsTestService != null) { hdfsTestService.stop(); @@ -192,10 +269,18 @@ public abstract class HoodieClientTestHarness implements Serializable { FileSystem.closeAll(); } + /** + * Initializes executor service with a fixed thread pool. + * + * @param threadNum specify the capacity of the fixed thread pool + */ protected void initExecutorServiceWithFixedThreadPool(int threadNum) { executorService = Executors.newFixedThreadPool(threadNum); } + /** + * Cleanups the executor service. + */ protected void cleanupExecutorService() { if (this.executorService != null) { this.executorService.shutdownNow(); @@ -203,4 +288,19 @@ public abstract class HoodieClientTestHarness implements Serializable { } } + private void initFileSystemWithConfiguration(Configuration configuration) { + if (basePath == null) { + throw new IllegalStateException("The base path has not been initialized."); + } + + fs = FSUtils.getFs(basePath, configuration); + if (fs instanceof LocalFileSystem) { + LocalFileSystem lfs = (LocalFileSystem) fs; + // With LocalFileSystem, with checksum disabled, fs.open() returns an inputStream which is FSInputStream + // This causes ClassCastExceptions in LogRecordScanner (and potentially other places) calling fs.open + // So, for the tests, we enforce checksum verification to circumvent the problem + lfs.setVerifyChecksum(true); + } + } + } diff --git a/hudi-client/src/test/java/org/apache/hudi/TestAsyncCompaction.java b/hudi-client/src/test/java/org/apache/hudi/TestAsyncCompaction.java index f26c452..e00afe4 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestAsyncCompaction.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestAsyncCompaction.java @@ -59,9 +59,7 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.HoodieTable; import org.apache.spark.api.java.JavaRDD; -import org.junit.After; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; /** @@ -86,22 +84,6 @@ public class TestAsyncCompaction extends TestHoodieClientBase { .build()); } - @Before - public void setUp() throws Exception { - initTempFolderAndPath(); - initTestDataGenerator(); - initSparkContexts(); - initTableType(); - } - - @After - public void tearDown() throws Exception { - cleanupTableType(); - cleanupSparkContexts(); - cleanupTestDataGenerator(); - cleanupTempFolderAndPath(); - } - @Test public void testRollbackForInflightCompaction() throws Exception { // Rollback inflight compaction diff --git a/hudi-client/src/test/java/org/apache/hudi/TestCleaner.java b/hudi-client/src/test/java/org/apache/hudi/TestCleaner.java index f96b192..b256e86 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestCleaner.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestCleaner.java @@ -72,9 +72,7 @@ import org.apache.spark.api.java.JavaRDD; import org.apache.spark.scheduler.SparkListener; import org.apache.spark.scheduler.SparkListenerTaskEnd; import org.apache.spark.util.AccumulatorV2; -import org.junit.After; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import scala.collection.Iterator; @@ -86,24 +84,6 @@ public class TestCleaner extends TestHoodieClientBase { private static final int BIG_BATCH_INSERT_SIZE = 500; private static Logger logger = LogManager.getLogger(TestHoodieClientBase.class); - @Before - public void setUp() throws Exception { - initTempFolderAndPath(); - initSparkContexts(); - initTestDataGenerator(); - initFileSystem(); - initTableType(); - } - - @After - public void tearDown() throws Exception { - cleanupTableType(); - cleanupSparkContexts(); - cleanupTestDataGenerator(); - cleanupFileSystem(); - cleanupTempFolderAndPath(); - } - /** * Helper method to do first batch of insert for clean by versions/commits tests * diff --git a/hudi-client/src/test/java/org/apache/hudi/TestClientRollback.java b/hudi-client/src/test/java/org/apache/hudi/TestClientRollback.java index b0ab3b5..20cc86c 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestClientRollback.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestClientRollback.java @@ -42,8 +42,6 @@ import org.apache.hudi.exception.HoodieRollbackException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.HoodieTable; import org.apache.spark.api.java.JavaRDD; -import org.junit.After; -import org.junit.Before; import org.junit.Test; /** @@ -51,24 +49,6 @@ import org.junit.Test; */ public class TestClientRollback extends TestHoodieClientBase { - @Before - public void setUp() throws Exception { - initTempFolderAndPath(); - initTestDataGenerator(); - initSparkContexts(); - initFileSystem(); - initTableType(); - } - - @After - public void tearDown() throws Exception { - cleanupTableType(); - cleanupSparkContexts(); - cleanupTestDataGenerator(); - cleanupFileSystem(); - cleanupTempFolderAndPath(); - } - /** * Test case for rollback-savepoint interaction */ diff --git a/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientBase.java b/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientBase.java index dca55d1..8de5e3a 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientBase.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientBase.java @@ -56,7 +56,9 @@ import org.apache.hudi.table.HoodieTable; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; /** * Base Class providing setup/cleanup and utility methods for testing Hoodie Client facing tests @@ -65,6 +67,16 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { protected static Logger logger = LogManager.getLogger(TestHoodieClientBase.class); + @Before + public void setUp() throws Exception { + initResourceGroupForHoodieClientTests(); + } + + @After + public void tearDown() throws Exception { + cleanupResourceGroupForHoodieClientTests(); + } + protected HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) { return getHoodieWriteClient(cfg, false); } diff --git a/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientOnCopyOnWriteStorage.java index 90a569d..e535149 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientOnCopyOnWriteStorage.java @@ -65,32 +65,12 @@ import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.HoodieTable; import org.apache.spark.api.java.JavaRDD; -import org.junit.After; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; @SuppressWarnings("unchecked") public class TestHoodieClientOnCopyOnWriteStorage extends TestHoodieClientBase { - @Before - public void setUp() throws Exception { - initTempFolderAndPath(); - initSparkContexts(); - initTestDataGenerator(); - initFileSystem(); - initTableType(); - } - - @After - public void tearDown() throws Exception { - cleanupTableType(); - cleanupTestDataGenerator(); - cleanupSparkContexts(); - cleanupFileSystem(); - cleanupTempFolderAndPath(); - } - /** * Test Auto Commit behavior for HoodieWriteClient insert API */ diff --git a/hudi-client/src/test/java/org/apache/hudi/TestHoodieReadClient.java b/hudi-client/src/test/java/org/apache/hudi/TestHoodieReadClient.java index abcc461..d279cbc 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestHoodieReadClient.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestHoodieReadClient.java @@ -28,9 +28,7 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.spark.api.java.JavaRDD; -import org.junit.After; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; @SuppressWarnings("unchecked") @@ -39,24 +37,6 @@ import org.junit.Test; */ public class TestHoodieReadClient extends TestHoodieClientBase { - @Before - public void setUp() throws Exception { - initTempFolderAndPath(); - initTestDataGenerator(); - initSparkContexts(); - initFileSystem(); - initTableType(); - } - - @After - public void tearDown() throws Exception { - cleanupTableType(); - cleanupTestDataGenerator(); - cleanupSparkContexts(); - cleanupFileSystem(); - cleanupTempFolderAndPath(); - } - /** * Test ReadFilter API after writing new records using HoodieWriteClient.insert */