This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 926794aa74 [HUDI-5247] Clean up java client tests (#7250)
926794aa74 is described below
commit 926794aa74b71c0748acadb3fe6465dfd77446d6
Author: Shiyan Xu <[email protected]>
AuthorDate: Mon Nov 21 10:42:11 2022 +0800
[HUDI-5247] Clean up java client tests (#7250)
---
.../TestHoodieJavaWriteClientInsert.java} | 37 +++---
.../TestJavaBulkInsertInternalPartitioner.java | 4 +-
.../commit/TestJavaCopyOnWriteActionExecutor.java | 4 +-
.../hudi/testutils/HoodieJavaClientTestBase.java | 48 --------
.../testutils/HoodieJavaClientTestHarness.java | 125 +++++----------------
.../commit/TestCopyOnWriteActionExecutor.java | 2 +-
6 files changed, 51 insertions(+), 169 deletions(-)
diff --git
a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestHoodieConcatHandle.java
b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java
similarity index 89%
rename from
hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestHoodieConcatHandle.java
rename to
hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java
index 4539c6de3b..ae73b0a65d 100644
---
a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestHoodieConcatHandle.java
+++
b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java
@@ -7,18 +7,18 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
-package org.apache.hudi.table.action.commit;
+package org.apache.hudi.client;
-import org.apache.hudi.client.HoodieJavaWriteClient;
import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.engine.EngineType;
import org.apache.hudi.common.fs.FSUtils;
@@ -34,7 +34,7 @@ import org.apache.hudi.common.util.BaseFileUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.hadoop.HoodieParquetInputFormat;
import org.apache.hudi.hadoop.utils.HoodieHiveUtils;
-import org.apache.hudi.testutils.HoodieJavaClientTestBase;
+import org.apache.hudi.testutils.HoodieJavaClientTestHarness;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
@@ -50,25 +50,24 @@ import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
-import static
org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
+import static
org.apache.hudi.common.testutils.HoodieTestDataGenerator.AVRO_SCHEMA;
import static
org.apache.hudi.common.testutils.HoodieTestTable.makeNewCommitTime;
import static
org.apache.hudi.common.testutils.SchemaTestUtil.getSchemaFromResource;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
-public class TestHoodieConcatHandle extends HoodieJavaClientTestBase {
- private static final Schema SCHEMA =
getSchemaFromResource(TestJavaCopyOnWriteActionExecutor.class,
"/exampleSchema.avsc");
+public class TestHoodieJavaWriteClientInsert extends
HoodieJavaClientTestHarness {
+ private static final Schema SCHEMA =
getSchemaFromResource(TestHoodieJavaWriteClientInsert.class,
"/exampleSchema.avsc");
- private HoodieWriteConfig.Builder makeHoodieClientConfigBuilder() {
- return makeHoodieClientConfigBuilder(SCHEMA.toString());
+ private static HoodieWriteConfig.Builder
makeHoodieClientConfigBuilder(String basePath) {
+ return makeHoodieClientConfigBuilder(basePath, SCHEMA);
}
- private HoodieWriteConfig.Builder makeHoodieClientConfigBuilder(String
schema) {
- // Prepare the AvroParquetIO
+ private static HoodieWriteConfig.Builder
makeHoodieClientConfigBuilder(String basePath, Schema schema) {
return HoodieWriteConfig.newBuilder()
.withEngineType(EngineType.JAVA)
.withPath(basePath)
- .withSchema(schema);
+ .withSchema(schema.toString());
}
private FileStatus[] getIncrementalFiles(String partitionPath, String
startCommitTime, int numCommitsToPull)
@@ -99,7 +98,7 @@ public class TestHoodieConcatHandle extends
HoodieJavaClientTestBase {
@Test
public void testInsert() throws Exception {
- HoodieWriteConfig config =
makeHoodieClientConfigBuilder().withMergeAllowDuplicateOnInserts(true).build();
+ HoodieWriteConfig config =
makeHoodieClientConfigBuilder(basePath).withMergeAllowDuplicateOnInserts(true).build();
HoodieJavaWriteClient writeClient = getHoodieWriteClient(config);
metaClient = HoodieTableMetaClient.reload(metaClient);
@@ -172,7 +171,7 @@ public class TestHoodieConcatHandle extends
HoodieJavaClientTestBase {
@ParameterizedTest
@ValueSource(booleans = {false, true})
public void testInsertWithDataGenerator(boolean
mergeAllowDuplicateOnInsertsEnable) throws Exception {
- HoodieWriteConfig config =
makeHoodieClientConfigBuilder(TRIP_EXAMPLE_SCHEMA)
+ HoodieWriteConfig config = makeHoodieClientConfigBuilder(basePath,
AVRO_SCHEMA)
.withMergeAllowDuplicateOnInserts(mergeAllowDuplicateOnInsertsEnable).build();
HoodieJavaWriteClient writeClient = getHoodieWriteClient(config);
diff --git
a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestJavaBulkInsertInternalPartitioner.java
b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestJavaBulkInsertInternalPartitioner.java
index ee507b6045..16ee0f9953 100644
---
a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestJavaBulkInsertInternalPartitioner.java
+++
b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestJavaBulkInsertInternalPartitioner.java
@@ -24,7 +24,7 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.table.BulkInsertPartitioner;
-import org.apache.hudi.testutils.HoodieJavaClientTestBase;
+import org.apache.hudi.testutils.HoodieJavaClientTestHarness;
import org.apache.avro.Schema;
import org.junit.jupiter.params.ParameterizedTest;
@@ -40,7 +40,7 @@ import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.assertEquals;
-public class TestJavaBulkInsertInternalPartitioner extends
HoodieJavaClientTestBase {
+public class TestJavaBulkInsertInternalPartitioner extends
HoodieJavaClientTestHarness {
private static final Comparator<HoodieRecord> KEY_COMPARATOR =
Comparator.comparing(o -> (o.getPartitionPath() + "+" +
o.getRecordKey()));
diff --git
a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
index 28d3ac6e9b..c6d83f9e94 100644
---
a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
+++
b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
@@ -43,7 +43,7 @@ import org.apache.hudi.io.HoodieCreateHandle;
import org.apache.hudi.table.HoodieJavaCopyOnWriteTable;
import org.apache.hudi.table.HoodieJavaTable;
import org.apache.hudi.table.HoodieTable;
-import org.apache.hudi.testutils.HoodieJavaClientTestBase;
+import org.apache.hudi.testutils.HoodieJavaClientTestHarness;
import org.apache.hudi.testutils.MetadataMergeWriteStatus;
import org.apache.avro.Schema;
@@ -76,7 +76,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-public class TestJavaCopyOnWriteActionExecutor extends
HoodieJavaClientTestBase {
+public class TestJavaCopyOnWriteActionExecutor extends
HoodieJavaClientTestHarness {
private static final Logger LOG =
LogManager.getLogger(TestJavaCopyOnWriteActionExecutor.class);
private static final Schema SCHEMA =
getSchemaFromResource(TestJavaCopyOnWriteActionExecutor.class,
"/exampleSchema.avsc");
diff --git
a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestBase.java
b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestBase.java
deleted file mode 100644
index 171df5fe59..0000000000
---
a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestBase.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.testutils;
-
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-
-import java.io.IOException;
-
-/**
- * Base Class providing setup/cleanup and utility methods for testing Hoodie
Client facing tests.
- */
-public class HoodieJavaClientTestBase extends HoodieJavaClientTestHarness {
-
- @BeforeEach
- public void setUp() throws Exception {
- initResources();
- }
-
- @AfterEach
- public void tearDown() throws Exception {
- cleanupResources();
- }
-
- // Functional Interfaces for passing lambda and Hoodie Write API contexts
-
- @FunctionalInterface
- public interface Function2<R, T1, T2> {
-
- R apply(T1 v1, T2 v2) throws IOException;
- }
-}
diff --git
a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
index cd8eb33f14..3a60d98921 100644
---
a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
+++
b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
@@ -23,70 +23,47 @@ import org.apache.hudi.common.engine.EngineProperty;
import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieTableType;
-import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.testutils.HoodieTestUtils;
-import org.apache.hudi.common.testutils.minicluster.HdfsTestService;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.TestInfo;
import java.io.IOException;
-import java.io.Serializable;
import java.util.concurrent.ExecutorService;
import java.util.function.Supplier;
/**
* The test harness for resource initialization and cleanup.
*/
-public abstract class HoodieJavaClientTestHarness extends
HoodieCommonTestHarness implements Serializable {
+public abstract class HoodieJavaClientTestHarness extends
HoodieCommonTestHarness {
private static final Logger LOG =
LogManager.getLogger(HoodieJavaClientTestHarness.class);
- private String testMethodName;
- protected transient Configuration hadoopConf = null;
- protected transient HoodieJavaEngineContext context = null;
- protected transient TestJavaTaskContextSupplier taskContextSupplier = null;
- protected transient FileSystem fs;
- protected transient ExecutorService executorService;
- protected transient HoodieTableFileSystemView tableView;
- protected transient HoodieJavaWriteClient writeClient;
-
- // dfs
- protected String dfsBasePath;
- protected transient HdfsTestService hdfsTestService;
- protected transient MiniDFSCluster dfsCluster;
- protected transient DistributedFileSystem dfs;
+ protected Configuration hadoopConf;
+ protected HoodieJavaEngineContext context;
+ protected TestJavaTaskContextSupplier taskContextSupplier;
+ protected FileSystem fs;
+ protected ExecutorService executorService;
+ protected HoodieTableFileSystemView tableView;
+ protected HoodieJavaWriteClient writeClient;
@BeforeEach
- public void setTestMethodName(TestInfo testInfo) {
- if (testInfo.getTestMethod().isPresent()) {
- testMethodName = testInfo.getTestMethod().get().getName();
- } else {
- testMethodName = "Unknown";
- }
- }
-
- /**
- * Initializes resource group for the subclasses of {@link
HoodieJavaClientTestHarness}.
- */
- public void initResources() throws IOException {
- initPath();
+ protected void initResources() throws IOException {
+ basePath = tempDir.resolve("java_client_tests" +
System.currentTimeMillis()).toUri().getPath();
hadoopConf = new Configuration();
taskContextSupplier = new TestJavaTaskContextSupplier();
context = new HoodieJavaEngineContext(hadoopConf, taskContextSupplier);
+ initFileSystem(basePath, hadoopConf);
initTestDataGenerator();
- initFileSystem();
initMetaClient();
}
@@ -120,30 +97,29 @@ public abstract class HoodieJavaClientTestHarness extends
HoodieCommonTestHarnes
}
}
- /**
- * Cleanups resource group for the subclasses of {@link
HoodieJavaClientTestHarness}.
- */
- public void cleanupResources() throws IOException {
+ @AfterEach
+ protected void cleanupResources() throws IOException {
cleanupClients();
cleanupTestDataGenerator();
cleanupFileSystem();
- cleanupDFS();
cleanupExecutorService();
- System.gc();
}
- /**
- * Initializes a file system with the hadoop configuration of Spark context.
- */
- protected void initFileSystem() {
- initFileSystemWithConfiguration(hadoopConf);
+ protected void initFileSystem(String basePath, Configuration hadoopConf) {
+ if (basePath == null) {
+ throw new IllegalStateException("The base path has not been
initialized.");
+ }
+
+ fs = FSUtils.getFs(basePath, hadoopConf);
+ if (fs instanceof LocalFileSystem) {
+ LocalFileSystem lfs = (LocalFileSystem) fs;
+ // With LocalFileSystem, with checksum disabled, fs.open() returns an
inputStream which is FSInputStream
+ // This causes ClassCastExceptions in LogRecordScanner (and potentially
other places) calling fs.open
+ // So, for the tests, we enforce checksum verification to circumvent the
problem
+ lfs.setVerifyChecksum(true);
+ }
}
- /**
- * Cleanups file system.
- *
- * @throws IOException
- */
protected void cleanupFileSystem() throws IOException {
if (fs != null) {
LOG.warn("Closing file-system instance used in previous test-run");
@@ -152,12 +128,6 @@ public abstract class HoodieJavaClientTestHarness extends
HoodieCommonTestHarnes
}
}
- /**
- * Initializes an instance of {@link HoodieTableMetaClient} with a special
table type specified by
- * {@code getTableType()}.
- *
- * @throws IOException
- */
protected void initMetaClient() throws IOException {
initMetaClient(getTableType());
}
@@ -170,9 +140,6 @@ public abstract class HoodieJavaClientTestHarness extends
HoodieCommonTestHarnes
metaClient = HoodieTestUtils.init(hadoopConf, basePath, tableType);
}
- /**
- * Cleanups hoodie clients.
- */
protected void cleanupClients() {
if (metaClient != null) {
metaClient = null;
@@ -187,27 +154,6 @@ public abstract class HoodieJavaClientTestHarness extends
HoodieCommonTestHarnes
}
}
- /**
- * Cleanups the distributed file system.
- *
- * @throws IOException
- */
- protected void cleanupDFS() throws IOException {
- if (hdfsTestService != null) {
- hdfsTestService.stop();
- dfsCluster.shutdown(true, true);
- hdfsTestService = null;
- dfsCluster = null;
- dfs = null;
- }
- // Need to closeAll to clear FileSystem.Cache, required because DFS and
LocalFS used in the
- // same JVM
- FileSystem.closeAll();
- }
-
- /**
- * Cleanups the executor service.
- */
protected void cleanupExecutorService() {
if (this.executorService != null) {
this.executorService.shutdownNow();
@@ -215,22 +161,7 @@ public abstract class HoodieJavaClientTestHarness extends
HoodieCommonTestHarnes
}
}
- private void initFileSystemWithConfiguration(Configuration configuration) {
- if (basePath == null) {
- throw new IllegalStateException("The base path has not been
initialized.");
- }
-
- fs = FSUtils.getFs(basePath, configuration);
- if (fs instanceof LocalFileSystem) {
- LocalFileSystem lfs = (LocalFileSystem) fs;
- // With LocalFileSystem, with checksum disabled, fs.open() returns an
inputStream which is FSInputStream
- // This causes ClassCastExceptions in LogRecordScanner (and potentially
other places) calling fs.open
- // So, for the tests, we enforce checksum verification to circumvent the
problem
- lfs.setVerifyChecksum(true);
- }
- }
-
- public HoodieJavaWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) {
+ protected HoodieJavaWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) {
if (null != writeClient) {
writeClient.close();
writeClient = null;
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java
index 907520bfe2..ebe041d638 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java
@@ -482,7 +482,7 @@ public class TestCopyOnWriteActionExecutor extends
HoodieClientTestBase {
assertEquals(updates.size() - numRecordsInPartition,
updateStatus.get(0).get(0).getTotalErrorRecords());
}
- public void testBulkInsertRecords(String bulkInsertMode) throws Exception {
+ private void testBulkInsertRecords(String bulkInsertMode) {
HoodieWriteConfig config = HoodieWriteConfig.newBuilder()
.withPath(basePath).withSchema(TRIP_EXAMPLE_SCHEMA)
.withBulkInsertParallelism(2).withBulkInsertSortMode(bulkInsertMode).build();
