This is an automated email from the ASF dual-hosted git repository.
codope pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 743628f5920 [HUDI-6902] Release resources safely (#10688)
743628f5920 is described below
commit 743628f5920590231a2921c6c0476ba15d180ec0
Author: Lin Liu <[email protected]>
AuthorDate: Sat Feb 17 00:53:34 2024 -0800
[HUDI-6902] Release resources safely (#10688)
---
.../apache/hudi/hive/ddl/HiveQueryDDLExecutor.java | 3 +
.../apache/hudi/hive/testutils/HiveTestUtil.java | 77 ++++++++++++++---
.../schema/TestFilebasedSchemaProvider.java | 2 +-
.../utilities/sources/TestSqlFileBasedSource.java | 4 +-
.../utilities/testutils/UtilitiesTestBase.java | 97 ++++++++++++++++------
5 files changed, 146 insertions(+), 37 deletions(-)
diff --git
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java
index 6f20d27d20b..7cba6f9b767 100644
---
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java
+++
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java
@@ -154,5 +154,8 @@ public class HiveQueryDDLExecutor extends
QueryBasedDDLExecutor {
if (metaStoreClient != null) {
Hive.closeCurrent();
}
+ if (hiveDriver != null) {
+ hiveDriver.close();
+ }
}
}
diff --git
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java
index 2c496d55662..328e6efda08 100644
---
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java
+++
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java
@@ -132,7 +132,7 @@ public class HiveTestUtil {
private static DateTimeFormatter dtfOut;
private static Set<String> createdTablesSet = new HashSet<>();
- public static void setUp(Option<TypedProperties> hiveSyncProperties, boolean
shouldClearBasePathAndTables) throws IOException, InterruptedException,
HiveException, MetaException {
+ public static void setUp(Option<TypedProperties> hiveSyncProperties, boolean
shouldClearBasePathAndTables) throws Exception {
configuration = new Configuration();
if (zkServer == null) {
zkService = new ZookeeperTestService(configuration);
@@ -166,6 +166,9 @@ public class HiveTestUtil {
fileSystem = hiveSyncConfig.getHadoopFileSystem();
dtfOut = DateTimeFormatter.ofPattern("yyyy/MM/dd");
+ if (ddlExecutor != null) {
+ ddlExecutor.close();
+ }
ddlExecutor = new HiveQueryDDLExecutor(hiveSyncConfig,
IMetaStoreClientUtil.getMSC(hiveSyncConfig.getHiveConf()));
if (shouldClearBasePathAndTables) {
@@ -196,18 +199,72 @@ public class HiveTestUtil {
return hiveSyncConfig;
}
- public static void shutdown() throws IOException {
- if (hiveServer != null) {
- hiveServer.stop();
+ public static void shutdown() {
+ List<String> failedReleases = new ArrayList<>();
+ try {
+ clear();
+ } catch (HiveException | MetaException | IOException he) {
+ he.printStackTrace();
+ failedReleases.add("HiveData");
+ }
+
+ try {
+ if (ddlExecutor != null) {
+ ddlExecutor.close();
+ ddlExecutor = null;
+ }
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ failedReleases.add("DDLExecutor");
+ }
+
+ try {
+ if (hiveServer != null) {
+ hiveServer.stop();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ failedReleases.add("HiveServer");
+ }
+
+ try {
+ if (hiveTestService != null) {
+ hiveTestService.stop();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ failedReleases.add("HiveTestService");
}
- if (hiveTestService != null) {
- hiveTestService.stop();
+
+ try {
+ if (zkServer != null) {
+ zkServer.shutdown(true);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ failedReleases.add("ZKServer");
}
- if (zkServer != null) {
- zkServer.shutdown(true);
+
+ try {
+ if (zkService != null) {
+ zkService.stop();
+ }
+ } catch (RuntimeException re) {
+ re.printStackTrace();
+ failedReleases.add("ZKService");
+ }
+
+ try {
+ if (fileSystem != null) {
+ fileSystem.close();
+ }
+ } catch (IOException ie) {
+ ie.printStackTrace();
+ failedReleases.add("FileSystem");
}
- if (fileSystem != null) {
- fileSystem.close();
+
+ if (!failedReleases.isEmpty()) {
+ LOG.error("Exception happened during releasing: " + String.join(",",
failedReleases));
}
}
diff --git
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/schema/TestFilebasedSchemaProvider.java
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/schema/TestFilebasedSchemaProvider.java
index 945ce6f774a..389282ddcdb 100644
---
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/schema/TestFilebasedSchemaProvider.java
+++
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/schema/TestFilebasedSchemaProvider.java
@@ -51,7 +51,7 @@ public class TestFilebasedSchemaProvider extends
UtilitiesTestBase {
}
@AfterAll
- public static void cleanUpUtilitiesTestServices() throws IOException {
+ public static void cleanUpUtilitiesTestServices() {
UtilitiesTestBase.cleanUpUtilitiesTestServices();
}
diff --git
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSqlFileBasedSource.java
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSqlFileBasedSource.java
index 3f106fce994..89769954d38 100644
---
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSqlFileBasedSource.java
+++
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSqlFileBasedSource.java
@@ -62,11 +62,11 @@ public class TestSqlFileBasedSource extends
UtilitiesTestBase {
@BeforeAll
public static void initClass() throws Exception {
- UtilitiesTestBase.initTestServices(false, true, false);
+ UtilitiesTestBase.initTestServices(false, false, false);
}
@AfterAll
- public static void cleanupClass() throws IOException {
+ public static void cleanupClass() {
UtilitiesTestBase.cleanUpUtilitiesTestServices();
}
diff --git
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java
index 89037ff507d..0dcba5e5fa2 100644
---
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java
+++
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java
@@ -73,6 +73,8 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.io.TempDir;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.FileInputStream;
@@ -101,7 +103,7 @@ import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_TABLE_NAME;
*
*/
public class UtilitiesTestBase {
-
+ private static final Logger LOG =
LoggerFactory.getLogger(UtilitiesTestBase.class);
@TempDir
protected static java.nio.file.Path sharedTempDir;
protected static FileSystem fs;
@@ -163,39 +165,86 @@ public class UtilitiesTestBase {
}
@AfterAll
- public static void cleanUpUtilitiesTestServices() throws IOException {
- if (fs != null) {
- fs.delete(new Path(basePath), true);
- fs.close();
- fs = null;
+ public static void cleanUpUtilitiesTestServices() {
+ List<String> failedReleases = new ArrayList<>();
+ try {
+ if (fs != null) {
+ fs.delete(new Path(basePath), true);
+ fs.close();
+ fs = null;
+ }
+ } catch (IOException ie) {
+ ie.printStackTrace();
+ failedReleases.add("FileSystem");
}
- if (hdfsTestService != null) {
- hdfsTestService.stop();
- hdfsTestService = null;
+
+ try {
+ if (hdfsTestService != null) {
+ hdfsTestService.stop();
+ hdfsTestService = null;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ failedReleases.add("HdfsTestService");
}
- if (hiveServer != null) {
- hiveServer.stop();
- hiveServer = null;
+
+ try {
+ if (hiveServer != null) {
+ hiveServer.stop();
+ hiveServer = null;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ failedReleases.add("HiveServer");
}
- if (hiveTestService != null) {
- hiveTestService.stop();
- hiveTestService = null;
+
+ try {
+ if (hiveTestService != null) {
+ hiveTestService.stop();
+ hiveTestService = null;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ failedReleases.add("HiveTestService");
}
- if (zookeeperTestService != null) {
- zookeeperTestService.stop();
- zookeeperTestService = null;
+
+ try {
+ if (zookeeperTestService != null) {
+ zookeeperTestService.stop();
+ zookeeperTestService = null;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ failedReleases.add("ZooKeeperTestService");
}
- if (jsc != null) {
- jsc.stop();
- jsc = null;
+
+ try {
+ if (jsc != null) {
+ jsc.stop();
+ jsc = null;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ failedReleases.add("JSC");
}
- if (sparkSession != null) {
- sparkSession.close();
- sparkSession = null;
+
+ try {
+ if (sparkSession != null) {
+ sparkSession.close();
+ sparkSession = null;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ failedReleases.add("SparkSession");
}
+
if (context != null) {
context = null;
}
+
+ if (!failedReleases.isEmpty()) {
+ LOG.error("Exception happened during releasing: " + String.join(",",
failedReleases));
+ }
}
@BeforeEach
