This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch release-0.14.1-spark35-scala213 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit c3d350d881afe3808f73a72019dc84b6301b0238 Author: Lin Liu <[email protected]> AuthorDate: Sat Feb 17 00:53:34 2024 -0800 [HUDI-6902] Release resources safely (#10688) --- .../apache/hudi/hive/ddl/HiveQueryDDLExecutor.java | 3 + .../apache/hudi/hive/testutils/HiveTestUtil.java | 77 ++++++++++++++--- .../schema/TestFilebasedSchemaProvider.java | 2 +- .../utilities/sources/TestSqlFileBasedSource.java | 4 +- .../utilities/testutils/UtilitiesTestBase.java | 97 ++++++++++++++++------ 5 files changed, 146 insertions(+), 37 deletions(-) diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java index 6f20d27d20b..7cba6f9b767 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java @@ -154,5 +154,8 @@ public class HiveQueryDDLExecutor extends QueryBasedDDLExecutor { if (metaStoreClient != null) { Hive.closeCurrent(); } + if (hiveDriver != null) { + hiveDriver.close(); + } } } diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java index cc7f6e7980b..22a2bd8188e 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java @@ -128,7 +128,7 @@ public class HiveTestUtil { private static DateTimeFormatter dtfOut; private static Set<String> createdTablesSet = new HashSet<>(); - public static void setUp() throws IOException, InterruptedException, HiveException, MetaException { + public static void setUp() throws Exception { configuration = new Configuration(); if (zkServer == null) { zkService = new ZookeeperTestService(configuration); @@ -158,6 +158,9 @@ public class HiveTestUtil { fileSystem = hiveSyncConfig.getHadoopFileSystem(); dtfOut = DateTimeFormatter.ofPattern("yyyy/MM/dd"); + if (ddlExecutor != null) { + ddlExecutor.close(); + } ddlExecutor = new HiveQueryDDLExecutor(hiveSyncConfig, IMetaStoreClientUtil.getMSC(hiveSyncConfig.getHiveConf())); clear(); @@ -182,18 +185,72 @@ public class HiveTestUtil { return hiveServer.getHiveConf(); } - public static void shutdown() throws IOException { - if (hiveServer != null) { - hiveServer.stop(); + public static void shutdown() { + List<String> failedReleases = new ArrayList<>(); + try { + clear(); + } catch (HiveException | MetaException | IOException he) { + he.printStackTrace(); + failedReleases.add("HiveData"); + } + + try { + if (ddlExecutor != null) { + ddlExecutor.close(); + ddlExecutor = null; + } + } catch (Exception ex) { + ex.printStackTrace(); + failedReleases.add("DDLExecutor"); + } + + try { + if (hiveServer != null) { + hiveServer.stop(); + } + } catch (Exception e) { + e.printStackTrace(); + failedReleases.add("HiveServer"); + } + + try { + if (hiveTestService != null) { + hiveTestService.stop(); + } + } catch (Exception e) { + e.printStackTrace(); + failedReleases.add("HiveTestService"); } - if (hiveTestService != null) { - hiveTestService.stop(); + + try { + if (zkServer != null) { + zkServer.shutdown(true); + } + } catch (Exception e) { + e.printStackTrace(); + failedReleases.add("ZKServer"); } - if (zkServer != null) { - zkServer.shutdown(true); + + try { + if (zkService != null) { + zkService.stop(); + } + } catch (RuntimeException re) { + re.printStackTrace(); + failedReleases.add("ZKService"); } - if (fileSystem != null) { - fileSystem.close(); + + try { + if (fileSystem != null) { + fileSystem.close(); + } + } catch (IOException ie) { + ie.printStackTrace(); + failedReleases.add("FileSystem"); + } + + if (!failedReleases.isEmpty()) { + LOG.error("Exception happened during releasing: " + String.join(",", failedReleases)); } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/schema/TestFilebasedSchemaProvider.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/schema/TestFilebasedSchemaProvider.java index 945ce6f774a..389282ddcdb 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/schema/TestFilebasedSchemaProvider.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/schema/TestFilebasedSchemaProvider.java @@ -51,7 +51,7 @@ public class TestFilebasedSchemaProvider extends UtilitiesTestBase { } @AfterAll - public static void cleanUpUtilitiesTestServices() throws IOException { + public static void cleanUpUtilitiesTestServices() { UtilitiesTestBase.cleanUpUtilitiesTestServices(); } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSqlFileBasedSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSqlFileBasedSource.java index 3f106fce994..89769954d38 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSqlFileBasedSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestSqlFileBasedSource.java @@ -62,11 +62,11 @@ public class TestSqlFileBasedSource extends UtilitiesTestBase { @BeforeAll public static void initClass() throws Exception { - UtilitiesTestBase.initTestServices(false, true, false); + UtilitiesTestBase.initTestServices(false, false, false); } @AfterAll - public static void cleanupClass() throws IOException { + public static void cleanupClass() { UtilitiesTestBase.cleanUpUtilitiesTestServices(); } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java index 0406ccddc4a..f68d88253e2 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java @@ -73,6 +73,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.io.TempDir; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.FileInputStream; @@ -102,7 +104,7 @@ import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_TABLE_NAME; * */ public class UtilitiesTestBase { - + private static final Logger LOG = LoggerFactory.getLogger(UtilitiesTestBase.class); @TempDir protected static java.nio.file.Path sharedTempDir; protected static FileSystem fs; @@ -164,39 +166,86 @@ public class UtilitiesTestBase { } @AfterAll - public static void cleanUpUtilitiesTestServices() throws IOException { - if (fs != null) { - fs.delete(new Path(basePath), true); - fs.close(); - fs = null; + public static void cleanUpUtilitiesTestServices() { + List<String> failedReleases = new ArrayList<>(); + try { + if (fs != null) { + fs.delete(new Path(basePath), true); + fs.close(); + fs = null; + } + } catch (IOException ie) { + ie.printStackTrace(); + failedReleases.add("FileSystem"); } - if (hdfsTestService != null) { - hdfsTestService.stop(); - hdfsTestService = null; + + try { + if (hdfsTestService != null) { + hdfsTestService.stop(); + hdfsTestService = null; + } + } catch (Exception e) { + e.printStackTrace(); + failedReleases.add("HdfsTestService"); } - if (hiveServer != null) { - hiveServer.stop(); - hiveServer = null; + + try { + if (hiveServer != null) { + hiveServer.stop(); + hiveServer = null; + } + } catch (Exception e) { + e.printStackTrace(); + failedReleases.add("HiveServer"); } - if (hiveTestService != null) { - hiveTestService.stop(); - hiveTestService = null; + + try { + if (hiveTestService != null) { + hiveTestService.stop(); + hiveTestService = null; + } + } catch (Exception e) { + e.printStackTrace(); + failedReleases.add("HiveTestService"); } - if (zookeeperTestService != null) { - zookeeperTestService.stop(); - zookeeperTestService = null; + + try { + if (zookeeperTestService != null) { + zookeeperTestService.stop(); + zookeeperTestService = null; + } + } catch (Exception e) { + e.printStackTrace(); + failedReleases.add("ZooKeeperTestService"); } - if (jsc != null) { - jsc.stop(); - jsc = null; + + try { + if (jsc != null) { + jsc.stop(); + jsc = null; + } + } catch (Exception e) { + e.printStackTrace(); + failedReleases.add("JSC"); } - if (sparkSession != null) { - sparkSession.close(); - sparkSession = null; + + try { + if (sparkSession != null) { + sparkSession.close(); + sparkSession = null; + } + } catch (Exception e) { + e.printStackTrace(); + failedReleases.add("SparkSession"); } + if (context != null) { context = null; } + + if (!failedReleases.isEmpty()) { + LOG.error("Exception happened during releasing: " + String.join(",", failedReleases)); + } } @BeforeEach
