[incubator-celeborn] branch main updated: [CELEBORN-990] Add exception handler when calling CelebornHadoopUtils.getHadoopFS

[zhouky]

This is an automated email from the ASF dual-hosted git repository.

zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/main by this push:
     new 428e2660b [CELEBORN-990] Add exception handler when calling 
CelebornHadoopUtils.getHadoopFS
428e2660b is described below

commit 428e2660bca3169a36fa01ef4691c8538345d154
Author: ming.li <ming...@dmall.com>
AuthorDate: Tue Sep 19 19:44:59 2023 +0800

    [CELEBORN-990] Add exception handler when calling 
CelebornHadoopUtils.getHadoopFS
    
    Add exception handler when calling CelebornHadoopUtils.getHadoopFS(conf) on 
Master and Worker, Avoid Concealing Initialization HDFS Exception Information
    
    ### What changes were proposed in this pull request?
    
    ### Why are the changes needed?
    
    ### Does this PR introduce _any_ user-facing change?
    
    ### How was this patch tested?
    
    Closes #1923 from leemingzixxoo/main.
    
    Authored-by: ming.li <ming...@dmall.com>
    Signed-off-by: zky.zhoukeyong <zky.zhoukey...@alibaba-inc.com>
---
 .../scala/org/apache/celeborn/service/deploy/master/Master.scala  | 8 +++++++-
 .../celeborn/service/deploy/worker/storage/StorageManager.scala   | 8 +++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git 
a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala 
b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
index 7cb4a12d7..f19213556 100644
--- 
a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
+++ 
b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
@@ -722,7 +722,13 @@ private[celeborn] class Master(
 
   private def checkAndCleanExpiredAppDirsOnHDFS(expiredDir: String = ""): Unit 
= {
     if (hadoopFs == null) {
-      hadoopFs = CelebornHadoopUtils.getHadoopFS(conf)
+      try {
+        hadoopFs = CelebornHadoopUtils.getHadoopFS(conf)
+      } catch {
+        case e: Exception =>
+          logError("Celeborn initialize HDFS failed.", e)
+          throw e
+      }
     }
     val hdfsWorkPath = new Path(conf.hdfsDir, conf.workerWorkingDir)
     if (hadoopFs.exists(hdfsWorkPath)) {
diff --git 
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
 
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
index 4aaedd2a8..20252e911 100644
--- 
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
+++ 
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
@@ -132,7 +132,13 @@ final private[worker] class StorageManager(conf: 
CelebornConf, workerSource: Abs
   val (hdfsFlusher, _totalHdfsFlusherThread) =
     if (hasHDFSStorage) {
       logInfo(s"Initialize HDFS support with path ${hdfsDir}")
-      StorageManager.hadoopFs = CelebornHadoopUtils.getHadoopFS(conf)
+      try {
+        StorageManager.hadoopFs = CelebornHadoopUtils.getHadoopFS(conf)
+      } catch {
+        case e: Exception =>
+          logError("Celeborn initialize HDFS failed.", e)
+          throw e
+      }
       (
         Some(new HdfsFlusher(
           workerSource,