[incubator-celeborn] branch branch-0.3 updated: [CELEBORN-990] Add exception handler when calling CelebornHadoopUtils.getHadoopFS

[zhouky]

This is an automated email from the ASF dual-hosted git repository.

zhouky pushed a commit to branch branch-0.3
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/branch-0.3 by this push:
     new aaa1c471b [CELEBORN-990] Add exception handler when calling 
CelebornHadoopUtils.getHadoopFS
aaa1c471b is described below

commit aaa1c471b05f7d309ffa4af3b19ba19fbfba58a5
Author: ming.li <ming...@dmall.com>
AuthorDate: Tue Sep 19 19:44:59 2023 +0800

    [CELEBORN-990] Add exception handler when calling 
CelebornHadoopUtils.getHadoopFS
    
    Add exception handler when calling CelebornHadoopUtils.getHadoopFS(conf) on 
Master and Worker, Avoid Concealing Initialization HDFS Exception Information
    
    ### What changes were proposed in this pull request?
    
    ### Why are the changes needed?
    
    ### Does this PR introduce _any_ user-facing change?
    
    ### How was this patch tested?
    
    Closes #1923 from leemingzixxoo/main.
    
    Authored-by: ming.li <ming...@dmall.com>
    Signed-off-by: zky.zhoukeyong <zky.zhoukey...@alibaba-inc.com>
    (cherry picked from commit 428e2660bca3169a36fa01ef4691c8538345d154)
    Signed-off-by: zky.zhoukeyong <zky.zhoukey...@alibaba-inc.com>
---
 .../scala/org/apache/celeborn/service/deploy/master/Master.scala  | 8 +++++++-
 .../celeborn/service/deploy/worker/storage/StorageManager.scala   | 8 +++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git 
a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala 
b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
index 268ff2423..4344461c5 100644
--- 
a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
+++ 
b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
@@ -734,7 +734,13 @@ private[celeborn] class Master(
 
   private def checkAndCleanExpiredAppDirsOnHDFS(expiredDir: String = ""): Unit 
= {
     if (hadoopFs == null) {
-      hadoopFs = CelebornHadoopUtils.getHadoopFS(conf)
+      try {
+        hadoopFs = CelebornHadoopUtils.getHadoopFS(conf)
+      } catch {
+        case e: Exception =>
+          logError("Celeborn initialize HDFS failed.", e)
+          throw e
+      }
     }
     val hdfsWorkPath = new Path(conf.hdfsDir, conf.workerWorkingDir)
     if (hadoopFs.exists(hdfsWorkPath)) {
diff --git 
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
 
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
index 64538f1fc..f9575c780 100644
--- 
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
+++ 
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
@@ -132,7 +132,13 @@ final private[worker] class StorageManager(conf: 
CelebornConf, workerSource: Abs
   val (hdfsFlusher, _totalHdfsFlusherThread) =
     if (hasHDFSStorage) {
       logInfo(s"Initialize HDFS support with path ${hdfsDir}")
-      StorageManager.hadoopFs = CelebornHadoopUtils.getHadoopFS(conf)
+      try {
+        StorageManager.hadoopFs = CelebornHadoopUtils.getHadoopFS(conf)
+      } catch {
+        case e: Exception =>
+          logError("Celeborn initialize HDFS failed.", e)
+          throw e
+      }
       (
         Some(new HdfsFlusher(
           workerSource,