(hadoop) branch branch-3.3 updated: HDFS-17768. Observer namenode network delay causing empty block location for getBatchedListing (#7593)

[weichiu]

This is an automated email from the ASF dual-hosted git repository.

weichiu pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
     new 448bbe1b963 HDFS-17768. Observer namenode network delay causing empty 
block location for getBatchedListing (#7593)
448bbe1b963 is described below

commit 448bbe1b963ae4aa6b9475a9b33f482e39059250
Author: Dimas Shidqi Parikesit <dimasparikes...@gmail.com>
AuthorDate: Wed Apr 16 20:27:43 2025 -0400

    HDFS-17768. Observer namenode network delay causing empty block location 
for getBatchedListing (#7593)
    
    (cherry picked from commit caf8af0de3e63ba8423760582bc640d51f0d558b)
    (cherry picked from commit 8eb2211b3ffa3e3b487093eee65a77d4c115737f)
---
 .../apache/hadoop/hdfs/server/namenode/FSNamesystem.java    | 13 +++++++++++++
 .../hadoop/hdfs/server/namenode/ha/TestObserverNode.java    |  3 +++
 2 files changed, 16 insertions(+)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 5ffe50caa61..0d0cfb4a501 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4188,11 +4188,22 @@ BatchedDirectoryListing getBatchedListing(String[] 
srcs, byte[] startAfter,
           if (dirListing == null) {
             throw new FileNotFoundException("Path " + src + " does not exist");
           }
+          if (needLocation && isObserver()) {
+            for (HdfsFileStatus fs : dirListing.getPartialListing()) {
+              if (fs instanceof HdfsLocatedFileStatus) {
+                LocatedBlocks lbs = ((HdfsLocatedFileStatus) 
fs).getLocatedBlocks();
+                checkBlockLocationsWhenObserver(lbs, fs.toString());
+              }
+            }
+          }
           listing = new HdfsPartialListing(
               srcsIndex, Lists.newArrayList(dirListing.getPartialListing()));
           numEntries += listing.getPartialListing().size();
           lastListing = dirListing;
         } catch (Exception e) {
+          if (e instanceof ObserverRetryOnActiveException) {
+            throw (ObserverRetryOnActiveException) e;
+          }
           if (e instanceof AccessControlException) {
             logAuditEvent(false, operationName, src);
           }
@@ -4255,6 +4266,8 @@ BatchedDirectoryListing getBatchedListing(String[] srcs, 
byte[] startAfter,
             true,
             returnedStartAfter);
       }
+    } catch(ObserverRetryOnActiveException e){
+      throw e;
     } finally {
       readUnlock(operationName);
     }
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
index bb901423bb5..39be61f6dec 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
@@ -376,6 +376,9 @@ public void testObserverNodeBlockMissingRetry() throws 
Exception {
     dfs.getClient().getLocatedFileInfo(testPath.toString(), false);
     assertSentTo(0);
 
+    dfs.getClient().batchedListPaths(new String[]{"/"}, new byte[0], true);
+    assertSentTo(0);
+
     Mockito.reset(bmSpy);
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org