(hadoop) branch trunk updated: HDFS-17376. Distcp creates Factor 1 replication file on target if Source is EC. (#6540)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 0bf439c0f9c3 HDFS-17376. Distcp creates Factor 1 replication file on 
target if Source is EC. (#6540)
0bf439c0f9c3 is described below

commit 0bf439c0f9c336a535ab8fbe97142a2016cc8168
Author: Sadanand Shenoy 
AuthorDate: Fri Feb 9 22:56:32 2024 +0530

HDFS-17376. Distcp creates Factor 1 replication file on target if Source is 
EC. (#6540)
---
 .../tools/mapred/RetriableFileCopyCommand.java |  3 +++
 .../hadoop/tools/TestDistCpWithRawXAttrs.java  | 24 +++---
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git 
a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
 
b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
index ae6b734f4860..84bb00100863 100644
--- 
a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
+++ 
b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
@@ -380,6 +380,9 @@ public class RetriableFileCopyCommand extends 
RetriableCommand {
   private static short getReplicationFactor(
   EnumSet fileAttributes, CopyListingFileStatus source,
   FileSystem targetFS, Path tmpTargetPath) {
+if (source.isErasureCoded()) {
+  return targetFS.getDefaultReplication(tmpTargetPath);
+}
 return fileAttributes.contains(FileAttribute.REPLICATION)
 ? source.getReplication()
 : targetFS.getDefaultReplication(tmpTargetPath);
diff --git 
a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java
 
b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java
index b16f6b233405..841501869b5e 100644
--- 
a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java
+++ 
b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java
@@ -62,6 +62,7 @@ public class TestDistCpWithRawXAttrs {
   private static final Path dir1 = new Path("/src/dir1");
   private static final Path subDir1 = new Path(dir1, "subdir1");
   private static final Path file1 = new Path("/src/file1");
+  private static final Path FILE_2 = new Path("/src/dir1/file2");
   private static final String rawRootName = "/.reserved/raw";
   private static final String rootedDestName = "/dest";
   private static final String rootedSrcName = "/src";
@@ -73,7 +74,7 @@ public class TestDistCpWithRawXAttrs {
 conf = new Configuration();
 conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true);
 conf.setInt(DFSConfigKeys.DFS_LIST_LIMIT, 2);
-cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true)
+cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true)
 .build();
 cluster.waitActive();
 fs = cluster.getFileSystem();
@@ -178,7 +179,7 @@ public class TestDistCpWithRawXAttrs {
   }
 
   @Test
-  public void testPreserveEC() throws Exception {
+  public void testPreserveAndNoPreserveEC() throws Exception {
 final String src = "/src";
 final String dest = "/dest";
 
@@ -190,9 +191,11 @@ public class TestDistCpWithRawXAttrs {
 
 fs.delete(new Path("/dest"), true);
 fs.mkdirs(subDir1);
-fs.create(file1).close();
 DistributedFileSystem dfs = (DistributedFileSystem) fs;
 dfs.enableErasureCodingPolicy("XOR-2-1-1024k");
+dfs.setErasureCodingPolicy(dir1, "XOR-2-1-1024k");
+fs.create(file1).close();
+fs.create(FILE_2).close();
 int res = ToolRunner.run(conf, new ECAdmin(conf), args);
 assertEquals("Unable to set EC policy on " + subDir1.toString(), res, 0);
 
@@ -203,6 +206,7 @@ public class TestDistCpWithRawXAttrs {
 FileStatus srcStatus = fs.getFileStatus(new Path(src));
 FileStatus srcDir1Status = fs.getFileStatus(dir1);
 FileStatus srcSubDir1Status = fs.getFileStatus(subDir1);
+FileStatus srcFile2Status = fs.getFileStatus(FILE_2);
 
 FileStatus destStatus = fs.getFileStatus(new Path(dest));
 FileStatus destDir1Status = fs.getFileStatus(destDir1);
@@ -214,12 +218,26 @@ public class TestDistCpWithRawXAttrs {
 destStatus.isErasureCoded());
 assertTrue("/src/dir1 is not erasure coded!",
 srcDir1Status.isErasureCoded());
+assertTrue("/src/dir1/file2 is not erasure coded",
+srcFile2Status.isErasureCoded());
 assertTrue("/dest/dir1 is not erasure coded!",
 destDir1Status.isErasureCoded());
 assertTrue("/src/dir1/subdir1 is no

[hadoop] branch branch-3.3 updated: HDFS-17237. Remove IPCLoggerChannelMetrics when the logger is closed (#6217)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 3ab30277cce8 HDFS-17237. Remove IPCLoggerChannelMetrics when the 
logger is closed (#6217)
3ab30277cce8 is described below

commit 3ab30277cce8d01da5682a06f28582e2caed3a69
Author: Stephen O'Donnell 
AuthorDate: Tue Oct 24 21:39:03 2023 +0100

HDFS-17237. Remove IPCLoggerChannelMetrics when the logger is closed (#6217)

(cherry picked from commit 882f08b4bc1d23ac3b0d78b339ddd3a5af53abdd)
---
 .../hdfs/qjournal/client/IPCLoggerChannel.java |  1 +
 .../qjournal/client/IPCLoggerChannelMetrics.java   | 39 --
 .../hdfs/qjournal/client/TestIPCLoggerChannel.java | 21 ++--
 3 files changed, 26 insertions(+), 35 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
index 94e945697129..55118317657a 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
@@ -217,6 +217,7 @@ public class IPCLoggerChannel implements AsyncLogger {
   // making any more calls after this point (eg clear the queue)
   RPC.stopProxy(proxy);
 }
+metrics.unregister();
   }
   
   protected QJournalProtocol getProxy() throws IOException {
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java
index 6eef8ffd3862..c1e27e2e98a7 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.hdfs.qjournal.client;
 
 import java.net.InetSocketAddress;
-import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -29,8 +28,6 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.lib.MetricsRegistry;
 import org.apache.hadoop.metrics2.lib.MutableQuantiles;
 
-import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
-
 /**
  * The metrics for a journal from the writer's perspective.
  */
@@ -43,21 +40,6 @@ class IPCLoggerChannelMetrics {
   private final MutableQuantiles[] writeEndToEndLatencyQuantiles;
   private final MutableQuantiles[] writeRpcLatencyQuantiles;
 
-  
-  /**
-   * In the case of the NN transitioning between states, edit logs are closed
-   * and reopened. Thus, the IPCLoggerChannel instance that writes to a
-   * given JournalNode may change over the lifetime of the process.
-   * However, metrics2 doesn't have a function to unregister a set of metrics
-   * and fails if a new metrics class is registered with the same name
-   * as the existing one. Hence, we have to maintain our own registry
-   * ("multiton") here, so that we have exactly one metrics instance
-   * per JournalNode, and switch out the pointer to the underlying
-   * IPCLoggerChannel instance.
-   */
-  private static final Map REGISTRY =
-  Maps.newHashMap();
-  
   private IPCLoggerChannelMetrics(IPCLoggerChannel ch) {
 this.ch = ch;
 
@@ -81,25 +63,16 @@ class IPCLoggerChannelMetrics {
   writeRpcLatencyQuantiles = null;
 }
   }
-  
-  private void setChannel(IPCLoggerChannel ch) {
-assert ch.getRemoteAddress().equals(this.ch.getRemoteAddress());
-this.ch = ch;
+
+  public void unregister() {
+DefaultMetricsSystem.instance().unregisterSource(getName(ch));
   }
 
   static IPCLoggerChannelMetrics create(IPCLoggerChannel ch) {
 String name = getName(ch);
-synchronized (REGISTRY) {
-  IPCLoggerChannelMetrics m = REGISTRY.get(name);
-  if (m != null) {
-m.setChannel(ch);
-  } else {
-m = new IPCLoggerChannelMetrics(ch);
-DefaultMetricsSystem.instance().register(name, null, m);
-REGISTRY.put(name, m);
-  }
-  return m;
-}
+IPCLoggerChannelMetrics m = new IPCLoggerChannelMetrics(ch);
+DefaultMetricsSystem.instance().register(name, null, m);
+return m;
   }
 
   private static String getName(IPCLoggerChannel ch) {
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestIPCLoggerChannel.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestIPCLoggerChannel.

[hadoop] branch trunk updated: HDFS-17237. Remove IPCLoggerChannelMetrics when the logger is closed (#6217)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 882f08b4bc1d HDFS-17237. Remove IPCLoggerChannelMetrics when the 
logger is closed (#6217)
882f08b4bc1d is described below

commit 882f08b4bc1d23ac3b0d78b339ddd3a5af53abdd
Author: Stephen O'Donnell 
AuthorDate: Tue Oct 24 21:39:03 2023 +0100

HDFS-17237. Remove IPCLoggerChannelMetrics when the logger is closed (#6217)
---
 .../hdfs/qjournal/client/IPCLoggerChannel.java |  1 +
 .../qjournal/client/IPCLoggerChannelMetrics.java   | 39 --
 .../hdfs/qjournal/client/TestIPCLoggerChannel.java | 21 ++--
 3 files changed, 26 insertions(+), 35 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
index 4b7e59c51f13..67fc85810278 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
@@ -206,6 +206,7 @@ public class IPCLoggerChannel implements AsyncLogger {
   // making any more calls after this point (eg clear the queue)
   RPC.stopProxy(proxy);
 }
+metrics.unregister();
   }
   
   protected QJournalProtocol getProxy() throws IOException {
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java
index 6eef8ffd3862..c1e27e2e98a7 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.hdfs.qjournal.client;
 
 import java.net.InetSocketAddress;
-import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -29,8 +28,6 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.lib.MetricsRegistry;
 import org.apache.hadoop.metrics2.lib.MutableQuantiles;
 
-import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
-
 /**
  * The metrics for a journal from the writer's perspective.
  */
@@ -43,21 +40,6 @@ class IPCLoggerChannelMetrics {
   private final MutableQuantiles[] writeEndToEndLatencyQuantiles;
   private final MutableQuantiles[] writeRpcLatencyQuantiles;
 
-  
-  /**
-   * In the case of the NN transitioning between states, edit logs are closed
-   * and reopened. Thus, the IPCLoggerChannel instance that writes to a
-   * given JournalNode may change over the lifetime of the process.
-   * However, metrics2 doesn't have a function to unregister a set of metrics
-   * and fails if a new metrics class is registered with the same name
-   * as the existing one. Hence, we have to maintain our own registry
-   * ("multiton") here, so that we have exactly one metrics instance
-   * per JournalNode, and switch out the pointer to the underlying
-   * IPCLoggerChannel instance.
-   */
-  private static final Map REGISTRY =
-  Maps.newHashMap();
-  
   private IPCLoggerChannelMetrics(IPCLoggerChannel ch) {
 this.ch = ch;
 
@@ -81,25 +63,16 @@ class IPCLoggerChannelMetrics {
   writeRpcLatencyQuantiles = null;
 }
   }
-  
-  private void setChannel(IPCLoggerChannel ch) {
-assert ch.getRemoteAddress().equals(this.ch.getRemoteAddress());
-this.ch = ch;
+
+  public void unregister() {
+DefaultMetricsSystem.instance().unregisterSource(getName(ch));
   }
 
   static IPCLoggerChannelMetrics create(IPCLoggerChannel ch) {
 String name = getName(ch);
-synchronized (REGISTRY) {
-  IPCLoggerChannelMetrics m = REGISTRY.get(name);
-  if (m != null) {
-m.setChannel(ch);
-  } else {
-m = new IPCLoggerChannelMetrics(ch);
-DefaultMetricsSystem.instance().register(name, null, m);
-REGISTRY.put(name, m);
-  }
-  return m;
-}
+IPCLoggerChannelMetrics m = new IPCLoggerChannelMetrics(ch);
+DefaultMetricsSystem.instance().register(name, null, m);
+return m;
   }
 
   private static String getName(IPCLoggerChannel ch) {
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestIPCLoggerChannel.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestIPCLoggerChannel.java
index f2f46424cfd5..06df99de1fe8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/tes

[hadoop] branch branch-3.2 updated: HDFS-16942. Addendum. Send error to datanode if FBR is rejected due to bad lease (#5478). Contributed by Stephen O'Donnell/

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new 822c7eee0bc HDFS-16942. Addendum. Send error to datanode if FBR is 
rejected due to bad lease (#5478). Contributed by Stephen O'Donnell/
822c7eee0bc is described below

commit 822c7eee0bcad4f3280a289f3ea548a0f12156ee
Author: Stephen O'Donnell 
AuthorDate: Wed Mar 15 04:33:00 2023 +

HDFS-16942. Addendum. Send error to datanode if FBR is rejected due to bad 
lease (#5478). Contributed by Stephen O'Donnell/

(cherry picked from commit eee2ea075d10b197837ceca8cbb2260b48b29d10)
---
 hadoop-client-modules/hadoop-client-api/pom.xml|  6 -
 .../hadoop/hdfs/server/protocol/package-info.java  | 27 --
 2 files changed, 33 deletions(-)

diff --git a/hadoop-client-modules/hadoop-client-api/pom.xml 
b/hadoop-client-modules/hadoop-client-api/pom.xml
index 266742cdbd8..41d7ccca224 100644
--- a/hadoop-client-modules/hadoop-client-api/pom.xml
+++ b/hadoop-client-modules/hadoop-client-api/pom.xml
@@ -122,12 +122,6 @@
 
org/apache/hadoop/yarn/client/api/package-info.class
   
 
-
-  org.apache.hadoop:*
-  
-
org/apache/hadoop/hdfs/server/protocol/package-info.class
-  
-
   
   
 
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/package-info.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/package-info.java
deleted file mode 100644
index 21743595548..000
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/package-info.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package provides classes for the namenode server protocol.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-package org.apache.hadoop.hdfs.server.protocol;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.3 updated: HDFS-16942. Addendum. Send error to datanode if FBR is rejected due to bad lease (#5478). Contributed by Stephen O'Donnell/

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new c39696ab1dd HDFS-16942. Addendum. Send error to datanode if FBR is 
rejected due to bad lease (#5478). Contributed by Stephen O'Donnell/
c39696ab1dd is described below

commit c39696ab1dd6a18ef1e6ac7623d452501b2f951a
Author: Stephen O'Donnell 
AuthorDate: Wed Mar 15 04:33:00 2023 +

HDFS-16942. Addendum. Send error to datanode if FBR is rejected due to bad 
lease (#5478). Contributed by Stephen O'Donnell/

(cherry picked from commit eee2ea075d10b197837ceca8cbb2260b48b29d10)
---
 hadoop-client-modules/hadoop-client-api/pom.xml|  6 -
 .../hadoop/hdfs/server/protocol/package-info.java  | 27 --
 2 files changed, 33 deletions(-)

diff --git a/hadoop-client-modules/hadoop-client-api/pom.xml 
b/hadoop-client-modules/hadoop-client-api/pom.xml
index bf3dcbb7e44..77051050c20 100644
--- a/hadoop-client-modules/hadoop-client-api/pom.xml
+++ b/hadoop-client-modules/hadoop-client-api/pom.xml
@@ -133,12 +133,6 @@
 
org/apache/hadoop/yarn/client/api/package-info.class
   
 
-
-  org.apache.hadoop:*
-  
-
org/apache/hadoop/hdfs/server/protocol/package-info.class
-  
-
   
   
 
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/package-info.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/package-info.java
deleted file mode 100644
index 21743595548..000
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/package-info.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package provides classes for the namenode server protocol.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-package org.apache.hadoop.hdfs.server.protocol;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.2 updated: HDFS-16942. Send error to datanode if FBR is rejected due to bad lease (#5460)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new 8bb0b577971 HDFS-16942. Send error to datanode if FBR is rejected due 
to bad lease (#5460)
8bb0b577971 is described below

commit 8bb0b57797114b2b751600f4b00d449a848a27d8
Author: Stephen O'Donnell 
AuthorDate: Sat Mar 11 16:40:07 2023 +

HDFS-16942. Send error to datanode if FBR is rejected due to bad lease 
(#5460)

(cherry picked from commit d7b89d08e180db3e9ef3d5c3e9d8e8136c348dd7)

 Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportLease.java
---
 hadoop-client-modules/hadoop-client-api/pom.xml|  6 ++
 .../hdfs/server/datanode/BPServiceActor.java   |  4 ++
 .../hdfs/server/namenode/NameNodeRpcServer.java|  3 +
 .../protocol/InvalidBlockReportLeaseException.java | 41 +
 .../hadoop/hdfs/server/protocol/package-info.java  | 27 +
 .../blockmanagement/TestBlockReportLease.java  | 69 ++
 .../hdfs/server/datanode/TestBPOfferService.java   |  7 ++-
 7 files changed, 154 insertions(+), 3 deletions(-)

diff --git a/hadoop-client-modules/hadoop-client-api/pom.xml 
b/hadoop-client-modules/hadoop-client-api/pom.xml
index 41d7ccca224..266742cdbd8 100644
--- a/hadoop-client-modules/hadoop-client-api/pom.xml
+++ b/hadoop-client-modules/hadoop-client-api/pom.xml
@@ -122,6 +122,12 @@
 
org/apache/hadoop/yarn/client/api/package-info.class
   
 
+
+  org.apache.hadoop:*
+  
+
org/apache/hadoop/hdfs/server/protocol/package-info.class
+  
+
   
   
 
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index 0b3eb14dff8..2ce0d8ba272 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -57,6 +57,7 @@ import 
org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
 import org.apache.hadoop.hdfs.server.protocol.DisallowedDatanodeException;
 import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
+import org.apache.hadoop.hdfs.server.protocol.InvalidBlockReportLeaseException;
 import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
 import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
 import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports;
@@ -726,6 +727,9 @@ class BPServiceActor implements Runnable {
   shouldServiceRun = false;
   return;
 }
+if (InvalidBlockReportLeaseException.class.getName().equals(reClass)) {
+  fullBlockReportLeaseId = 0;
+}
 LOG.warn("RemoteException in offerService", re);
 sleepAfterException();
   } catch (IOException e) {
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 9ab79018a4f..377b00d7570 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -168,6 +168,7 @@ import 
org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
 import org.apache.hadoop.hdfs.server.protocol.FinalizeCommand;
 import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
+import org.apache.hadoop.hdfs.server.protocol.InvalidBlockReportLeaseException;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
@@ -1607,6 +1608,8 @@ public class NameNodeRpcServer implements 
NamenodeProtocols {
 bm.processReport(nodeReg, reports[index].getStorage(),
 blocks, context));
 }
+  } else {
+throw new InvalidBlockReportLeaseException(context.getReportId(), 
context.getLeaseId());
   }
 } catch (UnregisteredNodeException une) {
   LOG.debug("Datanode {} is attempting to report but not register yet.",

[hadoop] branch branch-3.3 updated: HDFS-16942. Send error to datanode if FBR is rejected due to bad lease (#5460)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new d7b89d08e18 HDFS-16942. Send error to datanode if FBR is rejected due 
to bad lease (#5460)
d7b89d08e18 is described below

commit d7b89d08e180db3e9ef3d5c3e9d8e8136c348dd7
Author: Stephen O'Donnell 
AuthorDate: Sat Mar 11 16:40:07 2023 +

HDFS-16942. Send error to datanode if FBR is rejected due to bad lease 
(#5460)

(cherry picked from commit ca6f5afb6da7f1c40ba42061d34b60b54bb01712)

 Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportLease.java
---
 hadoop-client-modules/hadoop-client-api/pom.xml|  6 ++
 .../hdfs/server/datanode/BPServiceActor.java   |  4 ++
 .../hdfs/server/namenode/NameNodeRpcServer.java|  3 +
 .../protocol/InvalidBlockReportLeaseException.java | 41 +
 .../hadoop/hdfs/server/protocol/package-info.java  | 27 +
 .../blockmanagement/TestBlockReportLease.java  | 69 ++
 .../hdfs/server/datanode/TestBPOfferService.java   |  7 ++-
 7 files changed, 154 insertions(+), 3 deletions(-)

diff --git a/hadoop-client-modules/hadoop-client-api/pom.xml 
b/hadoop-client-modules/hadoop-client-api/pom.xml
index 77051050c20..bf3dcbb7e44 100644
--- a/hadoop-client-modules/hadoop-client-api/pom.xml
+++ b/hadoop-client-modules/hadoop-client-api/pom.xml
@@ -133,6 +133,12 @@
 
org/apache/hadoop/yarn/client/api/package-info.class
   
 
+
+  org.apache.hadoop:*
+  
+
org/apache/hadoop/hdfs/server/protocol/package-info.class
+  
+
   
   
 
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index d7ea8a83739..83f68b43705 100755
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -62,6 +62,7 @@ import 
org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
 import org.apache.hadoop.hdfs.server.protocol.DisallowedDatanodeException;
 import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
+import org.apache.hadoop.hdfs.server.protocol.InvalidBlockReportLeaseException;
 import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
 import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
 import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports;
@@ -778,6 +779,9 @@ class BPServiceActor implements Runnable {
   shouldServiceRun = false;
   return;
 }
+if (InvalidBlockReportLeaseException.class.getName().equals(reClass)) {
+  fullBlockReportLeaseId = 0;
+}
 LOG.warn("RemoteException in offerService", re);
 sleepAfterException();
   } catch (IOException e) {
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index f1014a90fc5..badbc647763 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -173,6 +173,7 @@ import 
org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
 import org.apache.hadoop.hdfs.server.protocol.FinalizeCommand;
 import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
+import org.apache.hadoop.hdfs.server.protocol.InvalidBlockReportLeaseException;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
@@ -1635,6 +1636,8 @@ public class NameNodeRpcServer implements 
NamenodeProtocols {
 bm.processReport(nodeReg, reports[index].getStorage(),
 blocks, context));
 }
+  } else {
+throw new InvalidBlockReportLeaseException(context.getReportId(), 
context.getLeaseId());
   }
 } catch (UnregisteredNodeException une) {
   LOG.debug("Datanode {} is attempting to report but not register yet.",

[hadoop] reference refs/remotes/asf/branch-3.3 created (now 8cc57f5eb12)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a change to reference refs/remotes/asf/branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


  at 8cc57f5eb12 HDFS-16939. Fix the thread safety bug in 
LowRedundancyBlocks. (#5450 #5471). Contributed by Shuyan Zhang.

No new revisions were added by this update.


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated (734f7abfb8b -> ca6f5afb6da)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


from 734f7abfb8b HADOOP-18646. Upgrade Netty to 4.1.89.Final to fix 
CVE-2022-41881 (#5435)
 add ca6f5afb6da HDFS-16942. Send error to datanode if FBR is rejected due 
to bad lease (#5460)

No new revisions were added by this update.

Summary of changes:
 hadoop-client-modules/hadoop-client-api/pom.xml|  6 ++
 .../hdfs/server/datanode/BPServiceActor.java   |  4 ++
 .../hdfs/server/namenode/NameNodeRpcServer.java|  3 +
 .../InvalidBlockReportLeaseException.java} | 15 +++--
 .../hadoop/hdfs/server/protocol}/package-info.java |  7 ++-
 .../blockmanagement/TestBlockReportLease.java  | 69 ++
 .../hdfs/server/datanode/TestBPOfferService.java   |  7 ++-
 7 files changed, 99 insertions(+), 12 deletions(-)
 copy 
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/{blockmanagement/OutOfLegacyGenerationStampsException.java
 => protocol/InvalidBlockReportLeaseException.java} (66%)
 copy 
{hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics
 => 
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol}/package-info.java
 (89%)


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: HDFS-16761. Namenode UI for Datanodes page not loading if any data node is down (#5390)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 4cbe19f3a22 HDFS-16761. Namenode UI for Datanodes page not loading if 
any data node is down (#5390)
4cbe19f3a22 is described below

commit 4cbe19f3a222576970e300db60a41f78545e9524
Author: Zita Dombi <50611074+dombiz...@users.noreply.github.com>
AuthorDate: Wed Feb 15 17:16:04 2023 +0100

HDFS-16761. Namenode UI for Datanodes page not loading if any data node is 
down (#5390)
---
 .../hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html| 1 +
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html | 1 +
 2 files changed, 2 insertions(+)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
index 9fa43a30858..f9a8d7a49b1 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
@@ -390,6 +390,7 @@
 
 
 
+
   
   {/DeadNodes}
 
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
index 4bf2e2bd7f5..889bc8aff51 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
@@ -361,6 +361,7 @@
 
 
 
+
   
   {/DeadNodes}
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.2 updated: HDFS-16610. Make fsck read timeout configurable (#4384)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new 58513d367e2 HDFS-16610. Make fsck read timeout configurable (#4384)
58513d367e2 is described below

commit 58513d367e2c3ae7581306a670ce8197f5edc465
Author: Stephen O'Donnell 
AuthorDate: Wed Jun 1 20:36:01 2022 +0100

HDFS-16610. Make fsck read timeout configurable (#4384)

(cherry picked from commit 34a973a90ef89b633c9b5c13a79aa1ac11c92eb5)

 Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml

(cherry picked from commit 7d6b133af3f7ee4b986bdd2b266b83693614b6ab)
---
 .../hadoop/hdfs/client/HdfsClientConfigKeys.java  |  8 
 .../main/java/org/apache/hadoop/hdfs/tools/DFSck.java | 13 -
 .../hadoop-hdfs/src/main/resources/hdfs-default.xml   | 19 +++
 3 files changed, 39 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index b74c1194cf5..26834ecf258 100755
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -205,6 +205,14 @@ public interface HdfsClientConfigKeys {
   String DFS_LEASE_HARDLIMIT_KEY = "dfs.namenode.lease-hard-limit-sec";
   long DFS_LEASE_HARDLIMIT_DEFAULT = 20 * 60;
 
+  String DFS_CLIENT_FSCK_CONNECT_TIMEOUT =
+  "dfs.client.fsck.connect.timeout";
+  int DFS_CLIENT_FSCK_CONNECT_TIMEOUT_DEFAULT = 60 * 1000;
+
+  String DFS_CLIENT_FSCK_READ_TIMEOUT =
+  "dfs.client.fsck.read.timeout";
+  int DFS_CLIENT_FSCK_READ_TIMEOUT_DEFAULT = 60 * 1000;
+
   /**
* These are deprecated config keys to client code.
*/
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index 8e6ed4a5726..ade461aa24c 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -27,6 +27,7 @@ import java.net.URL;
 import java.net.URLConnection;
 import java.net.URLEncoder;
 import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
@@ -37,6 +38,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HAUtil;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 import org.apache.hadoop.hdfs.server.namenode.NamenodeFsck;
 import org.apache.hadoop.hdfs.web.URLConnectionFactory;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -132,8 +134,17 @@ public class DFSck extends Configured implements Tool {
 super(conf);
 this.ugi = UserGroupInformation.getCurrentUser();
 this.out = out;
+int connectTimeout = (int) conf.getTimeDuration(
+HdfsClientConfigKeys.DFS_CLIENT_FSCK_CONNECT_TIMEOUT,
+HdfsClientConfigKeys.DFS_CLIENT_FSCK_CONNECT_TIMEOUT_DEFAULT,
+TimeUnit.MILLISECONDS);
+int readTimeout = (int) conf.getTimeDuration(
+HdfsClientConfigKeys.DFS_CLIENT_FSCK_READ_TIMEOUT,
+HdfsClientConfigKeys.DFS_CLIENT_FSCK_READ_TIMEOUT_DEFAULT,
+TimeUnit.MILLISECONDS);
+
 this.connectionFactory = URLConnectionFactory
-.newDefaultURLConnectionFactory(conf);
+.newDefaultURLConnectionFactory(connectTimeout, readTimeout, conf);
 this.isSpnegoEnabled = UserGroupInformation.isSecurityEnabled();
   }
 
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index f304ed0a720..702618cffe6 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -5508,4 +5508,23 @@
   Determines the namenode automatic lease recovery interval in seconds.
 
   
+
+  
+dfs.client.fsck.connect.timeout
+6ms
+
+  The amount of time the fsck client will wait to connect to the namenode
+  before timing out.
+
+  
+
+  
+dfs.client.fsck.read.timeout
+6ms
+
+  The amount of time the fsck client will wait to read from the namenode
+  before timing out. If the namenode does not report progress more
+ 

[hadoop] branch branch-3.3 updated: HDFS-16610. Make fsck read timeout configurable (#4384)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 7d6b133af3f HDFS-16610. Make fsck read timeout configurable (#4384)
7d6b133af3f is described below

commit 7d6b133af3f7ee4b986bdd2b266b83693614b6ab
Author: Stephen O'Donnell 
AuthorDate: Wed Jun 1 20:36:01 2022 +0100

HDFS-16610. Make fsck read timeout configurable (#4384)

(cherry picked from commit 34a973a90ef89b633c9b5c13a79aa1ac11c92eb5)

 Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
---
 .../hadoop/hdfs/client/HdfsClientConfigKeys.java  |  8 
 .../main/java/org/apache/hadoop/hdfs/tools/DFSck.java | 13 -
 .../hadoop-hdfs/src/main/resources/hdfs-default.xml   | 19 +++
 3 files changed, 39 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index cf12793c868..e6af2dbecc5 100755
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -261,6 +261,14 @@ public interface HdfsClientConfigKeys {
   String DFS_LEASE_HARDLIMIT_KEY = "dfs.namenode.lease-hard-limit-sec";
   long DFS_LEASE_HARDLIMIT_DEFAULT = 20 * 60;
 
+  String DFS_CLIENT_FSCK_CONNECT_TIMEOUT =
+  "dfs.client.fsck.connect.timeout";
+  int DFS_CLIENT_FSCK_CONNECT_TIMEOUT_DEFAULT = 60 * 1000;
+
+  String DFS_CLIENT_FSCK_READ_TIMEOUT =
+  "dfs.client.fsck.read.timeout";
+  int DFS_CLIENT_FSCK_READ_TIMEOUT_DEFAULT = 60 * 1000;
+
   /**
* These are deprecated config keys to client code.
*/
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index 8a2ef8b5920..33a117f314e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -27,6 +27,7 @@ import java.net.URL;
 import java.net.URLConnection;
 import java.net.URLEncoder;
 import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
@@ -37,6 +38,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HAUtil;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 import org.apache.hadoop.hdfs.server.namenode.NamenodeFsck;
 import org.apache.hadoop.hdfs.web.URLConnectionFactory;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -137,8 +139,17 @@ public class DFSck extends Configured implements Tool {
 super(conf);
 this.ugi = UserGroupInformation.getCurrentUser();
 this.out = out;
+int connectTimeout = (int) conf.getTimeDuration(
+HdfsClientConfigKeys.DFS_CLIENT_FSCK_CONNECT_TIMEOUT,
+HdfsClientConfigKeys.DFS_CLIENT_FSCK_CONNECT_TIMEOUT_DEFAULT,
+TimeUnit.MILLISECONDS);
+int readTimeout = (int) conf.getTimeDuration(
+HdfsClientConfigKeys.DFS_CLIENT_FSCK_READ_TIMEOUT,
+HdfsClientConfigKeys.DFS_CLIENT_FSCK_READ_TIMEOUT_DEFAULT,
+TimeUnit.MILLISECONDS);
+
 this.connectionFactory = URLConnectionFactory
-.newDefaultURLConnectionFactory(conf);
+.newDefaultURLConnectionFactory(connectTimeout, readTimeout, conf);
 this.isSpnegoEnabled = UserGroupInformation.isSecurityEnabled();
   }
 
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 0b573ddaf64..f8abb59daa0 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -6174,4 +6174,23 @@
   set on fs.protected.directories.
 
   
+
+  
+dfs.client.fsck.connect.timeout
+6ms
+
+  The amount of time the fsck client will wait to connect to the namenode
+  before timing out.
+
+  
+
+  
+dfs.client.fsck.read.timeout
+6ms
+
+  The amount of time the fsck client will wait to read from the namenode
+  before timing out. If the namenode does not report progress more
+  frequently than this time, t

[hadoop] branch trunk updated (4496e2daf63 -> 34a973a90ef)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


from 4496e2daf63 HDFS-16604. Install gtest via FetchContent_Declare in 
CMake (#4374)
 add 34a973a90ef HDFS-16610. Make fsck read timeout configurable (#4384)

No new revisions were added by this update.

Summary of changes:
 .../apache/hadoop/hdfs/client/HdfsClientConfigKeys.java |  8 
 .../main/java/org/apache/hadoop/hdfs/tools/DFSck.java   | 13 -
 .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 17 +
 3 files changed, 37 insertions(+), 1 deletion(-)


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.2 updated: Revert "HDFS-16531. Avoid setReplication writing an edit record if old replication equals the new value (#4148). Contributed by Stephen O'Donnell."

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new bacd0b9a928 Revert "HDFS-16531. Avoid setReplication writing an edit 
record if old replication equals the new value (#4148). Contributed by Stephen 
O'Donnell."
bacd0b9a928 is described below

commit bacd0b9a9289172efb21f9c9afcd7b04ea0732b3
Author: S O'Donnell 
AuthorDate: Wed Apr 20 20:57:23 2022 +0100

Revert "HDFS-16531. Avoid setReplication writing an edit record if old 
replication equals the new value (#4148). Contributed by Stephen O'Donnell."

This reverts commit 045d48530cbbd7baa2b67937e103b9400487f1a4.
---
 .../hadoop/hdfs/server/namenode/FSDirAttrOp.java   | 31 +++---
 .../hadoop/hdfs/server/namenode/FSNamesystem.java  | 11 
 .../apache/hadoop/hdfs/TestSetrepIncreasing.java   |  6 -
 3 files changed, 15 insertions(+), 33 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index 99ce83d0bdb..d65b84cc610 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -50,11 +50,6 @@ import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_QUOTA_BY_STORAGETYPE_ENAB
 import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY;
 
 public class FSDirAttrOp {
-
-  protected enum SetRepStatus {
-UNCHANGED, INVALID, SUCCESS
-  }
-
   static FileStatus setPermission(
   FSDirectory fsd, FSPermissionChecker pc, final String src,
   FsPermission permission) throws IOException {
@@ -135,11 +130,11 @@ public class FSDirAttrOp {
 return fsd.getAuditFileInfo(iip);
   }
 
-  static SetRepStatus setReplication(
+  static boolean setReplication(
   FSDirectory fsd, FSPermissionChecker pc, BlockManager bm, String src,
   final short replication) throws IOException {
 bm.verifyReplication(src, replication, null);
-final SetRepStatus status;
+final boolean isFile;
 fsd.writeLock();
 try {
   final INodesInPath iip = fsd.resolvePath(pc, src, DirOp.WRITE);
@@ -147,14 +142,16 @@ public class FSDirAttrOp {
 fsd.checkPathAccess(pc, iip, FsAction.WRITE);
   }
 
-  status = unprotectedSetReplication(fsd, iip, replication);
-  if (status == SetRepStatus.SUCCESS) {
+  final BlockInfo[] blocks = unprotectedSetReplication(fsd, iip,
+   replication);
+  isFile = blocks != null;
+  if (isFile) {
 fsd.getEditLog().logSetReplication(iip.getPath(), replication);
   }
 } finally {
   fsd.writeUnlock();
 }
-return status;
+return isFile;
   }
 
   static FileStatus unsetStoragePolicy(FSDirectory fsd, FSPermissionChecker pc,
@@ -379,7 +376,7 @@ public class FSDirAttrOp {
 }
   }
 
-  static SetRepStatus unprotectedSetReplication(
+  static BlockInfo[] unprotectedSetReplication(
   FSDirectory fsd, INodesInPath iip, short replication)
   throws QuotaExceededException, UnresolvedLinkException,
   SnapshotAccessControlException, UnsupportedActionException {
@@ -389,20 +386,12 @@ public class FSDirAttrOp {
 final INode inode = iip.getLastINode();
 if (inode == null || !inode.isFile() || inode.asFile().isStriped()) {
   // TODO we do not support replication on stripe layout files yet
-  // We return invalid here, so we skip writing an edit, but also write an
-  // unsuccessful audit message.
-  return SetRepStatus.INVALID;
+  return null;
 }
 
 INodeFile file = inode.asFile();
 // Make sure the directory has sufficient quotas
 short oldBR = file.getPreferredBlockReplication();
-if (oldBR == replication) {
-  // No need to do anything as the requested rep factor is the same as
-  // existing. Returning UNCHANGED to we can skip writing edits, but still
-  // log a successful audit message.
-  return SetRepStatus.UNCHANGED;
-}
 
 long size = file.computeFileSize(true, true);
 // Ensure the quota does not exceed
@@ -433,7 +422,7 @@ public class FSDirAttrOp {
  oldBR, iip.getPath());
   }
 }
-return SetRepStatus.SUCCESS;
+return file.getBlocks();
   }
 
   static void unprotectedSetStoragePolicy(FSDirectory fsd, BlockManager bm,
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index f1b339ceba8

[hadoop] branch branch-3.3 updated: Revert "HDFS-16531. Avoid setReplication writing an edit record if old replication equals the new value (#4148). Contributed by Stephen O'Donnell."

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 5e137ac33ee Revert "HDFS-16531. Avoid setReplication writing an edit 
record if old replication equals the new value (#4148). Contributed by Stephen 
O'Donnell."
5e137ac33ee is described below

commit 5e137ac33eea37203574c57f7bc11671067aa055
Author: S O'Donnell 
AuthorDate: Wed Apr 20 20:45:17 2022 +0100

Revert "HDFS-16531. Avoid setReplication writing an edit record if old 
replication equals the new value (#4148). Contributed by Stephen O'Donnell."

This reverts commit 8ae033d1a3d65281d3ad1f07f430e74d3a01b1b9.
---
 .../hadoop/hdfs/server/namenode/FSDirAttrOp.java   | 31 +++---
 .../hadoop/hdfs/server/namenode/FSNamesystem.java  | 11 
 .../apache/hadoop/hdfs/TestSetrepIncreasing.java   |  6 -
 3 files changed, 15 insertions(+), 33 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index 9654f599ad9..5c347655fc8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -49,11 +49,6 @@ import java.util.List;
 import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_QUOTA_BY_STORAGETYPE_ENABLED_KEY;
 
 public class FSDirAttrOp {
-
-  protected enum SetRepStatus {
-UNCHANGED, INVALID, SUCCESS
-  }
-
   static FileStatus setPermission(
   FSDirectory fsd, FSPermissionChecker pc, final String src,
   FsPermission permission) throws IOException {
@@ -134,11 +129,11 @@ public class FSDirAttrOp {
 return fsd.getAuditFileInfo(iip);
   }
 
-  static SetRepStatus setReplication(
+  static boolean setReplication(
   FSDirectory fsd, FSPermissionChecker pc, BlockManager bm, String src,
   final short replication) throws IOException {
 bm.verifyReplication(src, replication, null);
-final SetRepStatus status;
+final boolean isFile;
 fsd.writeLock();
 try {
   final INodesInPath iip = fsd.resolvePath(pc, src, DirOp.WRITE);
@@ -146,14 +141,16 @@ public class FSDirAttrOp {
 fsd.checkPathAccess(pc, iip, FsAction.WRITE);
   }
 
-  status = unprotectedSetReplication(fsd, iip, replication);
-  if (status == SetRepStatus.SUCCESS) {
+  final BlockInfo[] blocks = unprotectedSetReplication(fsd, iip,
+   replication);
+  isFile = blocks != null;
+  if (isFile) {
 fsd.getEditLog().logSetReplication(iip.getPath(), replication);
   }
 } finally {
   fsd.writeUnlock();
 }
-return status;
+return isFile;
   }
 
   static FileStatus unsetStoragePolicy(FSDirectory fsd, FSPermissionChecker pc,
@@ -377,7 +374,7 @@ public class FSDirAttrOp {
 return dirNode;
   }
 
-  static SetRepStatus unprotectedSetReplication(
+  static BlockInfo[] unprotectedSetReplication(
   FSDirectory fsd, INodesInPath iip, short replication)
   throws QuotaExceededException, UnresolvedLinkException,
   SnapshotAccessControlException, UnsupportedActionException {
@@ -387,20 +384,12 @@ public class FSDirAttrOp {
 final INode inode = iip.getLastINode();
 if (inode == null || !inode.isFile() || inode.asFile().isStriped()) {
   // TODO we do not support replication on stripe layout files yet
-  // We return invalid here, so we skip writing an edit, but also write an
-  // unsuccessful audit message.
-  return SetRepStatus.INVALID;
+  return null;
 }
 
 INodeFile file = inode.asFile();
 // Make sure the directory has sufficient quotas
 short oldBR = file.getPreferredBlockReplication();
-if (oldBR == replication) {
-  // No need to do anything as the requested rep factor is the same as
-  // existing. Returning UNCHANGED to we can skip writing edits, but still
-  // log a successful audit message.
-  return SetRepStatus.UNCHANGED;
-}
 
 long size = file.computeFileSize(true, true);
 // Ensure the quota does not exceed
@@ -431,7 +420,7 @@ public class FSDirAttrOp {
  oldBR, iip.getPath());
   }
 }
-return SetRepStatus.SUCCESS;
+return file.getBlocks();
   }
 
   static void unprotectedSetStoragePolicy(FSDirectory fsd, BlockManager bm,
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index c74286ece09..3ce8a80be65 100644
--- 
a/hadoop-hdfs

[hadoop] branch trunk updated: Revert "HDFS-16531. Avoid setReplication writing an edit record if old replication equals the new value (#4148). Contributed by Stephen O'Donnell."

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new a4683be65ec Revert "HDFS-16531. Avoid setReplication writing an edit 
record if old replication equals the new value (#4148). Contributed by Stephen 
O'Donnell."
a4683be65ec is described below

commit a4683be65ec315f1561ffa1639aed3c99be61f3f
Author: S O'Donnell 
AuthorDate: Wed Apr 20 20:34:43 2022 +0100

Revert "HDFS-16531. Avoid setReplication writing an edit record if old 
replication equals the new value (#4148). Contributed by Stephen O'Donnell."

This reverts commit db03639f41a022dd07d5fc04e3aa65a94b5f.
---
 .../hadoop/hdfs/server/namenode/FSDirAttrOp.java   | 31 +++---
 .../hadoop/hdfs/server/namenode/FSNamesystem.java  | 11 
 .../apache/hadoop/hdfs/TestSetrepIncreasing.java   |  6 -
 3 files changed, 15 insertions(+), 33 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index a2c9f6bd76b..04913d1a7ce 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -48,11 +48,6 @@ import java.util.List;
 import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_QUOTA_BY_STORAGETYPE_ENABLED_KEY;
 
 public class FSDirAttrOp {
-
-  protected enum SetRepStatus {
-UNCHANGED, INVALID, SUCCESS
-  }
-
   static FileStatus setPermission(
   FSDirectory fsd, FSPermissionChecker pc, final String src,
   FsPermission permission) throws IOException {
@@ -139,11 +134,11 @@ public class FSDirAttrOp {
 return fsd.getAuditFileInfo(iip);
   }
 
-  static SetRepStatus setReplication(
+  static boolean setReplication(
   FSDirectory fsd, FSPermissionChecker pc, BlockManager bm, String src,
   final short replication) throws IOException {
 bm.verifyReplication(src, replication, null);
-final SetRepStatus status;
+final boolean isFile;
 fsd.writeLock();
 try {
   final INodesInPath iip = fsd.resolvePath(pc, src, DirOp.WRITE);
@@ -151,14 +146,16 @@ public class FSDirAttrOp {
 fsd.checkPathAccess(pc, iip, FsAction.WRITE);
   }
 
-  status = unprotectedSetReplication(fsd, iip, replication);
-  if (status == SetRepStatus.SUCCESS) {
+  final BlockInfo[] blocks = unprotectedSetReplication(fsd, iip,
+   replication);
+  isFile = blocks != null;
+  if (isFile) {
 fsd.getEditLog().logSetReplication(iip.getPath(), replication);
   }
 } finally {
   fsd.writeUnlock();
 }
-return status;
+return isFile;
   }
 
   static FileStatus unsetStoragePolicy(FSDirectory fsd, FSPermissionChecker pc,
@@ -384,7 +381,7 @@ public class FSDirAttrOp {
 return dirNode;
   }
 
-  static SetRepStatus unprotectedSetReplication(
+  static BlockInfo[] unprotectedSetReplication(
   FSDirectory fsd, INodesInPath iip, short replication)
   throws QuotaExceededException, UnresolvedLinkException,
   SnapshotAccessControlException, UnsupportedActionException {
@@ -394,20 +391,12 @@ public class FSDirAttrOp {
 final INode inode = iip.getLastINode();
 if (inode == null || !inode.isFile() || inode.asFile().isStriped()) {
   // TODO we do not support replication on stripe layout files yet
-  // We return invalid here, so we skip writing an edit, but also write an
-  // unsuccessful audit message.
-  return SetRepStatus.INVALID;
+  return null;
 }
 
 INodeFile file = inode.asFile();
 // Make sure the directory has sufficient quotas
 short oldBR = file.getPreferredBlockReplication();
-if (oldBR == replication) {
-  // No need to do anything as the requested rep factor is the same as
-  // existing. Returning UNCHANGED to we can skip writing edits, but still
-  // log a successful audit message.
-  return SetRepStatus.UNCHANGED;
-}
 
 long size = file.computeFileSize(true, true);
 // Ensure the quota does not exceed
@@ -438,7 +427,7 @@ public class FSDirAttrOp {
  oldBR, iip.getPath());
   }
 }
-return SetRepStatus.SUCCESS;
+return file.getBlocks();
   }
 
   static void unprotectedSetStoragePolicy(FSDirectory fsd, BlockManager bm,
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 6ab57ed880a..0a00aa65c0b 100644
--- 
a/hadoop-hdfs-project/

[hadoop] branch branch-3.2 updated: HDFS-16531. Avoid setReplication writing an edit record if old replication equals the new value (#4148). Contributed by Stephen O'Donnell.

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new 045d48530cb HDFS-16531. Avoid setReplication writing an edit record if 
old replication equals the new value (#4148). Contributed by Stephen O'Donnell.
045d48530cb is described below

commit 045d48530cbbd7baa2b67937e103b9400487f1a4
Author: Stephen O'Donnell 
AuthorDate: Sun Apr 17 13:05:11 2022 +0100

HDFS-16531. Avoid setReplication writing an edit record if old replication 
equals the new value (#4148). Contributed by Stephen O'Donnell.

(cherry picked from commit db03639f41a022dd07d5fc04e3aa65a94b5f)

 Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
---
 .../hadoop/hdfs/server/namenode/FSDirAttrOp.java   | 31 +++---
 .../hadoop/hdfs/server/namenode/FSNamesystem.java  | 11 
 .../apache/hadoop/hdfs/TestSetrepIncreasing.java   |  6 +
 3 files changed, 33 insertions(+), 15 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index d65b84cc610..99ce83d0bdb 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -50,6 +50,11 @@ import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_QUOTA_BY_STORAGETYPE_ENAB
 import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY;
 
 public class FSDirAttrOp {
+
+  protected enum SetRepStatus {
+UNCHANGED, INVALID, SUCCESS
+  }
+
   static FileStatus setPermission(
   FSDirectory fsd, FSPermissionChecker pc, final String src,
   FsPermission permission) throws IOException {
@@ -130,11 +135,11 @@ public class FSDirAttrOp {
 return fsd.getAuditFileInfo(iip);
   }
 
-  static boolean setReplication(
+  static SetRepStatus setReplication(
   FSDirectory fsd, FSPermissionChecker pc, BlockManager bm, String src,
   final short replication) throws IOException {
 bm.verifyReplication(src, replication, null);
-final boolean isFile;
+final SetRepStatus status;
 fsd.writeLock();
 try {
   final INodesInPath iip = fsd.resolvePath(pc, src, DirOp.WRITE);
@@ -142,16 +147,14 @@ public class FSDirAttrOp {
 fsd.checkPathAccess(pc, iip, FsAction.WRITE);
   }
 
-  final BlockInfo[] blocks = unprotectedSetReplication(fsd, iip,
-   replication);
-  isFile = blocks != null;
-  if (isFile) {
+  status = unprotectedSetReplication(fsd, iip, replication);
+  if (status == SetRepStatus.SUCCESS) {
 fsd.getEditLog().logSetReplication(iip.getPath(), replication);
   }
 } finally {
   fsd.writeUnlock();
 }
-return isFile;
+return status;
   }
 
   static FileStatus unsetStoragePolicy(FSDirectory fsd, FSPermissionChecker pc,
@@ -376,7 +379,7 @@ public class FSDirAttrOp {
 }
   }
 
-  static BlockInfo[] unprotectedSetReplication(
+  static SetRepStatus unprotectedSetReplication(
   FSDirectory fsd, INodesInPath iip, short replication)
   throws QuotaExceededException, UnresolvedLinkException,
   SnapshotAccessControlException, UnsupportedActionException {
@@ -386,12 +389,20 @@ public class FSDirAttrOp {
 final INode inode = iip.getLastINode();
 if (inode == null || !inode.isFile() || inode.asFile().isStriped()) {
   // TODO we do not support replication on stripe layout files yet
-  return null;
+  // We return invalid here, so we skip writing an edit, but also write an
+  // unsuccessful audit message.
+  return SetRepStatus.INVALID;
 }
 
 INodeFile file = inode.asFile();
 // Make sure the directory has sufficient quotas
 short oldBR = file.getPreferredBlockReplication();
+if (oldBR == replication) {
+  // No need to do anything as the requested rep factor is the same as
+  // existing. Returning UNCHANGED to we can skip writing edits, but still
+  // log a successful audit message.
+  return SetRepStatus.UNCHANGED;
+}
 
 long size = file.computeFileSize(true, true);
 // Ensure the quota does not exceed
@@ -422,7 +433,7 @@ public class FSDirAttrOp {
  oldBR, iip.getPath());
   }
 }
-return file.getBlocks();
+return SetRepStatus.SUCCESS;
   }
 
   static void unprotectedSetStoragePolicy(FSDirectory fsd, BlockManager bm,
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop

[hadoop] branch branch-3.3 updated: HDFS-16531. Avoid setReplication writing an edit record if old replication equals the new value (#4148). Contributed by Stephen O'Donnell.

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 8ae033d1a3d HDFS-16531. Avoid setReplication writing an edit record if 
old replication equals the new value (#4148). Contributed by Stephen O'Donnell.
8ae033d1a3d is described below

commit 8ae033d1a3d65281d3ad1f07f430e74d3a01b1b9
Author: Stephen O'Donnell 
AuthorDate: Sun Apr 17 13:05:11 2022 +0100

HDFS-16531. Avoid setReplication writing an edit record if old replication 
equals the new value (#4148). Contributed by Stephen O'Donnell.

(cherry picked from commit db03639f41a022dd07d5fc04e3aa65a94b5f)
---
 .../hadoop/hdfs/server/namenode/FSDirAttrOp.java   | 31 +++---
 .../hadoop/hdfs/server/namenode/FSNamesystem.java  | 11 
 .../apache/hadoop/hdfs/TestSetrepIncreasing.java   |  6 +
 3 files changed, 33 insertions(+), 15 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index 5c347655fc8..9654f599ad9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -49,6 +49,11 @@ import java.util.List;
 import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_QUOTA_BY_STORAGETYPE_ENABLED_KEY;
 
 public class FSDirAttrOp {
+
+  protected enum SetRepStatus {
+UNCHANGED, INVALID, SUCCESS
+  }
+
   static FileStatus setPermission(
   FSDirectory fsd, FSPermissionChecker pc, final String src,
   FsPermission permission) throws IOException {
@@ -129,11 +134,11 @@ public class FSDirAttrOp {
 return fsd.getAuditFileInfo(iip);
   }
 
-  static boolean setReplication(
+  static SetRepStatus setReplication(
   FSDirectory fsd, FSPermissionChecker pc, BlockManager bm, String src,
   final short replication) throws IOException {
 bm.verifyReplication(src, replication, null);
-final boolean isFile;
+final SetRepStatus status;
 fsd.writeLock();
 try {
   final INodesInPath iip = fsd.resolvePath(pc, src, DirOp.WRITE);
@@ -141,16 +146,14 @@ public class FSDirAttrOp {
 fsd.checkPathAccess(pc, iip, FsAction.WRITE);
   }
 
-  final BlockInfo[] blocks = unprotectedSetReplication(fsd, iip,
-   replication);
-  isFile = blocks != null;
-  if (isFile) {
+  status = unprotectedSetReplication(fsd, iip, replication);
+  if (status == SetRepStatus.SUCCESS) {
 fsd.getEditLog().logSetReplication(iip.getPath(), replication);
   }
 } finally {
   fsd.writeUnlock();
 }
-return isFile;
+return status;
   }
 
   static FileStatus unsetStoragePolicy(FSDirectory fsd, FSPermissionChecker pc,
@@ -374,7 +377,7 @@ public class FSDirAttrOp {
 return dirNode;
   }
 
-  static BlockInfo[] unprotectedSetReplication(
+  static SetRepStatus unprotectedSetReplication(
   FSDirectory fsd, INodesInPath iip, short replication)
   throws QuotaExceededException, UnresolvedLinkException,
   SnapshotAccessControlException, UnsupportedActionException {
@@ -384,12 +387,20 @@ public class FSDirAttrOp {
 final INode inode = iip.getLastINode();
 if (inode == null || !inode.isFile() || inode.asFile().isStriped()) {
   // TODO we do not support replication on stripe layout files yet
-  return null;
+  // We return invalid here, so we skip writing an edit, but also write an
+  // unsuccessful audit message.
+  return SetRepStatus.INVALID;
 }
 
 INodeFile file = inode.asFile();
 // Make sure the directory has sufficient quotas
 short oldBR = file.getPreferredBlockReplication();
+if (oldBR == replication) {
+  // No need to do anything as the requested rep factor is the same as
+  // existing. Returning UNCHANGED to we can skip writing edits, but still
+  // log a successful audit message.
+  return SetRepStatus.UNCHANGED;
+}
 
 long size = file.computeFileSize(true, true);
 // Ensure the quota does not exceed
@@ -420,7 +431,7 @@ public class FSDirAttrOp {
  oldBR, iip.getPath());
   }
 }
-return file.getBlocks();
+return SetRepStatus.SUCCESS;
   }
 
   static void unprotectedSetStoragePolicy(FSDirectory fsd, BlockManager bm,
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 3ce8a80be65..c74286ece09 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src

[hadoop] branch branch-3.2 updated: HDFS-16530. setReplication debug log creates a new string even if debug is disabled (#4142)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new 72e48ac84e7 HDFS-16530. setReplication debug log creates a new string 
even if debug is disabled (#4142)
72e48ac84e7 is described below

commit 72e48ac84e7c98b85dd1cfde40541f491e838848
Author: Stephen O'Donnell 
AuthorDate: Wed Apr 6 08:49:52 2022 +0100

HDFS-16530. setReplication debug log creates a new string even if debug is 
disabled (#4142)

(cherry picked from commit bbfe3500cf76ce661e0b10bb5d16cb419f9b9b10)
---
 .../main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index 363f8898f51..d65b84cc610 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -410,7 +410,7 @@ public class FSDirAttrOp {
   bm.setReplication(oldBR, targetReplication, b);
 }
 
-if (oldBR != -1) {
+if (oldBR != -1 && FSDirectory.LOG.isDebugEnabled()) {
   if (oldBR > targetReplication) {
 FSDirectory.LOG.debug("Decreasing replication from {} to {} for {}",
  oldBR, targetReplication, iip.getPath());


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.3 updated: HDFS-16530. setReplication debug log creates a new string even if debug is disabled (#4142)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new bd0dbf319ab HDFS-16530. setReplication debug log creates a new string 
even if debug is disabled (#4142)
bd0dbf319ab is described below

commit bd0dbf319abda642621cd35272b75626f4004605
Author: Stephen O'Donnell 
AuthorDate: Wed Apr 6 08:49:52 2022 +0100

HDFS-16530. setReplication debug log creates a new string even if debug is 
disabled (#4142)

(cherry picked from commit bbfe3500cf76ce661e0b10bb5d16cb419f9b9b10)
---
 .../main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index 173348f3564..5c347655fc8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -408,7 +408,7 @@ public class FSDirAttrOp {
   bm.setReplication(oldBR, targetReplication, b);
 }
 
-if (oldBR != -1) {
+if (oldBR != -1 && FSDirectory.LOG.isDebugEnabled()) {
   if (oldBR > targetReplication) {
 FSDirectory.LOG.debug("Decreasing replication from {} to {} for {}",
  oldBR, targetReplication, iip.getPath());


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: HDFS-16530. setReplication debug log creates a new string even if debug is disabled (#4142)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new bbfe3500cf7 HDFS-16530. setReplication debug log creates a new string 
even if debug is disabled (#4142)
bbfe3500cf7 is described below

commit bbfe3500cf76ce661e0b10bb5d16cb419f9b9b10
Author: Stephen O'Donnell 
AuthorDate: Wed Apr 6 08:49:52 2022 +0100

HDFS-16530. setReplication debug log creates a new string even if debug is 
disabled (#4142)
---
 .../main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index db1baab66b3..04913d1a7ce 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -415,7 +415,7 @@ public class FSDirAttrOp {
   bm.setReplication(oldBR, targetReplication, b);
 }
 
-if (oldBR != -1) {
+if (oldBR != -1 && FSDirectory.LOG.isDebugEnabled()) {
   if (oldBR > targetReplication) {
 FSDirectory.LOG.debug("Decreasing replication from {} to {} for {}",
  oldBR, targetReplication, iip.getPath());


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated (66b7240 -> 49e4e4d)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git.


from 66b7240  HADOOP-18131. Upgrade maven enforcer plugin and relevant 
dependencies (#4000)
 add 49e4e4d  HDFS-16496. Snapshot diff on snapshotable directory fails 
with not snapshottable error (#4051)

No new revisions were added by this update.

Summary of changes:
 .../hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java   | 6 +++---
 .../hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java   | 3 ++-
 2 files changed, 5 insertions(+), 4 deletions(-)

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.3 updated: HDFS-16262. Async refresh of cached locations in DFSInputStream (#3527)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new bd13d73  HDFS-16262. Async refresh of cached locations in 
DFSInputStream (#3527)
bd13d73 is described below

commit bd13d73334c8468c5345f80487e8a7f671707428
Author: Bryan Beaudreault 
AuthorDate: Tue Jan 25 06:42:35 2022 -0500

HDFS-16262. Async refresh of cached locations in DFSInputStream (#3527)

(cherry picked from commit 94b884ae553e9d67f57e3ae7d2b22468c6f342fa)
---
 .../java/org/apache/hadoop/hdfs/ClientContext.java |  42 +++
 .../java/org/apache/hadoop/hdfs/DFSClient.java |  34 ++-
 .../org/apache/hadoop/hdfs/DFSInputStream.java | 324 ++---
 .../apache/hadoop/hdfs/DFSStripedInputStream.java  |  10 +-
 .../apache/hadoop/hdfs/LocatedBlocksRefresher.java | 210 +
 .../hadoop/hdfs/client/HdfsClientConfigKeys.java   |  13 +
 .../hadoop/hdfs/client/impl/DfsClientConf.java |  18 +-
 .../src/main/resources/hdfs-default.xml|  19 ++
 .../hdfs/TestDFSInputStreamBlockLocations.java | 244 +++-
 .../hadoop/hdfs/TestLocatedBlocksRefresher.java| 266 +
 10 files changed, 915 insertions(+), 265 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
index 5bb7e03..f83346e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
@@ -70,6 +70,11 @@ public class ClientContext {
   private final String name;
 
   /**
+   * The client conf used to initialize context.
+   */
+  private final DfsClientConf dfsClientConf;
+
+  /**
* String representation of the configuration.
*/
   private final String confString;
@@ -131,6 +136,17 @@ public class ClientContext {
   private volatile DeadNodeDetector deadNodeDetector = null;
 
   /**
+   * The switch for the {@link LocatedBlocksRefresher}.
+   */
+  private final boolean locatedBlocksRefresherEnabled;
+
+  /**
+   * Periodically refresh the {@link 
org.apache.hadoop.hdfs.protocol.LocatedBlocks} backing
+   * registered {@link DFSInputStream}s, to take advantage of changes in block 
placement.
+   */
+  private volatile LocatedBlocksRefresher locatedBlocksRefresher = null;
+
+  /**
* Count the reference of ClientContext.
*/
   private int counter = 0;
@@ -146,6 +162,7 @@ public class ClientContext {
 final ShortCircuitConf scConf = conf.getShortCircuitConf();
 
 this.name = name;
+this.dfsClientConf = conf;
 this.confString = scConf.confAsString();
 this.clientShortCircuitNum = conf.getClientShortCircuitNum();
 this.shortCircuitCache = new ShortCircuitCache[this.clientShortCircuitNum];
@@ -164,6 +181,7 @@ public class ClientContext {
 this.byteArrayManager = ByteArrayManager.newInstance(
 conf.getWriteByteArrayManagerConf());
 this.deadNodeDetectionEnabled = conf.isDeadNodeDetectionEnabled();
+this.locatedBlocksRefresherEnabled = 
conf.isLocatedBlocksRefresherEnabled();
 initTopologyResolution(config);
   }
 
@@ -302,6 +320,21 @@ public class ClientContext {
   }
 
   /**
+   * If true, LocatedBlocksRefresher will be periodically refreshing 
LocatedBlocks
+   * of registered DFSInputStreams.
+   */
+  public boolean isLocatedBlocksRefresherEnabled() {
+return locatedBlocksRefresherEnabled;
+  }
+
+  /**
+   * Obtain LocatedBlocksRefresher of the current client.
+   */
+  public LocatedBlocksRefresher getLocatedBlocksRefresher() {
+return locatedBlocksRefresher;
+  }
+
+  /**
* Increment the counter. Start the dead node detector thread if there is no
* reference.
*/
@@ -311,6 +344,10 @@ public class ClientContext {
   deadNodeDetector = new DeadNodeDetector(name, configuration);
   deadNodeDetector.start();
 }
+if (locatedBlocksRefresherEnabled && locatedBlocksRefresher == null) {
+  locatedBlocksRefresher = new LocatedBlocksRefresher(name, configuration, 
dfsClientConf);
+  locatedBlocksRefresher.start();
+}
   }
 
   /**
@@ -324,5 +361,10 @@ public class ClientContext {
   deadNodeDetector.shutdown();
   deadNodeDetector = null;
 }
+
+if (counter == 0 && locatedBlocksRefresherEnabled && 
locatedBlocksRefresher != null) {
+  locatedBlocksRefresher.shutdown();
+  locatedBlocksRefresher = null;
+}
   }
 }
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 7500273..ea9736d 100755
--- 
a/hadoop-

[hadoop] branch trunk updated (43153e8 -> 94b884a)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git.


from 43153e8  HDFS-16428. Source path with storagePolicy cause wrong 
typeConsumed while rename (#3898). Contributed by lei w.
 add 94b884a  HDFS-16262. Async refresh of cached locations in 
DFSInputStream (#3527)

No new revisions were added by this update.

Summary of changes:
 .../java/org/apache/hadoop/hdfs/ClientContext.java |  42 +++
 .../java/org/apache/hadoop/hdfs/DFSClient.java |  34 ++-
 .../org/apache/hadoop/hdfs/DFSInputStream.java | 324 ++---
 .../apache/hadoop/hdfs/DFSStripedInputStream.java  |  10 +-
 .../apache/hadoop/hdfs/LocatedBlocksRefresher.java | 210 +
 .../hadoop/hdfs/client/HdfsClientConfigKeys.java   |  13 +
 .../hadoop/hdfs/client/impl/DfsClientConf.java |  18 +-
 .../src/main/resources/hdfs-default.xml|  19 ++
 .../hdfs/TestDFSInputStreamBlockLocations.java | 244 +++-
 .../hadoop/hdfs/TestLocatedBlocksRefresher.java| 266 +
 10 files changed, 915 insertions(+), 265 deletions(-)
 create mode 100644 
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/LocatedBlocksRefresher.java
 create mode 100644 
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocatedBlocksRefresher.java

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.2 updated: HDFS-16408. Ensure LeaseRecheckIntervalMs is greater than zero (#3856)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new 1e1e4a2  HDFS-16408. Ensure LeaseRecheckIntervalMs is greater than 
zero (#3856)
1e1e4a2 is described below

commit 1e1e4a2f51740cc97dc9622fda7a16fb86002edf
Author: liever18 <47295183+lieve...@users.noreply.github.com>
AuthorDate: Wed Jan 5 23:42:58 2022 +0800

HDFS-16408. Ensure LeaseRecheckIntervalMs is greater than zero (#3856)

(cherry picked from commit e1d0aa9ee5851d3cdc3b125daebe5cc9305ad6f0)
---
 .../java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 4 
 1 file changed, 4 insertions(+)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 1644d6f..3b54509 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -907,6 +907,10 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
   this.leaseRecheckIntervalMs = conf.getLong(
   DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_KEY,
   DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_DEFAULT);
+  Preconditions.checkArgument(
+  leaseRecheckIntervalMs > 0,
+  DFSConfigKeys.DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_KEY +
+  " must be greater than zero");
   this.maxLockHoldToReleaseLeaseMs = conf.getLong(
   DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_KEY,
   DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_DEFAULT);

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.3 updated: HDFS-16408. Ensure LeaseRecheckIntervalMs is greater than zero (#3856)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 3a82899  HDFS-16408. Ensure LeaseRecheckIntervalMs is greater than 
zero (#3856)
3a82899 is described below

commit 3a82899493d5dd7d021c3d8a4ca93e773cf36e73
Author: liever18 <47295183+lieve...@users.noreply.github.com>
AuthorDate: Wed Jan 5 23:42:58 2022 +0800

HDFS-16408. Ensure LeaseRecheckIntervalMs is greater than zero (#3856)

(cherry picked from commit e1d0aa9ee5851d3cdc3b125daebe5cc9305ad6f0)
---
 .../java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 4 
 1 file changed, 4 insertions(+)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index fbe32c0..8daa474 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -959,6 +959,10 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
   this.leaseRecheckIntervalMs = conf.getLong(
   DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_KEY,
   DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_DEFAULT);
+  Preconditions.checkArgument(
+  leaseRecheckIntervalMs > 0,
+  DFSConfigKeys.DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_KEY +
+  " must be greater than zero");
   this.maxLockHoldToReleaseLeaseMs = conf.getLong(
   DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_KEY,
   DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_DEFAULT);

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: HDFS-16408. Ensure LeaseRecheckIntervalMs is greater than zero (#3856)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new e1d0aa9  HDFS-16408. Ensure LeaseRecheckIntervalMs is greater than 
zero (#3856)
e1d0aa9 is described below

commit e1d0aa9ee5851d3cdc3b125daebe5cc9305ad6f0
Author: liever18 <47295183+lieve...@users.noreply.github.com>
AuthorDate: Wed Jan 5 23:42:58 2022 +0800

HDFS-16408. Ensure LeaseRecheckIntervalMs is greater than zero (#3856)
---
 .../java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 4 
 1 file changed, 4 insertions(+)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index c07115e..610303a 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1014,6 +1014,10 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
   this.leaseRecheckIntervalMs = conf.getLong(
   DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_KEY,
   DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_DEFAULT);
+  Preconditions.checkArgument(
+  leaseRecheckIntervalMs > 0,
+  DFSConfigKeys.DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_KEY +
+  " must be greater than zero");
   this.maxLockHoldToReleaseLeaseMs = conf.getLong(
   DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_KEY,
   DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_DEFAULT);

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.2 updated: HDFS-16391. Avoid evaluation of LOG.debug statement in NameNodeHeartbeatService (#3820)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new c66bae3  HDFS-16391. Avoid evaluation of LOG.debug statement in 
NameNodeHeartbeatService (#3820)
c66bae3 is described below

commit c66bae3fc3bdcd550dd0ee5370c8a5a8f8bd8ac6
Author: wangzhaohui <32935220+wzhallri...@users.noreply.github.com>
AuthorDate: Tue Dec 21 19:28:57 2021 +0800

HDFS-16391. Avoid evaluation of LOG.debug statement in 
NameNodeHeartbeatService (#3820)

(cherry picked from commit 8d251bd629c2db33f03c80e93c7e54fceedbfbe6)
---
 .../server/federation/router/NamenodeHeartbeatService.java   | 12 
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
index a1adf77..455dfcd 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
@@ -208,12 +208,16 @@ public class NamenodeHeartbeatService extends 
PeriodicService {
   LOG.error("Namenode is not operational: {}", getNamenodeDesc());
 } else if (report.haStateValid()) {
   // block and HA status available
-  LOG.debug("Received service state: {} from HA namenode: {}",
-  report.getState(), getNamenodeDesc());
+  if (LOG.isDebugEnabled()) {
+LOG.debug("Received service state: {} from HA namenode: {}",
+report.getState(), getNamenodeDesc());
+  }
 } else if (localTarget == null) {
   // block info available, HA status not expected
-  LOG.debug(
-  "Reporting non-HA namenode as operational: " + getNamenodeDesc());
+  if (LOG.isDebugEnabled()) {
+LOG.debug(
+"Reporting non-HA namenode as operational: {}", getNamenodeDesc());
+  }
 } else {
   // block info available, HA status should be available, but was not
   // fetched do nothing and let the current state stand

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.3 updated: HDFS-16391. Avoid evaluation of LOG.debug statement in NameNodeHeartbeatService (#3820)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new cc7b7e1  HDFS-16391. Avoid evaluation of LOG.debug statement in 
NameNodeHeartbeatService (#3820)
cc7b7e1 is described below

commit cc7b7e174c3b3ebbfe93cccef67a966785e204f1
Author: wangzhaohui <32935220+wzhallri...@users.noreply.github.com>
AuthorDate: Tue Dec 21 19:28:57 2021 +0800

HDFS-16391. Avoid evaluation of LOG.debug statement in 
NameNodeHeartbeatService (#3820)

(cherry picked from commit 8d251bd629c2db33f03c80e93c7e54fceedbfbe6)
---
 .../server/federation/router/NamenodeHeartbeatService.java   | 12 
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
index ce6db5c..ffae90e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
@@ -221,12 +221,16 @@ public class NamenodeHeartbeatService extends 
PeriodicService {
   LOG.error("Namenode is not operational: {}", getNamenodeDesc());
 } else if (report.haStateValid()) {
   // block and HA status available
-  LOG.debug("Received service state: {} from HA namenode: {}",
-  report.getState(), getNamenodeDesc());
+  if (LOG.isDebugEnabled()) {
+LOG.debug("Received service state: {} from HA namenode: {}",
+report.getState(), getNamenodeDesc());
+  }
 } else if (localTarget == null) {
   // block info available, HA status not expected
-  LOG.debug(
-  "Reporting non-HA namenode as operational: " + getNamenodeDesc());
+  if (LOG.isDebugEnabled()) {
+LOG.debug(
+"Reporting non-HA namenode as operational: {}", getNamenodeDesc());
+  }
 } else {
   // block info available, HA status should be available, but was not
   // fetched do nothing and let the current state stand

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.2.3 updated: HDFS-16386. Reduce DataNode load when FsDatasetAsyncDiskService is working. (#3806)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2.3 by this push:
 new d93760c  HDFS-16386. Reduce DataNode load when 
FsDatasetAsyncDiskService is working. (#3806)
d93760c is described below

commit d93760c8c238619f28ae1abbe210a0a316d1822d
Author: jianghuazhu <740087...@qq.com>
AuthorDate: Mon Dec 20 19:28:55 2021 +0800

HDFS-16386. Reduce DataNode load when FsDatasetAsyncDiskService is working. 
(#3806)

(cherry picked from commit 746b32855496c054854c3e02d9993d97f10073d2)
(cherry picked from commit b68084ab43d5a033ac33fd1017c502589a46a73c)
---
 .../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java  |  3 +++
 .../datanode/fsdataset/impl/FsDatasetAsyncDiskService.java   | 12 ++--
 .../hadoop-hdfs/src/main/resources/hdfs-default.xml  | 10 ++
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 1813787..2ed1e35 100755
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -131,6 +131,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final longDFS_DATANODE_MAX_LOCKED_MEMORY_DEFAULT = 0;
   public static final String  
DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_KEY = 
"dfs.datanode.fsdatasetcache.max.threads.per.volume";
   public static final int 
DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_DEFAULT = 4;
+  public static final String  
DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY =
+  "dfs.datanode.fsdatasetasyncdisk.max.threads.per.volume";
+  public static final int 
DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_DEFAULT = 4;
   public static final String  DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC = 
"dfs.datanode.lazywriter.interval.sec";
   public static final int DFS_DATANODE_LAZY_WRITER_INTERVAL_DEFAULT_SEC = 
60;
   public static final String  DFS_DATANODE_RAM_DISK_REPLICA_TRACKER_KEY = 
"dfs.datanode.ram.disk.replica.tracker";
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
index 81213a0..4503192 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
@@ -30,6 +30,8 @@ import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -65,7 +67,7 @@ class FsDatasetAsyncDiskService {
   // ThreadPool core pool size
   private static final int CORE_THREADS_PER_VOLUME = 1;
   // ThreadPool maximum pool size
-  private static final int MAXIMUM_THREADS_PER_VOLUME = 4;
+  private final int maxNumThreadsPerVolume;
   // ThreadPool keep-alive time for threads over core pool size
   private static final long THREADS_KEEP_ALIVE_SECONDS = 60; 
   
@@ -90,6 +92,12 @@ class FsDatasetAsyncDiskService {
 this.datanode = datanode;
 this.fsdatasetImpl = fsdatasetImpl;
 this.threadGroup = new ThreadGroup(getClass().getSimpleName());
+maxNumThreadsPerVolume = datanode.getConf().getInt(
+  DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY,
+  
DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_DEFAULT);
+Preconditions.checkArgument(maxNumThreadsPerVolume > 0,
+
DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY +
+  " must be a positive integer.");
   }
 
   private void addExecutorForVolume(final FsVolumeImpl volume) {
@@ -110,7 +118,7 @@ class FsDatasetAsyncDiskService {
 };
 
 ThreadPoolExecutor executor = new ThreadPoolExecutor(
-CORE_THREADS_PER_VOLUME, MAXIMUM_THREADS_PER_VOLUME,
+CORE_THREADS_PER_VOLUME, maxNumThreadsPerVolume,
 THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS,
 new LinkedBlockingQueue(), threadFactory);
 
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml 
b/hadoop-hdfs-project/hadoop-hdfs

[hadoop] branch trunk updated (9420cb6 -> 8d251bd)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git.


from 9420cb6  HDFS-16385. Fix Datanode retrieve slownode information bug. 
(#3803)
 add 8d251bd  HDFS-16391. Avoid evaluation of LOG.debug statement in 
NameNodeHeartbeatService (#3820)

No new revisions were added by this update.

Summary of changes:
 .../server/federation/router/NamenodeHeartbeatService.java   | 12 
 1 file changed, 8 insertions(+), 4 deletions(-)

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.2 updated: HDFS-16386. Reduce DataNode load when FsDatasetAsyncDiskService is working. (#3806)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new b68084a  HDFS-16386. Reduce DataNode load when 
FsDatasetAsyncDiskService is working. (#3806)
b68084a is described below

commit b68084ab43d5a033ac33fd1017c502589a46a73c
Author: jianghuazhu <740087...@qq.com>
AuthorDate: Mon Dec 20 19:28:55 2021 +0800

HDFS-16386. Reduce DataNode load when FsDatasetAsyncDiskService is working. 
(#3806)

(cherry picked from commit 746b32855496c054854c3e02d9993d97f10073d2)
---
 .../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java  |  3 +++
 .../datanode/fsdataset/impl/FsDatasetAsyncDiskService.java   | 12 ++--
 .../hadoop-hdfs/src/main/resources/hdfs-default.xml  | 10 ++
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 1813787..2ed1e35 100755
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -131,6 +131,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final longDFS_DATANODE_MAX_LOCKED_MEMORY_DEFAULT = 0;
   public static final String  
DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_KEY = 
"dfs.datanode.fsdatasetcache.max.threads.per.volume";
   public static final int 
DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_DEFAULT = 4;
+  public static final String  
DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY =
+  "dfs.datanode.fsdatasetasyncdisk.max.threads.per.volume";
+  public static final int 
DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_DEFAULT = 4;
   public static final String  DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC = 
"dfs.datanode.lazywriter.interval.sec";
   public static final int DFS_DATANODE_LAZY_WRITER_INTERVAL_DEFAULT_SEC = 
60;
   public static final String  DFS_DATANODE_RAM_DISK_REPLICA_TRACKER_KEY = 
"dfs.datanode.ram.disk.replica.tracker";
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
index 706c078..aa577a4 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
@@ -30,6 +30,8 @@ import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -65,7 +67,7 @@ class FsDatasetAsyncDiskService {
   // ThreadPool core pool size
   private static final int CORE_THREADS_PER_VOLUME = 1;
   // ThreadPool maximum pool size
-  private static final int MAXIMUM_THREADS_PER_VOLUME = 4;
+  private final int maxNumThreadsPerVolume;
   // ThreadPool keep-alive time for threads over core pool size
   private static final long THREADS_KEEP_ALIVE_SECONDS = 60; 
   
@@ -90,6 +92,12 @@ class FsDatasetAsyncDiskService {
 this.datanode = datanode;
 this.fsdatasetImpl = fsdatasetImpl;
 this.threadGroup = new ThreadGroup(getClass().getSimpleName());
+maxNumThreadsPerVolume = datanode.getConf().getInt(
+  DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY,
+  
DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_DEFAULT);
+Preconditions.checkArgument(maxNumThreadsPerVolume > 0,
+
DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY +
+  " must be a positive integer.");
   }
 
   private void addExecutorForVolume(final FsVolumeImpl volume) {
@@ -110,7 +118,7 @@ class FsDatasetAsyncDiskService {
 };
 
 ThreadPoolExecutor executor = new ThreadPoolExecutor(
-CORE_THREADS_PER_VOLUME, MAXIMUM_THREADS_PER_VOLUME,
+CORE_THREADS_PER_VOLUME, maxNumThreadsPerVolume,
 THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS,
 new LinkedBlockingQueue(), threadFactory);
 
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 5dd6e83..0d31b28 100644
--- a/hadoop-h

[hadoop] branch branch-3.3 updated: HDFS-16386. Reduce DataNode load when FsDatasetAsyncDiskService is working. (#3806)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new a4496a0  HDFS-16386. Reduce DataNode load when 
FsDatasetAsyncDiskService is working. (#3806)
a4496a0 is described below

commit a4496a0187f448592ac96ca7e4037bcec091e4d7
Author: jianghuazhu <740087...@qq.com>
AuthorDate: Mon Dec 20 19:28:55 2021 +0800

HDFS-16386. Reduce DataNode load when FsDatasetAsyncDiskService is working. 
(#3806)

(cherry picked from commit 746b32855496c054854c3e02d9993d97f10073d2)
---
 .../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java  |  3 +++
 .../datanode/fsdataset/impl/FsDatasetAsyncDiskService.java   | 12 ++--
 .../hadoop-hdfs/src/main/resources/hdfs-default.xml  | 10 ++
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 9eb1fdf..7196def 100755
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -143,6 +143,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final longDFS_DATANODE_MAX_LOCKED_MEMORY_DEFAULT = 0;
   public static final String  
DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_KEY = 
"dfs.datanode.fsdatasetcache.max.threads.per.volume";
   public static final int 
DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_DEFAULT = 4;
+  public static final String  
DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY =
+  "dfs.datanode.fsdatasetasyncdisk.max.threads.per.volume";
+  public static final int 
DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_DEFAULT = 4;
   public static final String  DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC = 
"dfs.datanode.lazywriter.interval.sec";
   public static final int DFS_DATANODE_LAZY_WRITER_INTERVAL_DEFAULT_SEC = 
60;
   public static final String  DFS_DATANODE_RAM_DISK_REPLICA_TRACKER_KEY = 
"dfs.datanode.ram.disk.replica.tracker";
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
index 706c078..aa577a4 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
@@ -30,6 +30,8 @@ import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -65,7 +67,7 @@ class FsDatasetAsyncDiskService {
   // ThreadPool core pool size
   private static final int CORE_THREADS_PER_VOLUME = 1;
   // ThreadPool maximum pool size
-  private static final int MAXIMUM_THREADS_PER_VOLUME = 4;
+  private final int maxNumThreadsPerVolume;
   // ThreadPool keep-alive time for threads over core pool size
   private static final long THREADS_KEEP_ALIVE_SECONDS = 60; 
   
@@ -90,6 +92,12 @@ class FsDatasetAsyncDiskService {
 this.datanode = datanode;
 this.fsdatasetImpl = fsdatasetImpl;
 this.threadGroup = new ThreadGroup(getClass().getSimpleName());
+maxNumThreadsPerVolume = datanode.getConf().getInt(
+  DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY,
+  
DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_DEFAULT);
+Preconditions.checkArgument(maxNumThreadsPerVolume > 0,
+
DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY +
+  " must be a positive integer.");
   }
 
   private void addExecutorForVolume(final FsVolumeImpl volume) {
@@ -110,7 +118,7 @@ class FsDatasetAsyncDiskService {
 };
 
 ThreadPoolExecutor executor = new ThreadPoolExecutor(
-CORE_THREADS_PER_VOLUME, MAXIMUM_THREADS_PER_VOLUME,
+CORE_THREADS_PER_VOLUME, maxNumThreadsPerVolume,
 THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS,
 new LinkedBlockingQueue(), threadFactory);
 
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 582e010..80c4818 100755
--- a/hadoop-h

[hadoop] branch trunk updated (32a78e0 -> 746b328)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git.


from 32a78e0  HDFS-16168. Fix TestHDFSFileSystemContract.testAppend timeout 
(#3815)
 add 746b328  HDFS-16386. Reduce DataNode load when 
FsDatasetAsyncDiskService is working. (#3806)

No new revisions were added by this update.

Summary of changes:
 .../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java  |  3 +++
 .../datanode/fsdataset/impl/FsDatasetAsyncDiskService.java   | 12 ++--
 .../hadoop-hdfs/src/main/resources/hdfs-default.xml  | 10 ++
 3 files changed, 23 insertions(+), 2 deletions(-)

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.2.3 updated: HADOOP-18023. Allow cp command to run with multi threads. (#3721)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2.3 by this push:
 new 4e9b9ad  HADOOP-18023. Allow cp command to run with multi threads. 
(#3721)
4e9b9ad is described below

commit 4e9b9adb964935b4737ee877ae737dd2c46d15ed
Author: smarthan <1139557...@qq.com>
AuthorDate: Mon Nov 29 20:45:08 2021 +0800

HADOOP-18023. Allow cp command to run with multi threads. (#3721)

(cherry picked from commit 932a78fe38b34a923f6852a1a19482075806ecba)
(cherry picked from commit 2f3e18697802cce81eb3c976088583ef365daea0)
---
 .../org/apache/hadoop/fs/shell/CopyCommands.java   |  39 ++--
 .../src/site/markdown/FileSystemShell.md   |  11 +-
 .../org/apache/hadoop/fs/shell/TestCpCommand.java  | 210 +
 .../hadoop-common/src/test/resources/testConf.xml  |  47 +++--
 4 files changed, 275 insertions(+), 32 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
index 7372992..7b2e835 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
@@ -146,33 +146,40 @@ class CopyCommands {
 }
   }
 
-  static class Cp extends CommandWithDestination {
+  static class Cp extends CopyCommandWithMultiThread {
 public static final String NAME = "cp";
 public static final String USAGE =
-"[-f] [-p | -p[topax]] [-d]  ... ";
+"[-f] [-p | -p[topax]] [-d] [-t ]"
++ " [-q ]  ... ";
 public static final String DESCRIPTION =
-  "Copy files that match the file pattern  to a " +
-  "destination.  When copying multiple files, the destination " +
-  "must be a directory. Passing -p preserves status " +
-  "[topax] (timestamps, ownership, permission, ACLs, XAttr). " +
-  "If -p is specified with no , then preserves " +
-  "timestamps, ownership, permission. If -pa is specified, " +
-  "then preserves permission also because ACL is a super-set of " +
-  "permission. Passing -f overwrites the destination if it " +
-  "already exists. raw namespace extended attributes are preserved " +
-  "if (1) they are supported (HDFS only) and, (2) all of the source and " +
-  "target pathnames are in the /.reserved/raw hierarchy. raw namespace " +
-  "xattr preservation is determined solely by the presence (or absence) " +
-"of the /.reserved/raw prefix and not by the -p option. Passing -d "+
-"will skip creation of temporary file(._COPYING_).\n";
+"Copy files that match the file pattern  to a destination."
++ " When copying multiple files, the destination must be a "
++ "directory.\nFlags :\n"
++ "  -p[topax] : Preserve file attributes [topx] (timestamps, "
++ "ownership, permission, ACL, XAttr). If -p is specified with "
++ "no arg, then preserves timestamps, ownership, permission. "
++ "If -pa is specified, then preserves permission also because "
++ "ACL is a super-set of permission. Determination of whether raw "
++ "namespace extended attributes are preserved is independent of "
++ "the -p flag.\n"
++ "  -f : Overwrite the destination if it already exists.\n"
++ "  -d : Skip creation of temporary file(._COPYING_).\n"
++ "  -t  : Number of threads to be used, "
++ "default is 1.\n"
++ "  -q  : Thread pool queue size to be "
++ "used, default is 1024.\n";
 
 @Override
 protected void processOptions(LinkedList args) throws IOException {
   popPreserveOption(args);
   CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "f", "d");
+  cf.addOptionWithValue("t");
+  cf.addOptionWithValue("q");
   cf.parse(args);
   setDirectWrite(cf.getOpt("d"));
   setOverwrite(cf.getOpt("f"));
+  setThreadCount(cf.getOptValue("t"));
+  setThreadPoolQueueSize(cf.getOptValue("q"));
   // should have a -r option
   setRecursive(true);
   getRemoteDestination(args);
diff --git 
a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md 
b/hadoop-common-p

[hadoop] branch branch-3.2 updated: HADOOP-18023. Allow cp command to run with multi threads. (#3721)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new 2f3e186  HADOOP-18023. Allow cp command to run with multi threads. 
(#3721)
2f3e186 is described below

commit 2f3e18697802cce81eb3c976088583ef365daea0
Author: smarthan <1139557...@qq.com>
AuthorDate: Mon Nov 29 20:45:08 2021 +0800

HADOOP-18023. Allow cp command to run with multi threads. (#3721)

(cherry picked from commit 932a78fe38b34a923f6852a1a19482075806ecba)
---
 .../org/apache/hadoop/fs/shell/CopyCommands.java   |  39 ++--
 .../src/site/markdown/FileSystemShell.md   |  11 +-
 .../org/apache/hadoop/fs/shell/TestCpCommand.java  | 210 +
 .../hadoop-common/src/test/resources/testConf.xml  |  47 +++--
 4 files changed, 275 insertions(+), 32 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
index 7372992..7b2e835 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
@@ -146,33 +146,40 @@ class CopyCommands {
 }
   }
 
-  static class Cp extends CommandWithDestination {
+  static class Cp extends CopyCommandWithMultiThread {
 public static final String NAME = "cp";
 public static final String USAGE =
-"[-f] [-p | -p[topax]] [-d]  ... ";
+"[-f] [-p | -p[topax]] [-d] [-t ]"
++ " [-q ]  ... ";
 public static final String DESCRIPTION =
-  "Copy files that match the file pattern  to a " +
-  "destination.  When copying multiple files, the destination " +
-  "must be a directory. Passing -p preserves status " +
-  "[topax] (timestamps, ownership, permission, ACLs, XAttr). " +
-  "If -p is specified with no , then preserves " +
-  "timestamps, ownership, permission. If -pa is specified, " +
-  "then preserves permission also because ACL is a super-set of " +
-  "permission. Passing -f overwrites the destination if it " +
-  "already exists. raw namespace extended attributes are preserved " +
-  "if (1) they are supported (HDFS only) and, (2) all of the source and " +
-  "target pathnames are in the /.reserved/raw hierarchy. raw namespace " +
-  "xattr preservation is determined solely by the presence (or absence) " +
-"of the /.reserved/raw prefix and not by the -p option. Passing -d "+
-"will skip creation of temporary file(._COPYING_).\n";
+"Copy files that match the file pattern  to a destination."
++ " When copying multiple files, the destination must be a "
++ "directory.\nFlags :\n"
++ "  -p[topax] : Preserve file attributes [topx] (timestamps, "
++ "ownership, permission, ACL, XAttr). If -p is specified with "
++ "no arg, then preserves timestamps, ownership, permission. "
++ "If -pa is specified, then preserves permission also because "
++ "ACL is a super-set of permission. Determination of whether raw "
++ "namespace extended attributes are preserved is independent of "
++ "the -p flag.\n"
++ "  -f : Overwrite the destination if it already exists.\n"
++ "  -d : Skip creation of temporary file(._COPYING_).\n"
++ "  -t  : Number of threads to be used, "
++ "default is 1.\n"
++ "  -q  : Thread pool queue size to be "
++ "used, default is 1024.\n";
 
 @Override
 protected void processOptions(LinkedList args) throws IOException {
   popPreserveOption(args);
   CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "f", "d");
+  cf.addOptionWithValue("t");
+  cf.addOptionWithValue("q");
   cf.parse(args);
   setDirectWrite(cf.getOpt("d"));
   setOverwrite(cf.getOpt("f"));
+  setThreadCount(cf.getOptValue("t"));
+  setThreadPoolQueueSize(cf.getOptValue("q"));
   // should have a -r option
   setRecursive(true);
   getRemoteDestination(args);
diff --git 
a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index bb4ce45..946516

[hadoop] branch branch-3.3.2 updated: HADOOP-18023. Allow cp command to run with multi threads. (#3721)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3.2 by this push:
 new 3e24bc6  HADOOP-18023. Allow cp command to run with multi threads. 
(#3721)
3e24bc6 is described below

commit 3e24bc669a8efecac6729e253ed138b5600acb19
Author: smarthan <1139557...@qq.com>
AuthorDate: Mon Nov 29 20:45:08 2021 +0800

HADOOP-18023. Allow cp command to run with multi threads. (#3721)

(cherry picked from commit 932a78fe38b34a923f6852a1a19482075806ecba)
---
 .../org/apache/hadoop/fs/shell/CopyCommands.java   |  39 ++--
 .../src/site/markdown/FileSystemShell.md   |  11 +-
 .../org/apache/hadoop/fs/shell/TestCpCommand.java  | 210 +
 .../hadoop-common/src/test/resources/testConf.xml  |  47 +++--
 4 files changed, 275 insertions(+), 32 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
index 5a0652b..b03d7de 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
@@ -147,33 +147,40 @@ class CopyCommands {
 }
   }
 
-  static class Cp extends CommandWithDestination {
+  static class Cp extends CopyCommandWithMultiThread {
 public static final String NAME = "cp";
 public static final String USAGE =
-"[-f] [-p | -p[topax]] [-d]  ... ";
+"[-f] [-p | -p[topax]] [-d] [-t ]"
++ " [-q ]  ... ";
 public static final String DESCRIPTION =
-  "Copy files that match the file pattern  to a " +
-  "destination.  When copying multiple files, the destination " +
-  "must be a directory. Passing -p preserves status " +
-  "[topax] (timestamps, ownership, permission, ACLs, XAttr). " +
-  "If -p is specified with no , then preserves " +
-  "timestamps, ownership, permission. If -pa is specified, " +
-  "then preserves permission also because ACL is a super-set of " +
-  "permission. Passing -f overwrites the destination if it " +
-  "already exists. raw namespace extended attributes are preserved " +
-  "if (1) they are supported (HDFS only) and, (2) all of the source and " +
-  "target pathnames are in the /.reserved/raw hierarchy. raw namespace " +
-  "xattr preservation is determined solely by the presence (or absence) " +
-"of the /.reserved/raw prefix and not by the -p option. Passing -d "+
-"will skip creation of temporary file(._COPYING_).\n";
+"Copy files that match the file pattern  to a destination."
++ " When copying multiple files, the destination must be a "
++ "directory.\nFlags :\n"
++ "  -p[topax] : Preserve file attributes [topx] (timestamps, "
++ "ownership, permission, ACL, XAttr). If -p is specified with "
++ "no arg, then preserves timestamps, ownership, permission. "
++ "If -pa is specified, then preserves permission also because "
++ "ACL is a super-set of permission. Determination of whether raw "
++ "namespace extended attributes are preserved is independent of "
++ "the -p flag.\n"
++ "  -f : Overwrite the destination if it already exists.\n"
++ "  -d : Skip creation of temporary file(._COPYING_).\n"
++ "  -t  : Number of threads to be used, "
++ "default is 1.\n"
++ "  -q  : Thread pool queue size to be "
++ "used, default is 1024.\n";
 
 @Override
 protected void processOptions(LinkedList args) throws IOException {
   popPreserveOption(args);
   CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "f", "d");
+  cf.addOptionWithValue("t");
+  cf.addOptionWithValue("q");
   cf.parse(args);
   setDirectWrite(cf.getOpt("d"));
   setOverwrite(cf.getOpt("f"));
+  setThreadCount(cf.getOptValue("t"));
+  setThreadPoolQueueSize(cf.getOptValue("q"));
   // should have a -r option
   setRecursive(true);
   getRemoteDestination(args);
diff --git 
a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 4

[hadoop] branch branch-3.3 updated: HADOOP-18023. Allow cp command to run with multi threads. (#3721)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new bc40a41  HADOOP-18023. Allow cp command to run with multi threads. 
(#3721)
bc40a41 is described below

commit bc40a410643e30cb41b327bcf8d7ff3040dd15ff
Author: smarthan <1139557...@qq.com>
AuthorDate: Mon Nov 29 20:45:08 2021 +0800

HADOOP-18023. Allow cp command to run with multi threads. (#3721)

(cherry picked from commit 932a78fe38b34a923f6852a1a19482075806ecba)
---
 .../org/apache/hadoop/fs/shell/CopyCommands.java   |  39 ++--
 .../src/site/markdown/FileSystemShell.md   |  11 +-
 .../org/apache/hadoop/fs/shell/TestCpCommand.java  | 210 +
 .../hadoop-common/src/test/resources/testConf.xml  |  47 +++--
 4 files changed, 275 insertions(+), 32 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
index 5a0652b..b03d7de 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
@@ -147,33 +147,40 @@ class CopyCommands {
 }
   }
 
-  static class Cp extends CommandWithDestination {
+  static class Cp extends CopyCommandWithMultiThread {
 public static final String NAME = "cp";
 public static final String USAGE =
-"[-f] [-p | -p[topax]] [-d]  ... ";
+"[-f] [-p | -p[topax]] [-d] [-t ]"
++ " [-q ]  ... ";
 public static final String DESCRIPTION =
-  "Copy files that match the file pattern  to a " +
-  "destination.  When copying multiple files, the destination " +
-  "must be a directory. Passing -p preserves status " +
-  "[topax] (timestamps, ownership, permission, ACLs, XAttr). " +
-  "If -p is specified with no , then preserves " +
-  "timestamps, ownership, permission. If -pa is specified, " +
-  "then preserves permission also because ACL is a super-set of " +
-  "permission. Passing -f overwrites the destination if it " +
-  "already exists. raw namespace extended attributes are preserved " +
-  "if (1) they are supported (HDFS only) and, (2) all of the source and " +
-  "target pathnames are in the /.reserved/raw hierarchy. raw namespace " +
-  "xattr preservation is determined solely by the presence (or absence) " +
-"of the /.reserved/raw prefix and not by the -p option. Passing -d "+
-"will skip creation of temporary file(._COPYING_).\n";
+"Copy files that match the file pattern  to a destination."
++ " When copying multiple files, the destination must be a "
++ "directory.\nFlags :\n"
++ "  -p[topax] : Preserve file attributes [topx] (timestamps, "
++ "ownership, permission, ACL, XAttr). If -p is specified with "
++ "no arg, then preserves timestamps, ownership, permission. "
++ "If -pa is specified, then preserves permission also because "
++ "ACL is a super-set of permission. Determination of whether raw "
++ "namespace extended attributes are preserved is independent of "
++ "the -p flag.\n"
++ "  -f : Overwrite the destination if it already exists.\n"
++ "  -d : Skip creation of temporary file(._COPYING_).\n"
++ "  -t  : Number of threads to be used, "
++ "default is 1.\n"
++ "  -q  : Thread pool queue size to be "
++ "used, default is 1024.\n";
 
 @Override
 protected void processOptions(LinkedList args) throws IOException {
   popPreserveOption(args);
   CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "f", "d");
+  cf.addOptionWithValue("t");
+  cf.addOptionWithValue("q");
   cf.parse(args);
   setDirectWrite(cf.getOpt("d"));
   setOverwrite(cf.getOpt("f"));
+  setThreadCount(cf.getOptValue("t"));
+  setThreadPoolQueueSize(cf.getOptValue("q"));
   // should have a -r option
   setRecursive(true);
   getRemoteDestination(args);
diff --git 
a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 46ae405..b1e465

[hadoop] branch trunk updated (829af89 -> 932a78f)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git.


from 829af89  HDFS-16344. Improve DirectoryScanner.Stats#toString (#3695)
 add 932a78f  HADOOP-18023. Allow cp command to run with multi threads. 
(#3721)

No new revisions were added by this update.

Summary of changes:
 .../org/apache/hadoop/fs/shell/CopyCommands.java   | 39 ++
 .../src/site/markdown/FileSystemShell.md   | 11 +++--
 .../{TestCopyToLocal.java => TestCpCommand.java}   | 34 
 .../hadoop-common/src/test/resources/testConf.xml  | 47 --
 4 files changed, 82 insertions(+), 49 deletions(-)
 copy 
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/{TestCopyToLocal.java
 => TestCpCommand.java} (86%)

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.2 updated: HADOOP-17975. Fallback to simple auth does not work for a secondary DistributedFileSystem instance. (#3579)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new dbcadcc  HADOOP-17975. Fallback to simple auth does not work for a 
secondary DistributedFileSystem instance. (#3579)
dbcadcc is described below

commit dbcadcc21e0b02e55ff55ff0dd8a73a35d3fd77d
Author: Istvan Fajth 
AuthorDate: Wed Nov 24 11:44:57 2021 +0100

HADOOP-17975. Fallback to simple auth does not work for a secondary 
DistributedFileSystem instance. (#3579)

(cherry picked from commit ae3ba45db58467ce57b0a440e236fd80f6be9ec6)
---
 .../main/java/org/apache/hadoop/ipc/Client.java|  71 ---
 .../java/org/apache/hadoop/ipc/TestRpcBase.java|  20 +-
 .../java/org/apache/hadoop/ipc/TestSaslRPC.java| 212 +++--
 3 files changed, 217 insertions(+), 86 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index b9ce20a..ef44c7f 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -807,17 +807,18 @@ public class Client implements AutoCloseable {
  */
 private synchronized void setupIOstreams(
 AtomicBoolean fallbackToSimpleAuth) {
-  if (socket != null || shouldCloseConnection.get()) {
-return;
-  }
-  UserGroupInformation ticket = remoteId.getTicket();
-  if (ticket != null) {
-final UserGroupInformation realUser = ticket.getRealUser();
-if (realUser != null) {
-  ticket = realUser;
-}
-  }
   try {
+if (socket != null || shouldCloseConnection.get()) {
+  setFallBackToSimpleAuth(fallbackToSimpleAuth);
+  return;
+}
+UserGroupInformation ticket = remoteId.getTicket();
+if (ticket != null) {
+  final UserGroupInformation realUser = ticket.getRealUser();
+  if (realUser != null) {
+ticket = realUser;
+  }
+}
 connectingThread.set(Thread.currentThread());
 if (LOG.isDebugEnabled()) {
   LOG.debug("Connecting to "+server);
@@ -863,20 +864,8 @@ public class Client implements AutoCloseable {
   remoteId.saslQop =
   (String)saslRpcClient.getNegotiatedProperty(Sasl.QOP);
   LOG.debug("Negotiated QOP is :" + remoteId.saslQop);
-  if (fallbackToSimpleAuth != null) {
-fallbackToSimpleAuth.set(false);
-  }
-} else if (UserGroupInformation.isSecurityEnabled()) {
-  if (!fallbackAllowed) {
-throw new AccessControlException(
-"Server asks us to fall back to SIMPLE " +
-"auth, but this client is configured to only allow secure 
" +
-"connections.");
-  }
-  if (fallbackToSimpleAuth != null) {
-fallbackToSimpleAuth.set(true);
-  }
 }
+setFallBackToSimpleAuth(fallbackToSimpleAuth);
   }
 
   if (doPing) {
@@ -909,7 +898,41 @@ public class Client implements AutoCloseable {
 connectingThread.set(null);
   }
 }
-
+
+private void setFallBackToSimpleAuth(AtomicBoolean fallbackToSimpleAuth)
+throws AccessControlException {
+  if (authMethod == null || authProtocol != AuthProtocol.SASL) {
+if (authProtocol == AuthProtocol.SASL) {
+  LOG.trace("Auth method is not set, yield from setting auth 
fallback.");
+}
+return;
+  }
+  if (fallbackToSimpleAuth == null) {
+// this should happen only during testing.
+LOG.trace("Connection {} will skip to set fallbackToSimpleAuth as it 
is null.", remoteId);
+  } else {
+if (fallbackToSimpleAuth.get()) {
+  // we already set the value to true, we do not need to examine again.
+  return;
+}
+  }
+  if (authMethod != AuthMethod.SIMPLE) {
+if (fallbackToSimpleAuth != null) {
+  LOG.trace("Disabling fallbackToSimpleAuth, target does not use 
SIMPLE authentication.");
+  fallbackToSimpleAuth.set(false);
+}
+  } else if (UserGroupInformation.isSecurityEnabled()) {
+if (!fallbackAllowed) {
+  throw new AccessControlException("Server asks us to fall back to 
SIMPLE auth, but this "
+  + "client is configured to only allow secure connections.");
+}
+if (fallbackToSimpleAuth != null) {
+  LOG.trace("Enabling fall

[hadoop] branch branch-3.3 updated: HADOOP-17975. Fallback to simple auth does not work for a secondary DistributedFileSystem instance. (#3579)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 48e95d8  HADOOP-17975. Fallback to simple auth does not work for a 
secondary DistributedFileSystem instance. (#3579)
48e95d8 is described below

commit 48e95d8109356c16a68ca50c3f9ecd22f39fa19b
Author: Istvan Fajth 
AuthorDate: Wed Nov 24 11:44:57 2021 +0100

HADOOP-17975. Fallback to simple auth does not work for a secondary 
DistributedFileSystem instance. (#3579)

(cherry picked from commit ae3ba45db58467ce57b0a440e236fd80f6be9ec6)
---
 .../main/java/org/apache/hadoop/ipc/Client.java|  71 ---
 .../java/org/apache/hadoop/ipc/TestRpcBase.java|  20 +-
 .../java/org/apache/hadoop/ipc/TestSaslRPC.java| 212 +++--
 3 files changed, 217 insertions(+), 86 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index 712db04..d5662d14 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -807,17 +807,18 @@ public class Client implements AutoCloseable {
  */
 private synchronized void setupIOstreams(
 AtomicBoolean fallbackToSimpleAuth) {
-  if (socket != null || shouldCloseConnection.get()) {
-return;
-  }
-  UserGroupInformation ticket = remoteId.getTicket();
-  if (ticket != null) {
-final UserGroupInformation realUser = ticket.getRealUser();
-if (realUser != null) {
-  ticket = realUser;
-}
-  }
   try {
+if (socket != null || shouldCloseConnection.get()) {
+  setFallBackToSimpleAuth(fallbackToSimpleAuth);
+  return;
+}
+UserGroupInformation ticket = remoteId.getTicket();
+if (ticket != null) {
+  final UserGroupInformation realUser = ticket.getRealUser();
+  if (realUser != null) {
+ticket = realUser;
+  }
+}
 connectingThread.set(Thread.currentThread());
 if (LOG.isDebugEnabled()) {
   LOG.debug("Connecting to "+server);
@@ -863,20 +864,8 @@ public class Client implements AutoCloseable {
   remoteId.saslQop =
   (String)saslRpcClient.getNegotiatedProperty(Sasl.QOP);
   LOG.debug("Negotiated QOP is :" + remoteId.saslQop);
-  if (fallbackToSimpleAuth != null) {
-fallbackToSimpleAuth.set(false);
-  }
-} else if (UserGroupInformation.isSecurityEnabled()) {
-  if (!fallbackAllowed) {
-throw new AccessControlException(
-"Server asks us to fall back to SIMPLE " +
-"auth, but this client is configured to only allow secure 
" +
-"connections.");
-  }
-  if (fallbackToSimpleAuth != null) {
-fallbackToSimpleAuth.set(true);
-  }
 }
+setFallBackToSimpleAuth(fallbackToSimpleAuth);
   }
 
   if (doPing) {
@@ -909,7 +898,41 @@ public class Client implements AutoCloseable {
 connectingThread.set(null);
   }
 }
-
+
+private void setFallBackToSimpleAuth(AtomicBoolean fallbackToSimpleAuth)
+throws AccessControlException {
+  if (authMethod == null || authProtocol != AuthProtocol.SASL) {
+if (authProtocol == AuthProtocol.SASL) {
+  LOG.trace("Auth method is not set, yield from setting auth 
fallback.");
+}
+return;
+  }
+  if (fallbackToSimpleAuth == null) {
+// this should happen only during testing.
+LOG.trace("Connection {} will skip to set fallbackToSimpleAuth as it 
is null.", remoteId);
+  } else {
+if (fallbackToSimpleAuth.get()) {
+  // we already set the value to true, we do not need to examine again.
+  return;
+}
+  }
+  if (authMethod != AuthMethod.SIMPLE) {
+if (fallbackToSimpleAuth != null) {
+  LOG.trace("Disabling fallbackToSimpleAuth, target does not use 
SIMPLE authentication.");
+  fallbackToSimpleAuth.set(false);
+}
+  } else if (UserGroupInformation.isSecurityEnabled()) {
+if (!fallbackAllowed) {
+  throw new AccessControlException("Server asks us to fall back to 
SIMPLE auth, but this "
+  + "client is configured to only allow secure connections.");
+}
+if (fallbackToSimpleAuth != null) {
+  LOG.trace("Enabling fall

[hadoop] branch trunk updated: HADOOP-17975. Fallback to simple auth does not work for a secondary DistributedFileSystem instance. (#3579)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new ae3ba45  HADOOP-17975. Fallback to simple auth does not work for a 
secondary DistributedFileSystem instance. (#3579)
ae3ba45 is described below

commit ae3ba45db58467ce57b0a440e236fd80f6be9ec6
Author: Istvan Fajth 
AuthorDate: Wed Nov 24 11:44:57 2021 +0100

HADOOP-17975. Fallback to simple auth does not work for a secondary 
DistributedFileSystem instance. (#3579)
---
 .../main/java/org/apache/hadoop/ipc/Client.java|  71 ---
 .../java/org/apache/hadoop/ipc/TestRpcBase.java|  20 +-
 .../java/org/apache/hadoop/ipc/TestSaslRPC.java| 212 +++--
 3 files changed, 217 insertions(+), 86 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index 96f925f..49432af 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -807,17 +807,18 @@ public class Client implements AutoCloseable {
  */
 private synchronized void setupIOstreams(
 AtomicBoolean fallbackToSimpleAuth) {
-  if (socket != null || shouldCloseConnection.get()) {
-return;
-  }
-  UserGroupInformation ticket = remoteId.getTicket();
-  if (ticket != null) {
-final UserGroupInformation realUser = ticket.getRealUser();
-if (realUser != null) {
-  ticket = realUser;
-}
-  }
   try {
+if (socket != null || shouldCloseConnection.get()) {
+  setFallBackToSimpleAuth(fallbackToSimpleAuth);
+  return;
+}
+UserGroupInformation ticket = remoteId.getTicket();
+if (ticket != null) {
+  final UserGroupInformation realUser = ticket.getRealUser();
+  if (realUser != null) {
+ticket = realUser;
+  }
+}
 connectingThread.set(Thread.currentThread());
 if (LOG.isDebugEnabled()) {
   LOG.debug("Connecting to "+server);
@@ -863,20 +864,8 @@ public class Client implements AutoCloseable {
   remoteId.saslQop =
   (String)saslRpcClient.getNegotiatedProperty(Sasl.QOP);
   LOG.debug("Negotiated QOP is :" + remoteId.saslQop);
-  if (fallbackToSimpleAuth != null) {
-fallbackToSimpleAuth.set(false);
-  }
-} else if (UserGroupInformation.isSecurityEnabled()) {
-  if (!fallbackAllowed) {
-throw new AccessControlException(
-"Server asks us to fall back to SIMPLE " +
-"auth, but this client is configured to only allow secure 
" +
-"connections.");
-  }
-  if (fallbackToSimpleAuth != null) {
-fallbackToSimpleAuth.set(true);
-  }
 }
+setFallBackToSimpleAuth(fallbackToSimpleAuth);
   }
 
   if (doPing) {
@@ -909,7 +898,41 @@ public class Client implements AutoCloseable {
 connectingThread.set(null);
   }
 }
-
+
+private void setFallBackToSimpleAuth(AtomicBoolean fallbackToSimpleAuth)
+throws AccessControlException {
+  if (authMethod == null || authProtocol != AuthProtocol.SASL) {
+if (authProtocol == AuthProtocol.SASL) {
+  LOG.trace("Auth method is not set, yield from setting auth 
fallback.");
+}
+return;
+  }
+  if (fallbackToSimpleAuth == null) {
+// this should happen only during testing.
+LOG.trace("Connection {} will skip to set fallbackToSimpleAuth as it 
is null.", remoteId);
+  } else {
+if (fallbackToSimpleAuth.get()) {
+  // we already set the value to true, we do not need to examine again.
+  return;
+}
+  }
+  if (authMethod != AuthMethod.SIMPLE) {
+if (fallbackToSimpleAuth != null) {
+  LOG.trace("Disabling fallbackToSimpleAuth, target does not use 
SIMPLE authentication.");
+  fallbackToSimpleAuth.set(false);
+}
+  } else if (UserGroupInformation.isSecurityEnabled()) {
+if (!fallbackAllowed) {
+  throw new AccessControlException("Server asks us to fall back to 
SIMPLE auth, but this "
+  + "client is configured to only allow secure connections.");
+}
+if (fallbackToSimpleAuth != null) {
+  LOG.trace("Enabling fallbackToSimpleAuth for target, as we are 
allowed to fall back.");
+  fallbackToSimpleAuth.set(true);
+  

[hadoop] branch branch-3.2 updated: HDFS-16343. Add some debug logs when the dfsUsed are not used during Datanode startup. (#3694)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new f5ff4e1  HDFS-16343. Add some debug logs when the dfsUsed are not used 
during Datanode startup. (#3694)
f5ff4e1 is described below

commit f5ff4e1ca831b65e1dc38206c621fcfdc4ebd842
Author: Mukul Kumar Singh 
AuthorDate: Tue Nov 23 19:06:14 2021 +0530

HDFS-16343. Add some debug logs when the dfsUsed are not used during 
Datanode startup. (#3694)

(cherry picked from commit faa4eeacb1f708b7c3a0701886fa63068327e9b3)
---
 .../server/datanode/fsdataset/impl/BlockPoolSlice.java | 18 --
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index ea77505..98eacea 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -272,9 +272,13 @@ class BlockPoolSlice {
 long mtime;
 Scanner sc;
 
+File duCacheFile = new File(currentDir, DU_CACHE_FILE);
 try {
-  sc = new Scanner(new File(currentDir, DU_CACHE_FILE), "UTF-8");
+  sc = new Scanner(duCacheFile, "UTF-8");
 } catch (FileNotFoundException fnfe) {
+  FsDatasetImpl.LOG.warn("{} file missing in {}, will proceed with Du " +
+  "for space computation calculation, ",
+  DU_CACHE_FILE, currentDir);
   return -1;
 }
 
@@ -283,21 +287,31 @@ class BlockPoolSlice {
   if (sc.hasNextLong()) {
 cachedDfsUsed = sc.nextLong();
   } else {
+FsDatasetImpl.LOG.warn("cachedDfsUsed not found in file:{}, will " +
+"proceed with Du for space computation calculation, ",
+duCacheFile);
 return -1;
   }
   // Get the recorded mtime from the file.
   if (sc.hasNextLong()) {
 mtime = sc.nextLong();
   } else {
+FsDatasetImpl.LOG.warn("mtime not found in file:{}, will proceed" +
+" with Du for space computation calculation, ", duCacheFile);
 return -1;
   }
 
+  long elapsedTime = timer.now() - mtime;
   // Return the cached value if mtime is okay.
-  if (mtime > 0 && (timer.now() - mtime < cachedDfsUsedCheckTime)) {
+  if (mtime > 0 && (elapsedTime < cachedDfsUsedCheckTime)) {
 FsDatasetImpl.LOG.info("Cached dfsUsed found for " + currentDir + ": " 
+
 cachedDfsUsed);
 return cachedDfsUsed;
   }
+  FsDatasetImpl.LOG.warn("elapsed time:{} is greater than threshold:{}," +
+  " mtime:{} in file:{}, will proceed with Du for space" +
+  " computation calculation",
+  elapsedTime, cachedDfsUsedCheckTime, mtime, duCacheFile);
   return -1;
 } finally {
   sc.close();

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.3 updated: HDFS-16343. Add some debug logs when the dfsUsed are not used during Datanode startup. (#3694)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 97de402  HDFS-16343. Add some debug logs when the dfsUsed are not used 
during Datanode startup. (#3694)
97de402 is described below

commit 97de402970d390b25764b8a67f468a9594308cb3
Author: Mukul Kumar Singh 
AuthorDate: Tue Nov 23 19:06:14 2021 +0530

HDFS-16343. Add some debug logs when the dfsUsed are not used during 
Datanode startup. (#3694)

(cherry picked from commit faa4eeacb1f708b7c3a0701886fa63068327e9b3)
---
 .../server/datanode/fsdataset/impl/BlockPoolSlice.java | 18 --
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index 20df0e9..4dc6707 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -293,9 +293,13 @@ class BlockPoolSlice {
 long mtime;
 Scanner sc;
 
+File duCacheFile = new File(currentDir, DU_CACHE_FILE);
 try {
-  sc = new Scanner(new File(currentDir, DU_CACHE_FILE), "UTF-8");
+  sc = new Scanner(duCacheFile, "UTF-8");
 } catch (FileNotFoundException fnfe) {
+  FsDatasetImpl.LOG.warn("{} file missing in {}, will proceed with Du " +
+  "for space computation calculation, ",
+  DU_CACHE_FILE, currentDir);
   return -1;
 }
 
@@ -304,21 +308,31 @@ class BlockPoolSlice {
   if (sc.hasNextLong()) {
 cachedDfsUsed = sc.nextLong();
   } else {
+FsDatasetImpl.LOG.warn("cachedDfsUsed not found in file:{}, will " +
+"proceed with Du for space computation calculation, ",
+duCacheFile);
 return -1;
   }
   // Get the recorded mtime from the file.
   if (sc.hasNextLong()) {
 mtime = sc.nextLong();
   } else {
+FsDatasetImpl.LOG.warn("mtime not found in file:{}, will proceed" +
+" with Du for space computation calculation, ", duCacheFile);
 return -1;
   }
 
+  long elapsedTime = timer.now() - mtime;
   // Return the cached value if mtime is okay.
-  if (mtime > 0 && (timer.now() - mtime < cachedDfsUsedCheckTime)) {
+  if (mtime > 0 && (elapsedTime < cachedDfsUsedCheckTime)) {
 FsDatasetImpl.LOG.info("Cached dfsUsed found for " + currentDir + ": " 
+
 cachedDfsUsed);
 return cachedDfsUsed;
   }
+  FsDatasetImpl.LOG.warn("elapsed time:{} is greater than threshold:{}," +
+  " mtime:{} in file:{}, will proceed with Du for space" +
+  " computation calculation",
+  elapsedTime, cachedDfsUsedCheckTime, mtime, duCacheFile);
   return -1;
 } finally {
   sc.close();

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: HDFS-16343. Add some debug logs when the dfsUsed are not used during Datanode startup. (#3694)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new faa4eea  HDFS-16343. Add some debug logs when the dfsUsed are not used 
during Datanode startup. (#3694)
faa4eea is described below

commit faa4eeacb1f708b7c3a0701886fa63068327e9b3
Author: Mukul Kumar Singh 
AuthorDate: Tue Nov 23 19:06:14 2021 +0530

HDFS-16343. Add some debug logs when the dfsUsed are not used during 
Datanode startup. (#3694)
---
 .../server/datanode/fsdataset/impl/BlockPoolSlice.java | 18 --
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index f2d4e08..0ef1d56 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -298,9 +298,13 @@ class BlockPoolSlice {
 long mtime;
 Scanner sc;
 
+File duCacheFile = new File(currentDir, DU_CACHE_FILE);
 try {
-  sc = new Scanner(new File(currentDir, DU_CACHE_FILE), "UTF-8");
+  sc = new Scanner(duCacheFile, "UTF-8");
 } catch (FileNotFoundException fnfe) {
+  FsDatasetImpl.LOG.warn("{} file missing in {}, will proceed with Du " +
+  "for space computation calculation, ",
+  DU_CACHE_FILE, currentDir);
   return -1;
 }
 
@@ -309,21 +313,31 @@ class BlockPoolSlice {
   if (sc.hasNextLong()) {
 cachedDfsUsed = sc.nextLong();
   } else {
+FsDatasetImpl.LOG.warn("cachedDfsUsed not found in file:{}, will " +
+"proceed with Du for space computation calculation, ",
+duCacheFile);
 return -1;
   }
   // Get the recorded mtime from the file.
   if (sc.hasNextLong()) {
 mtime = sc.nextLong();
   } else {
+FsDatasetImpl.LOG.warn("mtime not found in file:{}, will proceed" +
+" with Du for space computation calculation, ", duCacheFile);
 return -1;
   }
 
+  long elapsedTime = timer.now() - mtime;
   // Return the cached value if mtime is okay.
-  if (mtime > 0 && (timer.now() - mtime < cachedDfsUsedCheckTime)) {
+  if (mtime > 0 && (elapsedTime < cachedDfsUsedCheckTime)) {
 FsDatasetImpl.LOG.info("Cached dfsUsed found for " + currentDir + ": " 
+
 cachedDfsUsed);
 return cachedDfsUsed;
   }
+  FsDatasetImpl.LOG.warn("elapsed time:{} is greater than threshold:{}," +
+  " mtime:{} in file:{}, will proceed with Du for space" +
+  " computation calculation",
+  elapsedTime, cachedDfsUsedCheckTime, mtime, duCacheFile);
   return -1;
 } finally {
   sc.close();

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] 01/02: HDFS-16173.Improve CopyCommands#Put#executor queue configurability. (#3302)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit c5c158df7319c55e47b9c88d424b52720af4aaa1
Author: jianghuazhu <740087...@qq.com>
AuthorDate: Fri Aug 27 11:41:44 2021 +0800

HDFS-16173.Improve CopyCommands#Put#executor queue configurability. (#3302)

Co-authored-by: zhujianghua 
Reviewed-by: Hui Fei 
Reviewed-by: Viraj Jasani 
(cherry picked from commit 4c94831364e9258247029c22a222a665771ab4c0)
(cherry picked from commit 7c663043b2c1e207dd8c05e09e17811d68badfac)
(cherry picked from commit dac74b0e115fa466467aa1de2a71fb8704a04639)
---
 .../org/apache/hadoop/fs/shell/CopyCommands.java   | 34 --
 .../src/site/markdown/FileSystemShell.md   |  4 ++-
 .../hadoop/fs/shell/TestCopyPreserveFlag.java  | 16 ++
 .../hadoop-common/src/test/resources/testConf.xml  | 18 +++-
 4 files changed, 62 insertions(+), 10 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
index f296712..ea79e4e 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
@@ -37,6 +37,8 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathIsDirectoryException;
 import org.apache.hadoop.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /** Various commands for copy files */
 @InterfaceAudience.Private
@@ -238,7 +240,11 @@ class CopyCommands {
*  Copy local files to a remote filesystem
*/
   public static class Put extends CommandWithDestination {
+
+public static final Logger LOG = LoggerFactory.getLogger(Put.class);
+
 private ThreadPoolExecutor executor = null;
+private int threadPoolQueueSize = 1024;
 private int numThreads = 1;
 
 private static final int MAX_THREADS =
@@ -246,7 +252,8 @@ class CopyCommands {
 
 public static final String NAME = "put";
 public static final String USAGE =
-"[-f] [-p] [-l] [-d] [-t ]  ... ";
+"[-f] [-p] [-l] [-d] [-t ] [-q ] 
" +
+" ... ";
 public static final String DESCRIPTION =
 "Copy files from the local file system " +
 "into fs. Copying fails if the file already " +
@@ -255,6 +262,8 @@ class CopyCommands {
 "  -p : Preserves timestamps, ownership and the mode.\n" +
 "  -f : Overwrites the destination if it already exists.\n" +
 "  -t  : Number of threads to be used, default is 1.\n" +
+"  -q  : ThreadPool queue size to be used, " +
+"default is 1024.\n" +
 "  -l : Allow DataNode to lazily persist the file to disk. Forces" +
 "  replication factor of 1. This flag will result in reduced" +
 "  durability. Use with care.\n" +
@@ -265,8 +274,10 @@ class CopyCommands {
   CommandFormat cf =
   new CommandFormat(1, Integer.MAX_VALUE, "f", "p", "l", "d");
   cf.addOptionWithValue("t");
+  cf.addOptionWithValue("q");
   cf.parse(args);
   setNumberThreads(cf.getOptValue("t"));
+  setThreadPoolQueueSize(cf.getOptValue("q"));
   setOverwrite(cf.getOpt("f"));
   setPreserve(cf.getOpt("p"));
   setLazyPersist(cf.getOpt("l"));
@@ -298,7 +309,7 @@ class CopyCommands {
   }
 
   executor = new ThreadPoolExecutor(numThreads, numThreads, 1,
-  TimeUnit.SECONDS, new ArrayBlockingQueue<>(1024),
+  TimeUnit.SECONDS, new ArrayBlockingQueue<>(threadPoolQueueSize),
   new ThreadPoolExecutor.CallerRunsPolicy());
   super.processArguments(args);
 
@@ -328,6 +339,25 @@ class CopyCommands {
   }
 }
 
+private void setThreadPoolQueueSize(String numThreadPoolQueueSize) {
+  if (numThreadPoolQueueSize != null) {
+int parsedValue = Integer.parseInt(numThreadPoolQueueSize);
+if (parsedValue < 1) {
+  LOG.warn("The value of the thread pool queue size cannot be " +
+  "less than 1, and the default value is used here. " +
+  "The default size is 1024.");
+  threadPoolQueueSize = 1024;
+} else {
+  threadPoolQueueSize = parsedValue;
+}
+  }
+}
+
+@VisibleForTesting
+protected int getThreadPoolQueueSize() {
+  return threadPoolQueueSize;
+}
+
 private void copyFile(PathData src, PathData t

[hadoop] 02/02: HADOOP-17998. Allow get command to run with multi threads. (#3645)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit d15afa3da832f76f53d8aefd40794531dbe18ce1
Author: smarthan <1139557...@qq.com>
AuthorDate: Mon Nov 22 19:37:05 2021 +0800

HADOOP-17998. Allow get command to run with multi threads. (#3645)

(cherry picked from commit 63018dc73f4d29632e93be08d035ab9a7e73531c)

 Conflicts:

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java

(cherry picked from commit cbb3ba135cfa3a9bc9f444e8b4a7875758721a3c)
(cherry picked from commit 6e179d421d73fd1a5b0f4dac5f6decaecd424376)
---
 .../hadoop/fs/shell/CommandWithDestination.java|   8 +-
 .../fs/shell/CopyCommandWithMultiThread.java   | 155 +
 .../org/apache/hadoop/fs/shell/CopyCommands.java   | 147 ---
 .../src/site/markdown/FileSystemShell.md   |  36 +++--
 .../apache/hadoop/fs/shell/TestCopyFromLocal.java  |  95 ++---
 .../hadoop/fs/shell/TestCopyPreserveFlag.java  |  43 +-
 ...TestCopyFromLocal.java => TestCopyToLocal.java} | 152 
 .../hadoop-common/src/test/resources/testConf.xml  |  90 ++--
 8 files changed, 429 insertions(+), 297 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
index 0bd4882..ffae5d5 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
@@ -387,11 +387,11 @@ abstract class CommandWithDestination extends FsCommand {
 
   /**
* If direct write is disabled ,copies the stream contents to a temporary
-   * file "._COPYING_". If the copy is
-   * successful, the temporary file will be renamed to the real path,
-   * else the temporary file will be deleted.
+   * file "target._COPYING_". If the copy is successful, the temporary file
+   * will be renamed to the real path, else the temporary file will be deleted.
* if direct write is enabled , then creation temporary file is skipped.
-   * @param in the input stream for the copy
+   *
+   * @param in the input stream for the copy
* @param target where to store the contents of the stream
* @throws IOException if copy fails
*/ 
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
new file mode 100644
index 000..7b54b9e
--- /dev/null
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.shell;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Abstract command to enable sub copy commands run with multi-thread.
+ */
+public abstract class CopyCommandWithMultiThread
+extends CommandWithDestination {
+
+  private int threadCount = 1;
+  private ThreadPoolExecutor executor = null;
+  private int threadPoolQueueSize = DEFAULT_QUEUE_SIZE;
+
+  public static final int DEFAULT_QUEUE_SIZE = 1024;
+
+  /**
+   * set thread count by option value, if the value less than 1,
+   * use 1 instead.
+   *
+   * @param optValue option value
+   */
+  protected void setThreadCount(String optValue) {
+if (optValue != null) {
+  threadCount = Math.max(Integer.parseInt(optValue), 1);
+}
+  }
+
+  /**
+   * set thread pool queue size by option value, if the value less than 1,
+   * use DEFAULT_QUEUE_SIZE 

[hadoop] branch branch-3.2.3 updated (fa737f7 -> d15afa3)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a change to branch branch-3.2.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git.


from fa737f7  YARN-10438. Handle null containerId in 
ClientRMService#getContainerReport() (#2313)
 new c5c158d  HDFS-16173.Improve CopyCommands#Put#executor queue 
configurability. (#3302)
 new d15afa3  HADOOP-17998. Allow get command to run with multi threads. 
(#3645)

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../hadoop/fs/shell/CommandWithDestination.java|   8 +-
 .../fs/shell/CopyCommandWithMultiThread.java   | 155 +
 .../org/apache/hadoop/fs/shell/CopyCommands.java   | 123 +---
 .../src/site/markdown/FileSystemShell.md   |  34 +++--
 .../apache/hadoop/fs/shell/TestCopyFromLocal.java  |  95 ++---
 .../hadoop/fs/shell/TestCopyPreserveFlag.java  |  59 +++-
 ...TestCopyFromLocal.java => TestCopyToLocal.java} | 152 
 .../hadoop-common/src/test/resources/testConf.xml  |  92 ++--
 8 files changed, 451 insertions(+), 267 deletions(-)
 create mode 100644 
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
 copy 
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/{TestCopyFromLocal.java
 => TestCopyToLocal.java} (54%)

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.2 updated: HADOOP-17998. Allow get command to run with multi threads. (#3645)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new 6e179d4  HADOOP-17998. Allow get command to run with multi threads. 
(#3645)
6e179d4 is described below

commit 6e179d421d73fd1a5b0f4dac5f6decaecd424376
Author: smarthan <1139557...@qq.com>
AuthorDate: Mon Nov 22 19:37:05 2021 +0800

HADOOP-17998. Allow get command to run with multi threads. (#3645)

(cherry picked from commit 63018dc73f4d29632e93be08d035ab9a7e73531c)

 Conflicts:

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java

(cherry picked from commit cbb3ba135cfa3a9bc9f444e8b4a7875758721a3c)
---
 .../hadoop/fs/shell/CommandWithDestination.java|   8 +-
 .../fs/shell/CopyCommandWithMultiThread.java   | 155 +
 .../org/apache/hadoop/fs/shell/CopyCommands.java   | 147 ---
 .../src/site/markdown/FileSystemShell.md   |  36 +++--
 .../apache/hadoop/fs/shell/TestCopyFromLocal.java  |  95 ++---
 .../hadoop/fs/shell/TestCopyPreserveFlag.java  |  43 +-
 ...TestCopyFromLocal.java => TestCopyToLocal.java} | 152 
 .../hadoop-common/src/test/resources/testConf.xml  |  90 ++--
 8 files changed, 429 insertions(+), 297 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
index 0bd4882..ffae5d5 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
@@ -387,11 +387,11 @@ abstract class CommandWithDestination extends FsCommand {
 
   /**
* If direct write is disabled ,copies the stream contents to a temporary
-   * file "._COPYING_". If the copy is
-   * successful, the temporary file will be renamed to the real path,
-   * else the temporary file will be deleted.
+   * file "target._COPYING_". If the copy is successful, the temporary file
+   * will be renamed to the real path, else the temporary file will be deleted.
* if direct write is enabled , then creation temporary file is skipped.
-   * @param in the input stream for the copy
+   *
+   * @param in the input stream for the copy
* @param target where to store the contents of the stream
* @throws IOException if copy fails
*/ 
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
new file mode 100644
index 000..aed4030
--- /dev/null
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.shell;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+
+/**
+ * Abstract command to enable sub copy commands run with multi-thread.
+ */
+public abstract class CopyCommandWithMultiThread
+extends CommandWithDestination {
+
+  private int threadCount = 1;
+  private ThreadPoolExecutor executor = null;
+  private int threadPoolQueueSize = DEFAULT_QUEUE_SIZE;
+
+  public static final int DEFAULT_QUEUE_SIZE = 1024;
+
+  /**
+   * set thread count by option value, if the value less than 1,
+   * use 1 instead.
+   *
+   * @param optValue option value
+   */
+  protected void setThreadCount(String optValue) {
+if (optValue != null) {
+  threadCount = Math.max(Integer.parseInt(optValue), 1);
+}
+  }
+

[hadoop] branch branch-3.3.2 updated: HADOOP-17998. Allow get command to run with multi threads. (#3645)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3.2 by this push:
 new da57ee9  HADOOP-17998. Allow get command to run with multi threads. 
(#3645)
da57ee9 is described below

commit da57ee936b16f3d30d8ea9bbbdae601059d8d279
Author: smarthan <1139557...@qq.com>
AuthorDate: Mon Nov 22 19:37:05 2021 +0800

HADOOP-17998. Allow get command to run with multi threads. (#3645)

(cherry picked from commit 63018dc73f4d29632e93be08d035ab9a7e73531c)

 Conflicts:

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java

(cherry picked from commit cbb3ba135cfa3a9bc9f444e8b4a7875758721a3c)
---
 .../hadoop/fs/shell/CommandWithDestination.java|   8 +-
 .../fs/shell/CopyCommandWithMultiThread.java   | 155 +
 .../org/apache/hadoop/fs/shell/CopyCommands.java   | 147 ---
 .../src/site/markdown/FileSystemShell.md   |  36 +++--
 .../apache/hadoop/fs/shell/TestCopyFromLocal.java  |  95 ++---
 .../hadoop/fs/shell/TestCopyPreserveFlag.java  |  43 +-
 ...TestCopyFromLocal.java => TestCopyToLocal.java} | 152 
 .../hadoop-common/src/test/resources/testConf.xml  |  90 ++--
 8 files changed, 429 insertions(+), 297 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
index 90a709d..f6f4247 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
@@ -397,11 +397,11 @@ abstract class CommandWithDestination extends FsCommand {
 
   /**
* If direct write is disabled ,copies the stream contents to a temporary
-   * file "._COPYING_". If the copy is
-   * successful, the temporary file will be renamed to the real path,
-   * else the temporary file will be deleted.
+   * file "target._COPYING_". If the copy is successful, the temporary file
+   * will be renamed to the real path, else the temporary file will be deleted.
* if direct write is enabled , then creation temporary file is skipped.
-   * @param in the input stream for the copy
+   *
+   * @param in the input stream for the copy
* @param target where to store the contents of the stream
* @throws IOException if copy fails
*/ 
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
new file mode 100644
index 000..aed4030
--- /dev/null
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.shell;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+
+/**
+ * Abstract command to enable sub copy commands run with multi-thread.
+ */
+public abstract class CopyCommandWithMultiThread
+extends CommandWithDestination {
+
+  private int threadCount = 1;
+  private ThreadPoolExecutor executor = null;
+  private int threadPoolQueueSize = DEFAULT_QUEUE_SIZE;
+
+  public static final int DEFAULT_QUEUE_SIZE = 1024;
+
+  /**
+   * set thread count by option value, if the value less than 1,
+   * use 1 instead.
+   *
+   * @param optValue option value
+   */
+  protected void setThreadCount(String optValue) {
+if (optValue != null) {
+  threadCount = Math.max(Integer.parseInt(optValue), 1);
+   

[hadoop] branch branch-3.3 updated: HADOOP-17998. Allow get command to run with multi threads. (#3645)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new cbb3ba1  HADOOP-17998. Allow get command to run with multi threads. 
(#3645)
cbb3ba1 is described below

commit cbb3ba135cfa3a9bc9f444e8b4a7875758721a3c
Author: smarthan <1139557...@qq.com>
AuthorDate: Mon Nov 22 19:37:05 2021 +0800

HADOOP-17998. Allow get command to run with multi threads. (#3645)

(cherry picked from commit 63018dc73f4d29632e93be08d035ab9a7e73531c)

 Conflicts:

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
---
 .../hadoop/fs/shell/CommandWithDestination.java|   8 +-
 .../fs/shell/CopyCommandWithMultiThread.java   | 155 +
 .../org/apache/hadoop/fs/shell/CopyCommands.java   | 147 ---
 .../src/site/markdown/FileSystemShell.md   |  36 +++--
 .../apache/hadoop/fs/shell/TestCopyFromLocal.java  |  95 ++---
 .../hadoop/fs/shell/TestCopyPreserveFlag.java  |  43 +-
 ...TestCopyFromLocal.java => TestCopyToLocal.java} | 152 
 .../hadoop-common/src/test/resources/testConf.xml  |  90 ++--
 8 files changed, 429 insertions(+), 297 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
index 90a709d..f6f4247 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
@@ -397,11 +397,11 @@ abstract class CommandWithDestination extends FsCommand {
 
   /**
* If direct write is disabled ,copies the stream contents to a temporary
-   * file "._COPYING_". If the copy is
-   * successful, the temporary file will be renamed to the real path,
-   * else the temporary file will be deleted.
+   * file "target._COPYING_". If the copy is successful, the temporary file
+   * will be renamed to the real path, else the temporary file will be deleted.
* if direct write is enabled , then creation temporary file is skipped.
-   * @param in the input stream for the copy
+   *
+   * @param in the input stream for the copy
* @param target where to store the contents of the stream
* @throws IOException if copy fails
*/ 
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
new file mode 100644
index 000..aed4030
--- /dev/null
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.shell;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+
+/**
+ * Abstract command to enable sub copy commands run with multi-thread.
+ */
+public abstract class CopyCommandWithMultiThread
+extends CommandWithDestination {
+
+  private int threadCount = 1;
+  private ThreadPoolExecutor executor = null;
+  private int threadPoolQueueSize = DEFAULT_QUEUE_SIZE;
+
+  public static final int DEFAULT_QUEUE_SIZE = 1024;
+
+  /**
+   * set thread count by option value, if the value less than 1,
+   * use 1 instead.
+   *
+   * @param optValue option value
+   */
+  protected void setThreadCount(String optValue) {
+if (optValue != null) {
+  threadCount = Math.max(Integer.parseInt(optValue), 1);
+}
+  }
+
+  /**
+   * set thread pool queue size by option value, if the value 

[hadoop] branch trunk updated: HADOOP-17998. Allow get command to run with multi threads. (#3645)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 63018dc  HADOOP-17998. Allow get command to run with multi threads. 
(#3645)
63018dc is described below

commit 63018dc73f4d29632e93be08d035ab9a7e73531c
Author: smarthan <1139557...@qq.com>
AuthorDate: Mon Nov 22 19:37:05 2021 +0800

HADOOP-17998. Allow get command to run with multi threads. (#3645)
---
 .../hadoop/fs/shell/CommandWithDestination.java|   8 +-
 .../fs/shell/CopyCommandWithMultiThread.java   | 155 +
 .../org/apache/hadoop/fs/shell/CopyCommands.java   | 147 ---
 .../src/site/markdown/FileSystemShell.md   |  36 +++--
 .../apache/hadoop/fs/shell/TestCopyFromLocal.java  |  95 ++---
 .../hadoop/fs/shell/TestCopyPreserveFlag.java  |  43 +-
 ...TestCopyFromLocal.java => TestCopyToLocal.java} | 152 
 .../hadoop-common/src/test/resources/testConf.xml  |  90 ++--
 8 files changed, 429 insertions(+), 297 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
index 90a709d..f6f4247 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
@@ -397,11 +397,11 @@ abstract class CommandWithDestination extends FsCommand {
 
   /**
* If direct write is disabled ,copies the stream contents to a temporary
-   * file "._COPYING_". If the copy is
-   * successful, the temporary file will be renamed to the real path,
-   * else the temporary file will be deleted.
+   * file "target._COPYING_". If the copy is successful, the temporary file
+   * will be renamed to the real path, else the temporary file will be deleted.
* if direct write is enabled , then creation temporary file is skipped.
-   * @param in the input stream for the copy
+   *
+   * @param in the input stream for the copy
* @param target where to store the contents of the stream
* @throws IOException if copy fails
*/ 
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
new file mode 100644
index 000..aed4030
--- /dev/null
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.shell;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+
+/**
+ * Abstract command to enable sub copy commands run with multi-thread.
+ */
+public abstract class CopyCommandWithMultiThread
+extends CommandWithDestination {
+
+  private int threadCount = 1;
+  private ThreadPoolExecutor executor = null;
+  private int threadPoolQueueSize = DEFAULT_QUEUE_SIZE;
+
+  public static final int DEFAULT_QUEUE_SIZE = 1024;
+
+  /**
+   * set thread count by option value, if the value less than 1,
+   * use 1 instead.
+   *
+   * @param optValue option value
+   */
+  protected void setThreadCount(String optValue) {
+if (optValue != null) {
+  threadCount = Math.max(Integer.parseInt(optValue), 1);
+}
+  }
+
+  /**
+   * set thread pool queue size by option value, if the value less than 1,
+   * use DEFAULT_QUEUE_SIZE instead.
+   *
+   * @param optValue option value
+   */
+  protected void setThreadPoolQueueSize(String optValue) {
+if (optValue != null) {
+  int size = Integer.parseInt

[hadoop] branch branch-3.2.3 updated: HDFS-16286. Add a debug tool to verify the correctness of erasure coding on file (#3593)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2.3 by this push:
 new cbf4a33  HDFS-16286. Add a debug tool to verify the correctness of 
erasure coding on file (#3593)
cbf4a33 is described below

commit cbf4a33549ce04be729a31351e9d34d21f4528c7
Author: daimin 
AuthorDate: Thu Nov 4 04:19:56 2021 +0800

HDFS-16286. Add a debug tool to verify the correctness of erasure coding on 
file (#3593)

(cherry picked from commit a21895a5b3644944fe04cf558d593b96da0263fd)

 Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java

(cherry picked from commit 29fd36e2f1e3b34b3cb4c047d93bf88a70dc8149)
---
 .../org/apache/hadoop/hdfs/tools/DebugAdmin.java   | 231 +
 .../hadoop-hdfs/src/site/markdown/HDFSCommands.md  |  10 +
 .../apache/hadoop/hdfs/tools/TestDebugAdmin.java   | 118 ++-
 3 files changed, 351 insertions(+), 8 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
index 642bfab..4e7b747 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
@@ -25,15 +25,42 @@ import java.io.FileOutputStream;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.hdfs.BlockReader;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.client.impl.BlockReaderRemote;
+import org.apache.hadoop.hdfs.net.Peer;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -68,6 +95,7 @@ public class DebugAdmin extends Configured implements Tool {
   new VerifyMetaCommand(),
   new ComputeMetaCommand(),
   new RecoverLeaseCommand(),
+  new VerifyECCommand(),
   new HelpCommand()
   };
 
@@ -387,6 +415,209 @@ public class DebugAdmin extends Configured implements 
Tool {
   }
 
   /**
+   * The command for verifying the correctness of erasure coding on an erasure 
coded file.
+   */
+  private class VerifyECCommand extends DebugCommand {
+private DFSClient client;
+private int dataBlkNum;
+private int parityBlkNum;
+private int cellSize;
+private boolean useDNHostname;
+private CachingStrategy cachingStrategy;
+private int stripedReadBufferSize;
+private CompletionService readService;
+private RawErasureEncoder encoder;
+private BlockReader[] blockReaders;
+
+
+VerifyECCommand() {
+  super("verifyEC",
+  "verifyEC -file ",
+  "  Verify HDFS erasure coding on all block groups of the file.");
+}
+
+int run(List args) throws IOException {
+  if (args.size() < 2) {
+System.out.println(usageText);
+System.out.println(helpText + System.lineSeparator());
+return 1;
+  }
+  String file = StringUtils.popOptionWithArgument("-file", args);
+  Path path = new Path(file);
+  DistributedFileSystem dfs = AdminHelper.getDFS(get

[hadoop] branch branch-3.2 updated: HDFS-16286. Add a debug tool to verify the correctness of erasure coding on file (#3593)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new 29fd36e  HDFS-16286. Add a debug tool to verify the correctness of 
erasure coding on file (#3593)
29fd36e is described below

commit 29fd36e2f1e3b34b3cb4c047d93bf88a70dc8149
Author: daimin 
AuthorDate: Thu Nov 4 04:19:56 2021 +0800

HDFS-16286. Add a debug tool to verify the correctness of erasure coding on 
file (#3593)

(cherry picked from commit a21895a5b3644944fe04cf558d593b96da0263fd)

 Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
---
 .../org/apache/hadoop/hdfs/tools/DebugAdmin.java   | 231 +
 .../hadoop-hdfs/src/site/markdown/HDFSCommands.md  |  10 +
 .../apache/hadoop/hdfs/tools/TestDebugAdmin.java   | 118 ++-
 3 files changed, 351 insertions(+), 8 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
index 642bfab..4e7b747 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
@@ -25,15 +25,42 @@ import java.io.FileOutputStream;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.hdfs.BlockReader;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.client.impl.BlockReaderRemote;
+import org.apache.hadoop.hdfs.net.Peer;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -68,6 +95,7 @@ public class DebugAdmin extends Configured implements Tool {
   new VerifyMetaCommand(),
   new ComputeMetaCommand(),
   new RecoverLeaseCommand(),
+  new VerifyECCommand(),
   new HelpCommand()
   };
 
@@ -387,6 +415,209 @@ public class DebugAdmin extends Configured implements 
Tool {
   }
 
   /**
+   * The command for verifying the correctness of erasure coding on an erasure 
coded file.
+   */
+  private class VerifyECCommand extends DebugCommand {
+private DFSClient client;
+private int dataBlkNum;
+private int parityBlkNum;
+private int cellSize;
+private boolean useDNHostname;
+private CachingStrategy cachingStrategy;
+private int stripedReadBufferSize;
+private CompletionService readService;
+private RawErasureEncoder encoder;
+private BlockReader[] blockReaders;
+
+
+VerifyECCommand() {
+  super("verifyEC",
+  "verifyEC -file ",
+  "  Verify HDFS erasure coding on all block groups of the file.");
+}
+
+int run(List args) throws IOException {
+  if (args.size() < 2) {
+System.out.println(usageText);
+System.out.println(helpText + System.lineSeparator());
+return 1;
+  }
+  String file = StringUtils.popOptionWithArgument("-file", args);
+  Path path = new Path(file);
+  DistributedFileSystem dfs = AdminHelper.getDFS(getConf());
+  this.client = dfs.getClient();
+
+  FileStatus fileStatus;
+  try {
+

[hadoop] branch branch-3.3 updated: HDFS-16286. Add a debug tool to verify the correctness of erasure coding on file (#3593)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 2844b98  HDFS-16286. Add a debug tool to verify the correctness of 
erasure coding on file (#3593)
2844b98 is described below

commit 2844b9855876a5caeb84160487431d730a9eb661
Author: daimin 
AuthorDate: Thu Nov 4 04:19:56 2021 +0800

HDFS-16286. Add a debug tool to verify the correctness of erasure coding on 
file (#3593)

(cherry picked from commit a21895a5b3644944fe04cf558d593b96da0263fd)
---
 .../org/apache/hadoop/hdfs/tools/DebugAdmin.java   | 230 +
 .../hadoop-hdfs/src/site/markdown/HDFSCommands.md  |  10 +
 .../apache/hadoop/hdfs/tools/TestDebugAdmin.java   | 118 ++-
 3 files changed, 350 insertions(+), 8 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
index efe355d..32e8248 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
@@ -24,15 +24,41 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.file.Files;
+import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.hdfs.BlockReader;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.client.impl.BlockReaderRemote;
+import org.apache.hadoop.hdfs.net.Peer;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.token.Token;
 import 
org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Uninterruptibles;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -69,6 +95,7 @@ public class DebugAdmin extends Configured implements Tool {
   new VerifyMetaCommand(),
   new ComputeMetaCommand(),
   new RecoverLeaseCommand(),
+  new VerifyECCommand(),
   new HelpCommand()
   };
 
@@ -388,6 +415,209 @@ public class DebugAdmin extends Configured implements 
Tool {
   }
 
   /**
+   * The command for verifying the correctness of erasure coding on an erasure 
coded file.
+   */
+  private class VerifyECCommand extends DebugCommand {
+private DFSClient client;
+private int dataBlkNum;
+private int parityBlkNum;
+private int cellSize;
+private boolean useDNHostname;
+private CachingStrategy cachingStrategy;
+private int stripedReadBufferSize;
+private CompletionService readService;
+private RawErasureEncoder encoder;
+private BlockReader[] blockReaders;
+
+
+VerifyECCommand() {
+  super("verifyEC",
+  "verifyEC -file ",
+  "  Verify HDFS erasure coding on all block groups of the file.");
+}
+
+int run(List args) throws IOException {
+  if (args.size() < 2) {
+System.out.println(usageText);
+System.out.println(helpText + System.lineSeparator());
+return 1;
+  }
+  String file = StringUtils.popOptionWithArgument("-file", args);
+  Path path = new Path(file);
+  DistributedFileSystem dfs = AdminHelper.getDFS(getConf());
+  this.client = dfs.getClient();
+
+  FileStatus fileStatus;
+  try {
+fileStatus = dfs.getFileStatus(path);
+  } catch (FileNotFoundException e) {
+System.err.println("File &

[hadoop] branch trunk updated: HDFS-16286. Add a debug tool to verify the correctness of erasure coding on file (#3593)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new a21895a  HDFS-16286. Add a debug tool to verify the correctness of 
erasure coding on file (#3593)
a21895a is described below

commit a21895a5b3644944fe04cf558d593b96da0263fd
Author: daimin 
AuthorDate: Thu Nov 4 04:19:56 2021 +0800

HDFS-16286. Add a debug tool to verify the correctness of erasure coding on 
file (#3593)
---
 .../org/apache/hadoop/hdfs/tools/DebugAdmin.java   | 230 +
 .../hadoop-hdfs/src/site/markdown/HDFSCommands.md  |  10 +
 .../apache/hadoop/hdfs/tools/TestDebugAdmin.java   | 118 ++-
 3 files changed, 350 insertions(+), 8 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
index efe355d..32e8248 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
@@ -24,15 +24,41 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.file.Files;
+import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.hdfs.BlockReader;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.client.impl.BlockReaderRemote;
+import org.apache.hadoop.hdfs.net.Peer;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.token.Token;
 import 
org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Uninterruptibles;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -69,6 +95,7 @@ public class DebugAdmin extends Configured implements Tool {
   new VerifyMetaCommand(),
   new ComputeMetaCommand(),
   new RecoverLeaseCommand(),
+  new VerifyECCommand(),
   new HelpCommand()
   };
 
@@ -388,6 +415,209 @@ public class DebugAdmin extends Configured implements 
Tool {
   }
 
   /**
+   * The command for verifying the correctness of erasure coding on an erasure 
coded file.
+   */
+  private class VerifyECCommand extends DebugCommand {
+private DFSClient client;
+private int dataBlkNum;
+private int parityBlkNum;
+private int cellSize;
+private boolean useDNHostname;
+private CachingStrategy cachingStrategy;
+private int stripedReadBufferSize;
+private CompletionService readService;
+private RawErasureEncoder encoder;
+private BlockReader[] blockReaders;
+
+
+VerifyECCommand() {
+  super("verifyEC",
+  "verifyEC -file ",
+  "  Verify HDFS erasure coding on all block groups of the file.");
+}
+
+int run(List args) throws IOException {
+  if (args.size() < 2) {
+System.out.println(usageText);
+System.out.println(helpText + System.lineSeparator());
+return 1;
+  }
+  String file = StringUtils.popOptionWithArgument("-file", args);
+  Path path = new Path(file);
+  DistributedFileSystem dfs = AdminHelper.getDFS(getConf());
+  this.client = dfs.getClient();
+
+  FileStatus fileStatus;
+  try {
+fileStatus = dfs.getFileStatus(path);
+  } catch (FileNotFoundException e) {
+System.err.println("File " + file + " does not exist.");
+return 1;
+  }
+
+  if (!fileStatus.isFile()) {
+

[hadoop] branch branch-3.3 updated: HDFS-16259. Catch and re-throw sub-classes of AccessControlException thrown by any permission provider plugins (eg Ranger) (#3598)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 9cf841b  HDFS-16259. Catch and re-throw sub-classes of 
AccessControlException thrown by any permission provider plugins (eg Ranger) 
(#3598)
9cf841b is described below

commit 9cf841b1a6444efe116b90a6751cd9fe8fe20764
Author: Stephen O'Donnell 
AuthorDate: Tue Nov 2 11:14:48 2021 +

HDFS-16259. Catch and re-throw sub-classes of AccessControlException thrown 
by any permission provider plugins (eg Ranger) (#3598)

(cherry picked from commit 2f35cc36cdbb5a54afa1545388790496c936b954)

 Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
---
 .../hdfs/server/namenode/FSPermissionChecker.java  | 60 +-
 .../namenode/TestINodeAttributeProvider.java   | 55 
 2 files changed, 90 insertions(+), 25 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
index 92e5858..5d136cb 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
@@ -216,31 +216,41 @@ public class FSPermissionChecker implements 
AccessControlEnforcer {
 AccessControlEnforcer enforcer = getAccessControlEnforcer();
 
 String opType = operationType.get();
-if (this.authorizeWithContext && opType != null) {
-  INodeAttributeProvider.AuthorizationContext.Builder builder =
-  new INodeAttributeProvider.AuthorizationContext.Builder();
-  builder.fsOwner(fsOwner).
-  supergroup(supergroup).
-  callerUgi(callerUgi).
-  inodeAttrs(inodeAttrs).
-  inodes(inodes).
-  pathByNameArr(components).
-  snapshotId(snapshotId).
-  path(path).
-  ancestorIndex(ancestorIndex).
-  doCheckOwner(doCheckOwner).
-  ancestorAccess(ancestorAccess).
-  parentAccess(parentAccess).
-  access(access).
-  subAccess(subAccess).
-  ignoreEmptyDir(ignoreEmptyDir).
-  operationName(opType).
-  callerContext(CallerContext.getCurrent());
-  enforcer.checkPermissionWithContext(builder.build());
-} else {
-  enforcer.checkPermission(fsOwner, supergroup, callerUgi, inodeAttrs,
-  inodes, components, snapshotId, path, ancestorIndex, doCheckOwner,
-  ancestorAccess, parentAccess, access, subAccess, ignoreEmptyDir);
+try {
+  if (this.authorizeWithContext && opType != null) {
+INodeAttributeProvider.AuthorizationContext.Builder builder =
+new INodeAttributeProvider.AuthorizationContext.Builder();
+builder.fsOwner(fsOwner).
+supergroup(supergroup).
+callerUgi(callerUgi).
+inodeAttrs(inodeAttrs).
+inodes(inodes).
+pathByNameArr(components).
+snapshotId(snapshotId).
+path(path).
+ancestorIndex(ancestorIndex).
+doCheckOwner(doCheckOwner).
+ancestorAccess(ancestorAccess).
+parentAccess(parentAccess).
+access(access).
+subAccess(subAccess).
+ignoreEmptyDir(ignoreEmptyDir).
+operationName(opType).
+callerContext(CallerContext.getCurrent());
+enforcer.checkPermissionWithContext(builder.build());
+  } else {
+enforcer.checkPermission(fsOwner, supergroup, callerUgi, inodeAttrs,
+inodes, components, snapshotId, path, ancestorIndex, doCheckOwner,
+ancestorAccess, parentAccess, access, subAccess, ignoreEmptyDir);
+  }
+} catch (AccessControlException ace) {
+  Class exceptionClass = ace.getClass();
+  if (exceptionClass.equals(AccessControlException.class)
+  || exceptionClass.equals(TraverseAccessControlException.class)) {
+throw ace;
+  }
+  // Only form a new ACE for subclasses which come from external enforcers
+  throw new AccessControlException(ace);
 }
 
   }
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
index 699ac17..c6f5417 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test

[hadoop] branch trunk updated: HDFS-16259. Catch and re-throw sub-classes of AccessControlException thrown by any permission provider plugins (eg Ranger) (#3598)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 2f35cc3  HDFS-16259. Catch and re-throw sub-classes of 
AccessControlException thrown by any permission provider plugins (eg Ranger) 
(#3598)
2f35cc3 is described below

commit 2f35cc36cdbb5a54afa1545388790496c936b954
Author: Stephen O'Donnell 
AuthorDate: Tue Nov 2 11:14:48 2021 +

HDFS-16259. Catch and re-throw sub-classes of AccessControlException thrown 
by any permission provider plugins (eg Ranger) (#3598)
---
 .../hdfs/server/namenode/FSPermissionChecker.java  | 60 +-
 .../namenode/TestINodeAttributeProvider.java   | 56 
 2 files changed, 91 insertions(+), 25 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
index ffc8ee9..c7430e3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
@@ -273,31 +273,41 @@ public class FSPermissionChecker implements 
AccessControlEnforcer {
 AccessControlEnforcer enforcer = getAccessControlEnforcer();
 
 String opType = operationType.get();
-if (this.authorizeWithContext && opType != null) {
-  INodeAttributeProvider.AuthorizationContext.Builder builder =
-  new INodeAttributeProvider.AuthorizationContext.Builder();
-  builder.fsOwner(fsOwner).
-  supergroup(supergroup).
-  callerUgi(callerUgi).
-  inodeAttrs(inodeAttrs).
-  inodes(inodes).
-  pathByNameArr(components).
-  snapshotId(snapshotId).
-  path(path).
-  ancestorIndex(ancestorIndex).
-  doCheckOwner(doCheckOwner).
-  ancestorAccess(ancestorAccess).
-  parentAccess(parentAccess).
-  access(access).
-  subAccess(subAccess).
-  ignoreEmptyDir(ignoreEmptyDir).
-  operationName(opType).
-  callerContext(CallerContext.getCurrent());
-  enforcer.checkPermissionWithContext(builder.build());
-} else {
-  enforcer.checkPermission(fsOwner, supergroup, callerUgi, inodeAttrs,
-  inodes, components, snapshotId, path, ancestorIndex, doCheckOwner,
-  ancestorAccess, parentAccess, access, subAccess, ignoreEmptyDir);
+try {
+  if (this.authorizeWithContext && opType != null) {
+INodeAttributeProvider.AuthorizationContext.Builder builder =
+new INodeAttributeProvider.AuthorizationContext.Builder();
+builder.fsOwner(fsOwner).
+supergroup(supergroup).
+callerUgi(callerUgi).
+inodeAttrs(inodeAttrs).
+inodes(inodes).
+pathByNameArr(components).
+snapshotId(snapshotId).
+path(path).
+ancestorIndex(ancestorIndex).
+doCheckOwner(doCheckOwner).
+ancestorAccess(ancestorAccess).
+parentAccess(parentAccess).
+access(access).
+subAccess(subAccess).
+ignoreEmptyDir(ignoreEmptyDir).
+operationName(opType).
+callerContext(CallerContext.getCurrent());
+enforcer.checkPermissionWithContext(builder.build());
+  } else {
+enforcer.checkPermission(fsOwner, supergroup, callerUgi, inodeAttrs,
+inodes, components, snapshotId, path, ancestorIndex, doCheckOwner,
+ancestorAccess, parentAccess, access, subAccess, ignoreEmptyDir);
+  }
+} catch (AccessControlException ace) {
+  Class exceptionClass = ace.getClass();
+  if (exceptionClass.equals(AccessControlException.class)
+  || exceptionClass.equals(TraverseAccessControlException.class)) {
+throw ace;
+  }
+  // Only form a new ACE for subclasses which come from external enforcers
+  throw new AccessControlException(ace);
 }
 
   }
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
index 776a198..700b32f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
@@ -48,6 +48,8 @@ import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.junit.Assert.fail;

[hadoop] branch branch-3.2.3 updated: HDFS-16272. Fix int overflow in computing safe length during EC block recovery (#3548)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2.3 by this push:
 new e1f11d5  HDFS-16272. Fix int overflow in computing safe length during 
EC block recovery (#3548)
e1f11d5 is described below

commit e1f11d5f63ac69328055b971e6b80fc9797561a8
Author: daimin 
AuthorDate: Mon Oct 18 18:55:55 2021 +0800

HDFS-16272. Fix int overflow in computing safe length during EC block 
recovery (#3548)

(cherry picked from commit 5337bebcc53a0144311877362787ed9ece0cb118)
---
 .../org/apache/hadoop/hdfs/util/StripedBlockUtil.java  |  7 +++
 .../apache/hadoop/hdfs/TestLeaseRecoveryStriped.java   | 18 ++
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
index 718c51d..4122636 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
@@ -245,8 +245,7 @@ public class StripedBlockUtil {
 Arrays.sort(cpy);
 // full stripe is a stripe has at least dataBlkNum full cells.
 // lastFullStripeIdx is the index of the last full stripe.
-int lastFullStripeIdx =
-(int) (cpy[cpy.length - dataBlkNum] / cellSize);
+long lastFullStripeIdx = cpy[cpy.length - dataBlkNum] / cellSize;
 return lastFullStripeIdx * stripeSize; // return the safeLength
 // TODO: Include lastFullStripeIdx+1 stripe in safeLength, if there exists
 // such a stripe (and it must be partial).
@@ -271,9 +270,9 @@ public class StripedBlockUtil {
*/
   public static long offsetInBlkToOffsetInBG(int cellSize, int dataBlkNum,
   long offsetInBlk, int idxInBlockGroup) {
-int cellIdxInBlk = (int) (offsetInBlk / cellSize);
+long cellIdxInBlk = offsetInBlk / cellSize;
 return cellIdxInBlk * cellSize * dataBlkNum // n full stripes before offset
-+ idxInBlockGroup * cellSize // m full cells before offset
++ (long)idxInBlockGroup * cellSize // m full cells before offset
 + offsetInBlk % cellSize; // partial cell
   }
 
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
index 36a3055..2193932 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
@@ -188,6 +188,24 @@ public class TestLeaseRecoveryStriped {
 }
   }
 
+  @Test
+  public void testSafeLength() {
+checkSafeLength(0, 0); // Length of: 0
+checkSafeLength(1024 * 1024, 6291456L); // Length of: 1 MiB
+checkSafeLength(64 * 1024 * 1024, 402653184L); // Length of: 64 MiB
+checkSafeLength(189729792, 1132462080L); // Length of: 189729792
+checkSafeLength(256 * 1024 * 1024, 1610612736L); // Length of: 256 MiB
+checkSafeLength(517399040, 3101687808L); // Length of: 517399040
+checkSafeLength(1024 * 1024 * 1024, 6442450944L); // Length of: 1 GiB
+  }
+
+  private void checkSafeLength(int blockLength, long expectedSafeLength) {
+int[] blockLengths = new int[]{blockLength, blockLength, blockLength, 
blockLength,
+blockLength, blockLength};
+long safeLength = new BlockLengths(ecPolicy, blockLengths).getSafeLength();
+Assert.assertEquals(expectedSafeLength, safeLength);
+  }
+
   private void runTest(int[] blockLengths, long safeLength) throws Exception {
 writePartialBlocks(blockLengths);
 

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.2 updated: HDFS-16272. Fix int overflow in computing safe length during EC block recovery (#3548)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new 9f2c89e  HDFS-16272. Fix int overflow in computing safe length during 
EC block recovery (#3548)
9f2c89e is described below

commit 9f2c89e690e9b3e839cfdbb167ec8071d2145f62
Author: daimin 
AuthorDate: Mon Oct 18 18:55:55 2021 +0800

HDFS-16272. Fix int overflow in computing safe length during EC block 
recovery (#3548)

(cherry picked from commit 5337bebcc53a0144311877362787ed9ece0cb118)
---
 .../org/apache/hadoop/hdfs/util/StripedBlockUtil.java  |  7 +++
 .../apache/hadoop/hdfs/TestLeaseRecoveryStriped.java   | 18 ++
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
index 718c51d..4122636 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
@@ -245,8 +245,7 @@ public class StripedBlockUtil {
 Arrays.sort(cpy);
 // full stripe is a stripe has at least dataBlkNum full cells.
 // lastFullStripeIdx is the index of the last full stripe.
-int lastFullStripeIdx =
-(int) (cpy[cpy.length - dataBlkNum] / cellSize);
+long lastFullStripeIdx = cpy[cpy.length - dataBlkNum] / cellSize;
 return lastFullStripeIdx * stripeSize; // return the safeLength
 // TODO: Include lastFullStripeIdx+1 stripe in safeLength, if there exists
 // such a stripe (and it must be partial).
@@ -271,9 +270,9 @@ public class StripedBlockUtil {
*/
   public static long offsetInBlkToOffsetInBG(int cellSize, int dataBlkNum,
   long offsetInBlk, int idxInBlockGroup) {
-int cellIdxInBlk = (int) (offsetInBlk / cellSize);
+long cellIdxInBlk = offsetInBlk / cellSize;
 return cellIdxInBlk * cellSize * dataBlkNum // n full stripes before offset
-+ idxInBlockGroup * cellSize // m full cells before offset
++ (long)idxInBlockGroup * cellSize // m full cells before offset
 + offsetInBlk % cellSize; // partial cell
   }
 
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
index 36a3055..2193932 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
@@ -188,6 +188,24 @@ public class TestLeaseRecoveryStriped {
 }
   }
 
+  @Test
+  public void testSafeLength() {
+checkSafeLength(0, 0); // Length of: 0
+checkSafeLength(1024 * 1024, 6291456L); // Length of: 1 MiB
+checkSafeLength(64 * 1024 * 1024, 402653184L); // Length of: 64 MiB
+checkSafeLength(189729792, 1132462080L); // Length of: 189729792
+checkSafeLength(256 * 1024 * 1024, 1610612736L); // Length of: 256 MiB
+checkSafeLength(517399040, 3101687808L); // Length of: 517399040
+checkSafeLength(1024 * 1024 * 1024, 6442450944L); // Length of: 1 GiB
+  }
+
+  private void checkSafeLength(int blockLength, long expectedSafeLength) {
+int[] blockLengths = new int[]{blockLength, blockLength, blockLength, 
blockLength,
+blockLength, blockLength};
+long safeLength = new BlockLengths(ecPolicy, blockLengths).getSafeLength();
+Assert.assertEquals(expectedSafeLength, safeLength);
+  }
+
   private void runTest(int[] blockLengths, long safeLength) throws Exception {
 writePartialBlocks(blockLengths);
 

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.3 updated: HDFS-16272. Fix int overflow in computing safe length during EC block recovery (#3548)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 6019239  HDFS-16272. Fix int overflow in computing safe length during 
EC block recovery (#3548)
6019239 is described below

commit 601923982522c235c67a4ea8facb6d23a6ae055c
Author: daimin 
AuthorDate: Mon Oct 18 18:55:55 2021 +0800

HDFS-16272. Fix int overflow in computing safe length during EC block 
recovery (#3548)

(cherry picked from commit 5337bebcc53a0144311877362787ed9ece0cb118)
---
 .../org/apache/hadoop/hdfs/util/StripedBlockUtil.java  |  7 +++
 .../apache/hadoop/hdfs/TestLeaseRecoveryStriped.java   | 18 ++
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
index 460c611..28334fe 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
@@ -245,8 +245,7 @@ public class StripedBlockUtil {
 Arrays.sort(cpy);
 // full stripe is a stripe has at least dataBlkNum full cells.
 // lastFullStripeIdx is the index of the last full stripe.
-int lastFullStripeIdx =
-(int) (cpy[cpy.length - dataBlkNum] / cellSize);
+long lastFullStripeIdx = cpy[cpy.length - dataBlkNum] / cellSize;
 return lastFullStripeIdx * stripeSize; // return the safeLength
 // TODO: Include lastFullStripeIdx+1 stripe in safeLength, if there exists
 // such a stripe (and it must be partial).
@@ -271,9 +270,9 @@ public class StripedBlockUtil {
*/
   public static long offsetInBlkToOffsetInBG(int cellSize, int dataBlkNum,
   long offsetInBlk, int idxInBlockGroup) {
-int cellIdxInBlk = (int) (offsetInBlk / cellSize);
+long cellIdxInBlk = offsetInBlk / cellSize;
 return cellIdxInBlk * cellSize * dataBlkNum // n full stripes before offset
-+ idxInBlockGroup * cellSize // m full cells before offset
++ (long)idxInBlockGroup * cellSize // m full cells before offset
 + offsetInBlk % cellSize; // partial cell
   }
 
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
index d0eccc1..8f42e47 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
@@ -188,6 +188,24 @@ public class TestLeaseRecoveryStriped {
 }
   }
 
+  @Test
+  public void testSafeLength() {
+checkSafeLength(0, 0); // Length of: 0
+checkSafeLength(1024 * 1024, 6291456L); // Length of: 1 MiB
+checkSafeLength(64 * 1024 * 1024, 402653184L); // Length of: 64 MiB
+checkSafeLength(189729792, 1132462080L); // Length of: 189729792
+checkSafeLength(256 * 1024 * 1024, 1610612736L); // Length of: 256 MiB
+checkSafeLength(517399040, 3101687808L); // Length of: 517399040
+checkSafeLength(1024 * 1024 * 1024, 6442450944L); // Length of: 1 GiB
+  }
+
+  private void checkSafeLength(int blockLength, long expectedSafeLength) {
+int[] blockLengths = new int[]{blockLength, blockLength, blockLength, 
blockLength,
+blockLength, blockLength};
+long safeLength = new BlockLengths(ecPolicy, blockLengths).getSafeLength();
+Assert.assertEquals(expectedSafeLength, safeLength);
+  }
+
   private void runTest(int[] blockLengths, long safeLength) throws Exception {
 writePartialBlocks(blockLengths);
 

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: HDFS-16272. Fix int overflow in computing safe length during EC block recovery (#3548)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 5337beb  HDFS-16272. Fix int overflow in computing safe length during 
EC block recovery (#3548)
5337beb is described below

commit 5337bebcc53a0144311877362787ed9ece0cb118
Author: daimin 
AuthorDate: Mon Oct 18 18:55:55 2021 +0800

HDFS-16272. Fix int overflow in computing safe length during EC block 
recovery (#3548)
---
 .../org/apache/hadoop/hdfs/util/StripedBlockUtil.java  |  7 +++
 .../apache/hadoop/hdfs/TestLeaseRecoveryStriped.java   | 18 ++
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
index 540187d..0b4f5fe 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
@@ -245,8 +245,7 @@ public class StripedBlockUtil {
 Arrays.sort(cpy);
 // full stripe is a stripe has at least dataBlkNum full cells.
 // lastFullStripeIdx is the index of the last full stripe.
-int lastFullStripeIdx =
-(int) (cpy[cpy.length - dataBlkNum] / cellSize);
+long lastFullStripeIdx = cpy[cpy.length - dataBlkNum] / cellSize;
 return lastFullStripeIdx * stripeSize; // return the safeLength
 // TODO: Include lastFullStripeIdx+1 stripe in safeLength, if there exists
 // such a stripe (and it must be partial).
@@ -271,9 +270,9 @@ public class StripedBlockUtil {
*/
   public static long offsetInBlkToOffsetInBG(int cellSize, int dataBlkNum,
   long offsetInBlk, int idxInBlockGroup) {
-int cellIdxInBlk = (int) (offsetInBlk / cellSize);
+long cellIdxInBlk = offsetInBlk / cellSize;
 return cellIdxInBlk * cellSize * dataBlkNum // n full stripes before offset
-+ idxInBlockGroup * cellSize // m full cells before offset
++ (long)idxInBlockGroup * cellSize // m full cells before offset
 + offsetInBlk % cellSize; // partial cell
   }
 
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
index 5aa1f63..728d17f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
@@ -188,6 +188,24 @@ public class TestLeaseRecoveryStriped {
 }
   }
 
+  @Test
+  public void testSafeLength() {
+checkSafeLength(0, 0); // Length of: 0
+checkSafeLength(1024 * 1024, 6291456L); // Length of: 1 MiB
+checkSafeLength(64 * 1024 * 1024, 402653184L); // Length of: 64 MiB
+checkSafeLength(189729792, 1132462080L); // Length of: 189729792
+checkSafeLength(256 * 1024 * 1024, 1610612736L); // Length of: 256 MiB
+checkSafeLength(517399040, 3101687808L); // Length of: 517399040
+checkSafeLength(1024 * 1024 * 1024, 6442450944L); // Length of: 1 GiB
+  }
+
+  private void checkSafeLength(int blockLength, long expectedSafeLength) {
+int[] blockLengths = new int[]{blockLength, blockLength, blockLength, 
blockLength,
+blockLength, blockLength};
+long safeLength = new BlockLengths(ecPolicy, blockLengths).getSafeLength();
+Assert.assertEquals(expectedSafeLength, safeLength);
+  }
+
   private void runTest(int[] blockLengths, long safeLength) throws Exception {
 writePartialBlocks(blockLengths);
 

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.3 updated: HDFS-16252. Correct docs for dfs.http.client.retry.policy.spec. Contributed by Stephen O'Donnell

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new f437b3b  HDFS-16252. Correct docs for 
dfs.http.client.retry.policy.spec. Contributed by Stephen O'Donnell
f437b3b is described below

commit f437b3bdafef124c51634e021e84f551772b3039
Author: S O'Donnell 
AuthorDate: Wed Oct 6 14:49:16 2021 +0100

HDFS-16252. Correct docs for dfs.http.client.retry.policy.spec. Contributed 
by Stephen O'Donnell

(cherry picked from commit a30231ff8e3f1e76009e28d8f8322ffbdeecd061)
---
 .../hadoop-hdfs/src/main/resources/hdfs-default.xml| 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 982f182..632f28e 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -3907,7 +3907,7 @@
   1,6,6,10
   
 Specify a policy of multiple linear random retry for WebHDFS client,
-e.g. given pairs of number of retries and sleep time (n0, t0), (n1, t1),
+e.g. given pairs of sleep time and number of retries (t0, n0), (t1, n1),
 ..., the first n0 retries sleep t0 milliseconds on average,
 the following n1 retries sleep t1 milliseconds on average, and so on.
   
@@ -4356,7 +4356,10 @@
   dfs.client.retry.policy.spec
   1,6,6,10
   
-Set to pairs of timeouts and retries for DFSClient.
+Specify a policy of multiple linear random retry for the DFS client,
+e.g. given pairs of sleep time and number of retries (t0, n0), (t1, n1),
+..., the first n0 retries sleep t0 milliseconds on average,
+the following n1 retries sleep t1 milliseconds on average, and so on.
   
 
 

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: HDFS-16252. Correct docs for dfs.http.client.retry.policy.spec. Contributed by Stephen O'Donnell

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new a30231f  HDFS-16252. Correct docs for 
dfs.http.client.retry.policy.spec. Contributed by Stephen O'Donnell
a30231f is described below

commit a30231ff8e3f1e76009e28d8f8322ffbdeecd061
Author: S O'Donnell 
AuthorDate: Wed Oct 6 14:49:16 2021 +0100

HDFS-16252. Correct docs for dfs.http.client.retry.policy.spec. Contributed 
by Stephen O'Donnell
---
 .../hadoop-hdfs/src/main/resources/hdfs-default.xml| 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 16eddff..a49137b 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4005,7 +4005,7 @@
   1,6,6,10
   
 Specify a policy of multiple linear random retry for WebHDFS client,
-e.g. given pairs of number of retries and sleep time (n0, t0), (n1, t1),
+e.g. given pairs of sleep time and number of retries (t0, n0), (t1, n1),
 ..., the first n0 retries sleep t0 milliseconds on average,
 the following n1 retries sleep t1 milliseconds on average, and so on.
   
@@ -4454,7 +4454,10 @@
   dfs.client.retry.policy.spec
   1,6,6,10
   
-Set to pairs of timeouts and retries for DFSClient.
+Specify a policy of multiple linear random retry for the DFS client,
+e.g. given pairs of sleep time and number of retries (t0, n0), (t1, n1),
+..., the first n0 retries sleep t0 milliseconds on average,
+the following n1 retries sleep t1 milliseconds on average, and so on.
   
 
 

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.2 updated: HDFS-16153. Avoid evaluation of LOG.debug statement in QuorumJournalManager (#3269). Contributed by wangzhaohui.

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new 6b10154  HDFS-16153. Avoid evaluation of LOG.debug statement in 
QuorumJournalManager (#3269). Contributed by wangzhaohui.
6b10154 is described below

commit 6b10154123bad3d05f35caeb0727fe7cfe991ea5
Author: wangzhaohui <32935220+wzhallri...@users.noreply.github.com>
AuthorDate: Fri Aug 6 16:27:05 2021 +0800

HDFS-16153. Avoid evaluation of LOG.debug statement in QuorumJournalManager 
(#3269). Contributed by wangzhaohui.

(cherry picked from commit a73b64f86b733177cf9b8b10088527e34099d73f)
---
 .../apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java | 9 +
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
index 9043f74..290f998 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
@@ -586,7 +586,7 @@ public class QuorumJournalManager implements JournalManager 
{
 responseCounts.get(responseCounts.size() - loggers.getMajoritySize());
 if (maxAllowedTxns == 0) {
   LOG.debug("No new edits available in logs; requested starting from " +
-  "ID " + fromTxnId);
+  "ID {}", fromTxnId);
   return;
 }
 LogAction logAction = selectInputStreamLogHelper.record(fromTxnId);
@@ -618,9 +618,10 @@ public class QuorumJournalManager implements 
JournalManager {
 Map resps =
 loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs,
 "selectStreamingInputStreams");
-
-LOG.debug("selectStreamingInputStream manifests:\n" +
-Joiner.on("\n").withKeyValueSeparator(": ").join(resps));
+if (LOG.isDebugEnabled()) {
+  LOG.debug("selectStreamingInputStream manifests:\n {}",
+  Joiner.on("\n").withKeyValueSeparator(": ").join(resps));
+}
 
 final PriorityQueue allStreams =
 new PriorityQueue(64,

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.3 updated: HDFS-16153. Avoid evaluation of LOG.debug statement in QuorumJournalManager (#3269). Contributed by wangzhaohui.

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new e9ba4f4  HDFS-16153. Avoid evaluation of LOG.debug statement in 
QuorumJournalManager (#3269). Contributed by wangzhaohui.
e9ba4f4 is described below

commit e9ba4f4591fdd66b57f711b47ad6c0cca031f34c
Author: wangzhaohui <32935220+wzhallri...@users.noreply.github.com>
AuthorDate: Fri Aug 6 16:27:05 2021 +0800

HDFS-16153. Avoid evaluation of LOG.debug statement in QuorumJournalManager 
(#3269). Contributed by wangzhaohui.

(cherry picked from commit a73b64f86b733177cf9b8b10088527e34099d73f)
---
 .../apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java | 9 +
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
index 354b250..ff7d6e1 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
@@ -586,7 +586,7 @@ public class QuorumJournalManager implements JournalManager 
{
 responseCounts.get(responseCounts.size() - loggers.getMajoritySize());
 if (maxAllowedTxns == 0) {
   LOG.debug("No new edits available in logs; requested starting from " +
-  "ID " + fromTxnId);
+  "ID {}", fromTxnId);
   return;
 }
 LogAction logAction = selectInputStreamLogHelper.record(fromTxnId);
@@ -618,9 +618,10 @@ public class QuorumJournalManager implements 
JournalManager {
 Map resps =
 loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs,
 "selectStreamingInputStreams");
-
-LOG.debug("selectStreamingInputStream manifests:\n" +
-Joiner.on("\n").withKeyValueSeparator(": ").join(resps));
+if (LOG.isDebugEnabled()) {
+  LOG.debug("selectStreamingInputStream manifests:\n {}",
+  Joiner.on("\n").withKeyValueSeparator(": ").join(resps));
+}
 
 final PriorityQueue allStreams =
 new PriorityQueue(64,

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: HDFS-16153. Avoid evaluation of LOG.debug statement in QuorumJournalManager (#3269). Contributed by wangzhaohui.

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new a73b64f  HDFS-16153. Avoid evaluation of LOG.debug statement in 
QuorumJournalManager (#3269). Contributed by wangzhaohui.
a73b64f is described below

commit a73b64f86b733177cf9b8b10088527e34099d73f
Author: wangzhaohui <32935220+wzhallri...@users.noreply.github.com>
AuthorDate: Fri Aug 6 16:27:05 2021 +0800

HDFS-16153. Avoid evaluation of LOG.debug statement in QuorumJournalManager 
(#3269). Contributed by wangzhaohui.
---
 .../apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java | 9 +
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
index fd75003..06a22d6 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
@@ -586,7 +586,7 @@ public class QuorumJournalManager implements JournalManager 
{
 responseCounts.get(responseCounts.size() - loggers.getMajoritySize());
 if (maxAllowedTxns == 0) {
   LOG.debug("No new edits available in logs; requested starting from " +
-  "ID " + fromTxnId);
+  "ID {}", fromTxnId);
   return;
 }
 LogAction logAction = selectInputStreamLogHelper.record(fromTxnId);
@@ -618,9 +618,10 @@ public class QuorumJournalManager implements 
JournalManager {
 Map resps =
 loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs,
 "selectStreamingInputStreams");
-
-LOG.debug("selectStreamingInputStream manifests:\n" +
-Joiner.on("\n").withKeyValueSeparator(": ").join(resps));
+if (LOG.isDebugEnabled()) {
+  LOG.debug("selectStreamingInputStream manifests:\n {}",
+  Joiner.on("\n").withKeyValueSeparator(": ").join(resps));
+}
 
 final PriorityQueue allStreams =
 new PriorityQueue(64,

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.2 updated: HDFS-16055. Quota is not preserved in snapshot INode (#3078)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new fe59fe67 HDFS-16055. Quota is not preserved in snapshot INode (#3078)
fe59fe67 is described below

commit fe59fe67446e92dac74d3d9dec4735ae529dd20f
Author: Siyao Meng <50227127+smen...@users.noreply.github.com>
AuthorDate: Mon Jun 14 10:48:22 2021 -0700

HDFS-16055. Quota is not preserved in snapshot INode (#3078)

(cherry picked from commit ebee2aed00b5589fe610ff994d81feba86039297)
(cherry picked from commit 72508e6430b9fe5145120ac23d3cf714c16e7579)
---
 .../server/namenode/DirectoryWithQuotaFeature.java |  5 
 .../hdfs/server/namenode/snapshot/Snapshot.java| 30 +++---
 .../namenode/snapshot/TestSnapshotDiffReport.java  | 23 ++---
 3 files changed, 45 insertions(+), 13 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java
index ec3015a..509c3ee 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java
@@ -176,6 +176,11 @@ public final class DirectoryWithQuotaFeature implements 
INode.Feature {
 usage.setTypeSpaces(c.getTypeSpaces());
   }
 
+  /** @return the namespace and storagespace and typespace allowed. */
+  public QuotaCounts getSpaceAllowed() {
+return new QuotaCounts.Builder().quotaCount(quota).build();
+  }
+
   /** @return the namespace and storagespace and typespace consumed. */
   public QuotaCounts getSpaceConsumed() {
 return new QuotaCounts.Builder().quotaCount(usage).build();
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java
index f13ef61..710acd9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java
@@ -24,17 +24,18 @@ import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.Date;
-import java.util.stream.Collectors;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.server.namenode.AclFeature;
 import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
+import org.apache.hadoop.hdfs.server.namenode.DirectoryWithQuotaFeature;
 import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
 import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
 import org.apache.hadoop.hdfs.server.namenode.INode;
 import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
+import org.apache.hadoop.hdfs.server.namenode.QuotaCounts;
 import org.apache.hadoop.hdfs.server.namenode.XAttrFeature;
 import org.apache.hadoop.hdfs.util.ReadOnlyList;
 
@@ -145,15 +146,26 @@ public class Snapshot implements Comparable {
   /** The root directory of the snapshot. */
   static public class Root extends INodeDirectory {
 Root(INodeDirectory other) {
-  // Always preserve ACL, XAttr.
-  super(other, false, Arrays.asList(other.getFeatures()).stream().filter(
-  input -> {
-if (AclFeature.class.isInstance(input)
-|| XAttrFeature.class.isInstance(input)) {
-  return true;
+  // Always preserve ACL, XAttr and Quota.
+  super(other, false,
+  Arrays.stream(other.getFeatures()).filter(feature ->
+  feature instanceof AclFeature
+  || feature instanceof XAttrFeature
+  || feature instanceof DirectoryWithQuotaFeature
+  ).map(feature -> {
+if (feature instanceof DirectoryWithQuotaFeature) {
+  // Return copy if feature is quota because a ref could be updated
+  final QuotaCounts quota =
+  ((DirectoryWithQuotaFeature) feature).getSpaceAllowed();
+  return new DirectoryWithQuotaFeature.Builder()
+  .nameSpaceQuota(quota.getNameSpace())
+  .storageSpaceQuota(quota.getStorageSpace())
+  .typeQuotas(quota.getTypeSpaces())
+  .build();
+} else {
+  return feature;
 }
-return false;
- 

[hadoop] branch branch-3.3 updated: HDFS-16055. Quota is not preserved in snapshot INode (#3078)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 72508e6  HDFS-16055. Quota is not preserved in snapshot INode (#3078)
72508e6 is described below

commit 72508e6430b9fe5145120ac23d3cf714c16e7579
Author: Siyao Meng <50227127+smen...@users.noreply.github.com>
AuthorDate: Mon Jun 14 10:48:22 2021 -0700

HDFS-16055. Quota is not preserved in snapshot INode (#3078)

(cherry picked from commit ebee2aed00b5589fe610ff994d81feba86039297)
---
 .../server/namenode/DirectoryWithQuotaFeature.java |  5 
 .../hdfs/server/namenode/snapshot/Snapshot.java| 30 +++---
 .../namenode/snapshot/TestSnapshotDiffReport.java  | 23 ++---
 3 files changed, 45 insertions(+), 13 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java
index 9597c6a..cddefd3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java
@@ -176,6 +176,11 @@ public final class DirectoryWithQuotaFeature implements 
INode.Feature {
 usage.setTypeSpaces(c.getTypeSpaces());
   }
 
+  /** @return the namespace and storagespace and typespace allowed. */
+  public QuotaCounts getSpaceAllowed() {
+return new QuotaCounts.Builder().quotaCount(quota).build();
+  }
+
   /** @return the namespace and storagespace and typespace consumed. */
   public QuotaCounts getSpaceConsumed() {
 return new QuotaCounts.Builder().quotaCount(usage).build();
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java
index f13ef61..710acd9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java
@@ -24,17 +24,18 @@ import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.Date;
-import java.util.stream.Collectors;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.server.namenode.AclFeature;
 import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
+import org.apache.hadoop.hdfs.server.namenode.DirectoryWithQuotaFeature;
 import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
 import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
 import org.apache.hadoop.hdfs.server.namenode.INode;
 import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
+import org.apache.hadoop.hdfs.server.namenode.QuotaCounts;
 import org.apache.hadoop.hdfs.server.namenode.XAttrFeature;
 import org.apache.hadoop.hdfs.util.ReadOnlyList;
 
@@ -145,15 +146,26 @@ public class Snapshot implements Comparable {
   /** The root directory of the snapshot. */
   static public class Root extends INodeDirectory {
 Root(INodeDirectory other) {
-  // Always preserve ACL, XAttr.
-  super(other, false, Arrays.asList(other.getFeatures()).stream().filter(
-  input -> {
-if (AclFeature.class.isInstance(input)
-|| XAttrFeature.class.isInstance(input)) {
-  return true;
+  // Always preserve ACL, XAttr and Quota.
+  super(other, false,
+  Arrays.stream(other.getFeatures()).filter(feature ->
+  feature instanceof AclFeature
+  || feature instanceof XAttrFeature
+  || feature instanceof DirectoryWithQuotaFeature
+  ).map(feature -> {
+if (feature instanceof DirectoryWithQuotaFeature) {
+  // Return copy if feature is quota because a ref could be updated
+  final QuotaCounts quota =
+  ((DirectoryWithQuotaFeature) feature).getSpaceAllowed();
+  return new DirectoryWithQuotaFeature.Builder()
+  .nameSpaceQuota(quota.getNameSpace())
+  .storageSpaceQuota(quota.getStorageSpace())
+  .typeQuotas(quota.getTypeSpaces())
+  .build();
+} else {
+  return feature;
 }
-return false;
-  }).collect(Collectors.toList()).toArray(new Feature[0]));
+  }).

[hadoop] branch branch-3.3 updated: HDFS-16144. Revert HDFS-15372 (Files in snapshots no longer see attribute provider permissions). Contributed by Stephen O'Donnell

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new d661afc  HDFS-16144. Revert HDFS-15372 (Files in snapshots no longer 
see attribute provider permissions). Contributed by Stephen O'Donnell
d661afc is described below

commit d661afc06f40008296afb71012bcb33977e23dce
Author: S O'Donnell 
AuthorDate: Wed Jul 28 14:49:23 2021 +0100

HDFS-16144. Revert HDFS-15372 (Files in snapshots no longer see attribute 
provider permissions). Contributed by Stephen O'Donnell

(cherry picked from commit 4eae284827c040cd9fd0eab2e7fd8aaae721326e)
---
 .../hadoop/hdfs/server/namenode/FSDirectory.java   |  18 +--
 .../hdfs/server/namenode/FSPermissionChecker.java  |  44 +++---
 .../hadoop/hdfs/server/namenode/INodesInPath.java  |  21 ---
 .../namenode/TestINodeAttributeProvider.java   | 155 +++--
 4 files changed, 71 insertions(+), 167 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index bd2ff52..7b902d5 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -2052,23 +2052,7 @@ public class FSDirectory implements Closeable {
   // first empty component for the root.  however file status
   // related calls are expected to strip out the root component according
   // to TestINodeAttributeProvider.
-  // Due to HDFS-15372 the attribute provider should received the resolved
-  // snapshot path. Ie, rather than seeing /d/.snapshot/sn/data it should
-  // see /d/data. However, for the path /d/.snapshot/sn it should see this
-  // full path. If the current inode is the snapshot name, it always has 
the
-  // same ID as its parent inode, so we can use that to check if it is the
-  // path which needs handled specially.
-  byte[][] components;
-  INodeDirectory parent = node.getParent();
-  if (iip.isSnapshot()
-  && parent != null && parent.getId() != node.getId()) {
-// For snapshot paths, we always user node.getPathComponents so the
-// snapshot path is resolved to the real path, unless the last 
component
-// is the snapshot name root directory.
-components = node.getPathComponents();
-  } else {
-components = iip.getPathComponents();
-  }
+  byte[][] components = iip.getPathComponents();
   components = Arrays.copyOfRange(components, 1, components.length);
   nodeAttrs = ap.getAttributes(components, nodeAttrs);
 }
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
index a539bf6..92e5858 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.namenode;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Stack;
@@ -208,7 +207,7 @@ public class FSPermissionChecker implements 
AccessControlEnforcer {
 final INodeAttributes[] inodeAttrs = new INodeAttributes[inodes.length];
 final byte[][] components = inodesInPath.getPathComponents();
 for (int i = 0; i < inodes.length && inodes[i] != null; i++) {
-  inodeAttrs[i] = getINodeAttrs(inodes[i], snapshotId);
+  inodeAttrs[i] = getINodeAttrs(components, i, inodes[i], snapshotId);
 }
 
 String path = inodesInPath.getPath();
@@ -258,7 +257,8 @@ public class FSPermissionChecker implements 
AccessControlEnforcer {
   void checkPermission(INode inode, int snapshotId, FsAction access)
   throws AccessControlException {
 byte[][] pathComponents = inode.getPathComponents();
-INodeAttributes nodeAttributes = getINodeAttrs(inode, snapshotId);
+INodeAttributes nodeAttributes = getINodeAttrs(pathComponents,
+pathComponents.length - 1, inode, snapshotId);
 try {
   INodeAttributes[] iNodeAttr = {nodeAttributes};
   AccessControlEnforcer enforcer = getAccessControlEnforcer();
@@ -367,31 +367,23 @@ public class FSPermissionChecker implements 
AccessControlEnforcer {
 authzContext.getSubAccess(), authzContext.isIgnoreEmptyDir());
   }
 
-  private INodeAttributes get

[hadoop] branch trunk updated: HDFS-16144. Revert HDFS-15372 (Files in snapshots no longer see attribute provider permissions). Contributed by Stephen O'Donnell

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 4eae284  HDFS-16144. Revert HDFS-15372 (Files in snapshots no longer 
see attribute provider permissions). Contributed by Stephen O'Donnell
4eae284 is described below

commit 4eae284827c040cd9fd0eab2e7fd8aaae721326e
Author: S O'Donnell 
AuthorDate: Wed Jul 28 14:49:23 2021 +0100

HDFS-16144. Revert HDFS-15372 (Files in snapshots no longer see attribute 
provider permissions). Contributed by Stephen O'Donnell
---
 .../hadoop/hdfs/server/namenode/FSDirectory.java   |  18 +--
 .../hdfs/server/namenode/FSPermissionChecker.java  |  44 +++---
 .../hadoop/hdfs/server/namenode/INodesInPath.java  |  21 ---
 .../namenode/TestINodeAttributeProvider.java   | 155 +++--
 4 files changed, 71 insertions(+), 167 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 497aa84..9ff54e6 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -2067,23 +2067,7 @@ public class FSDirectory implements Closeable {
   // first empty component for the root.  however file status
   // related calls are expected to strip out the root component according
   // to TestINodeAttributeProvider.
-  // Due to HDFS-15372 the attribute provider should received the resolved
-  // snapshot path. Ie, rather than seeing /d/.snapshot/sn/data it should
-  // see /d/data. However, for the path /d/.snapshot/sn it should see this
-  // full path. If the current inode is the snapshot name, it always has 
the
-  // same ID as its parent inode, so we can use that to check if it is the
-  // path which needs handled specially.
-  byte[][] components;
-  INodeDirectory parent = node.getParent();
-  if (iip.isSnapshot()
-  && parent != null && parent.getId() != node.getId()) {
-// For snapshot paths, we always user node.getPathComponents so the
-// snapshot path is resolved to the real path, unless the last 
component
-// is the snapshot name root directory.
-components = node.getPathComponents();
-  } else {
-components = iip.getPathComponents();
-  }
+  byte[][] components = iip.getPathComponents();
   components = Arrays.copyOfRange(components, 1, components.length);
   nodeAttrs = ap.getAttributes(components, nodeAttrs);
 }
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
index e8e2927..324cd5d 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.namenode;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Stack;
@@ -265,7 +264,7 @@ public class FSPermissionChecker implements 
AccessControlEnforcer {
 final INodeAttributes[] inodeAttrs = new INodeAttributes[inodes.length];
 final byte[][] components = inodesInPath.getPathComponents();
 for (int i = 0; i < inodes.length && inodes[i] != null; i++) {
-  inodeAttrs[i] = getINodeAttrs(inodes[i], snapshotId);
+  inodeAttrs[i] = getINodeAttrs(components, i, inodes[i], snapshotId);
 }
 
 String path = inodesInPath.getPath();
@@ -315,7 +314,8 @@ public class FSPermissionChecker implements 
AccessControlEnforcer {
   void checkPermission(INode inode, int snapshotId, FsAction access)
   throws AccessControlException {
 byte[][] pathComponents = inode.getPathComponents();
-INodeAttributes nodeAttributes = getINodeAttrs(inode, snapshotId);
+INodeAttributes nodeAttributes = getINodeAttrs(pathComponents,
+pathComponents.length - 1, inode, snapshotId);
 try {
   INodeAttributes[] iNodeAttr = {nodeAttributes};
   AccessControlEnforcer enforcer = getAccessControlEnforcer();
@@ -424,31 +424,23 @@ public class FSPermissionChecker implements 
AccessControlEnforcer {
 authzContext.getSubAccess(), authzContext.isIgnoreEmptyDir());
   }
 
-  private INodeAttributes getINodeAttrs(INode inode, int snapshotId) {
+  private INodeAttributes getINodeAttrs(byte[][]

[hadoop] branch branch-3.3 updated: HDFS-16145. CopyListing fails with FNF exception with snapshot diff. (#3234)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 18bd66e5 HDFS-16145. CopyListing fails with FNF exception with 
snapshot diff. (#3234)
18bd66e5 is described below

commit 18bd66e5b0f447c5354e44f215dd101955030023
Author: bshashikant 
AuthorDate: Wed Jul 28 10:29:00 2021 +0530

HDFS-16145. CopyListing fails with FNF exception with snapshot diff. (#3234)

(cherry picked from commit dac10fcc202ed6d1fe4bd852f57a6bbcbadd90fe)
---
 .../java/org/apache/hadoop/tools/DistCpSync.java   |  64 +++-
 .../org/apache/hadoop/tools/TestDistCpSync.java| 171 +
 2 files changed, 232 insertions(+), 3 deletions(-)

diff --git 
a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
 
b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
index 35ef3e4..5692049 100644
--- 
a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
+++ 
b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.tools;
 
+import 
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
@@ -36,6 +37,7 @@ import java.util.Random;
 import java.util.EnumMap;
 import java.util.ArrayList;
 import java.util.HashSet;
+import java.util.Collections;
 
 /**
  * This class provides the basic functionality to sync two FileSystems based on
@@ -57,6 +59,9 @@ class DistCpSync {
   //
   private EnumMap> diffMap;
   private DiffInfo[] renameDiffs;
+  // entries which are marked deleted because of rename to a excluded target
+  // path
+  private List deletedByExclusionDiffs;
   private CopyFilter copyFilter;
 
   DistCpSync(DistCpContext context, Configuration conf) {
@@ -66,6 +71,11 @@ class DistCpSync {
 this.copyFilter.initialize();
   }
 
+  @VisibleForTesting
+  public void setCopyFilter(CopyFilter copyFilter) {
+this.copyFilter = copyFilter;
+  }
+
   private boolean isRdiff() {
 return context.shouldUseRdiff();
   }
@@ -206,7 +216,7 @@ class DistCpSync {
   SnapshotDiffReport.DiffType.values()) {
 diffMap.put(type, new ArrayList());
   }
-
+  deletedByExclusionDiffs = null;
   for (SnapshotDiffReport.DiffReportEntry entry : report.getDiffList()) {
 // If the entry is the snapshot root, usually a item like "M\t."
 // in the diff report. We don't need to handle it and cannot handle it,
@@ -234,8 +244,13 @@ class DistCpSync {
   list.add(new DiffInfo(source, target, dt));
 } else {
   list = diffMap.get(SnapshotDiffReport.DiffType.DELETE);
-  list.add(new DiffInfo(source, target,
-  SnapshotDiffReport.DiffType.DELETE));
+  DiffInfo info = new DiffInfo(source, target,
+  SnapshotDiffReport.DiffType.DELETE);
+  list.add(info);
+  if (deletedByExclusionDiffs == null) {
+deletedByExclusionDiffs = new ArrayList<>();
+  }
+  deletedByExclusionDiffs.add(info);
 }
   } else if (copyFilter.shouldCopy(relativeTarget)) {
 list = diffMap.get(SnapshotDiffReport.DiffType.CREATE);
@@ -244,6 +259,9 @@ class DistCpSync {
   }
 }
   }
+  if (deletedByExclusionDiffs != null) {
+Collections.sort(deletedByExclusionDiffs, DiffInfo.sourceComparator);
+  }
   return true;
 } catch (IOException e) {
   DistCp.LOG.warn("Failed to compute snapshot diff on " + ssDir, e);
@@ -494,6 +512,33 @@ class DistCpSync {
   }
 
   /**
+   * checks if a parent dir is marked deleted as a part of dir rename happening
+   * to a path which is excluded by the the filter.
+   * @return true if it's marked deleted
+   */
+  private boolean isParentOrSelfMarkedDeleted(DiffInfo diff,
+  List deletedDirDiffArray) {
+for (DiffInfo item : deletedDirDiffArray) {
+  if (item.getSource().equals(diff.getSource())) {
+// The same path string may appear in:
+// 1. both deleted and modified snapshot diff entries.
+// 2. both deleted and created snapshot diff entries.
+// Case 1 is the about same file/directory, whereas case 2
+// is about two different files/directories.
+// We are finding case 1 here, thus we check against DiffType.MODIFY.
+if (diff.getType() == SnapshotDiffReport.DiffType.MODIFY) {
+  return true;
+}
+  } else if (isParentOf(item.getSource(), diff.getSource())) {
+// If deleted entry is the parent of diff entry, then both 

[hadoop] branch branch-3.2 updated: HDFS-15796. ConcurrentModificationException error happens on NameNode occasionally. Contributed by Daniel Ma

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new e2eacbd  HDFS-15796. ConcurrentModificationException error happens on 
NameNode occasionally. Contributed by Daniel Ma
e2eacbd is described below

commit e2eacbd08978665b2c90743dba628afe01475263
Author: S O'Donnell 
AuthorDate: Fri Jul 9 10:03:57 2021 +0100

HDFS-15796. ConcurrentModificationException error happens on NameNode 
occasionally. Contributed by Daniel Ma

(cherry picked from commit b6c06c4b76ca5c206639e2ddea4f3bef393dbb11)
---
 .../hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
index 6e1af57..8c01457 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
@@ -333,7 +333,7 @@ class PendingReconstructionBlocks {
 synchronized (pendingReconstructions) {
   PendingBlockInfo found = pendingReconstructions.get(block);
   if (found != null) {
-return found.targets;
+return new ArrayList<>(found.targets);
   }
 }
 return null;

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.3 updated: HDFS-15796. ConcurrentModificationException error happens on NameNode occasionally. Contributed by Daniel Ma

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 705884b  HDFS-15796. ConcurrentModificationException error happens on 
NameNode occasionally. Contributed by Daniel Ma
705884b is described below

commit 705884b87a3c410409f0e40f9d07a8e4fccb0df0
Author: S O'Donnell 
AuthorDate: Fri Jul 9 10:03:57 2021 +0100

HDFS-15796. ConcurrentModificationException error happens on NameNode 
occasionally. Contributed by Daniel Ma

(cherry picked from commit b6c06c4b76ca5c206639e2ddea4f3bef393dbb11)
---
 .../hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
index acf0598..81495eb 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
@@ -333,7 +333,7 @@ class PendingReconstructionBlocks {
 synchronized (pendingReconstructions) {
   PendingBlockInfo found = pendingReconstructions.get(block);
   if (found != null) {
-return found.targets;
+return new ArrayList<>(found.targets);
   }
 }
 return null;

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: HDFS-15796. ConcurrentModificationException error happens on NameNode occasionally. Contributed by Daniel Ma

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new b6c06c4  HDFS-15796. ConcurrentModificationException error happens on 
NameNode occasionally. Contributed by Daniel Ma
b6c06c4 is described below

commit b6c06c4b76ca5c206639e2ddea4f3bef393dbb11
Author: S O'Donnell 
AuthorDate: Fri Jul 9 10:03:57 2021 +0100

HDFS-15796. ConcurrentModificationException error happens on NameNode 
occasionally. Contributed by Daniel Ma
---
 .../hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
index acf0598..81495eb 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
@@ -333,7 +333,7 @@ class PendingReconstructionBlocks {
 synchronized (pendingReconstructions) {
   PendingBlockInfo found = pendingReconstructions.get(block);
   if (found != null) {
-return found.targets;
+return new ArrayList<>(found.targets);
   }
 }
 return null;

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.2 updated: HDFS-16121. Iterative snapshot diff report can generate duplicate records for creates, deletes and Renames. (#3188)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new 844b886  HDFS-16121. Iterative snapshot diff report can generate 
duplicate records for creates, deletes and Renames. (#3188)
844b886 is described below

commit 844b8865c057a1737eef421b67bc7079aa3f7192
Author: bshashikant 
AuthorDate: Fri Jul 9 10:26:42 2021 +0530

HDFS-16121. Iterative snapshot diff report can generate duplicate records 
for creates, deletes and Renames. (#3188)

(cherry picked from commit 6450c1ea9ab884015290876798a734cbe5c0038b)
---
 .../namenode/snapshot/SnapshotDiffListingInfo.java |  6 ++-
 .../namenode/snapshot/TestSnapshotDiffReport.java  | 52 ++
 2 files changed, 57 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java
index a796070..88b27cc 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java
@@ -116,7 +116,7 @@ class SnapshotDiffListingInfo {
 
 if (lastIndex == -1 || lastIndex >= clist.size()) {
   final List dlist =  diff.getDeletedUnmodifiable();
-  int size = dlist.size();
+  int size = clist.size();
   ListIterator iterator = lastIndex != -1 ?
   dlist.listIterator(lastIndex - size): dlist.listIterator();
   while (iterator.hasNext()) {
@@ -130,6 +130,10 @@ class SnapshotDiffListingInfo {
   deletedList.add(e);
 } else {
   setLastPath(parent);
+  // the offset will be set to created list + iterator index in the
+  // deleted list so that it points to the exact entry in the deleted
+  // list post checking the created list in the next iteration of rpc
+  // call
   setLastIndex(size + iterator.nextIndex());
   return false;
 }
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
index 18ec3c5..f6fdec8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
@@ -1400,6 +1400,58 @@ public class TestSnapshotDiffReport {
 DFSUtil.string2Bytes("dir3/file3")));
   }
 
+  /**
+   * Tests to verify the diff report with maximum SnapsdiffReportEntries limit
+   * over an rpc being set to 3.
+   * @throws Exception
+   */
+  @Test
+  public void testDiffReportWithRpcLimit3() throws Exception {
+final Path root = new Path("/");
+hdfs.mkdirs(root);
+Path path = new Path(root, "dir1");
+hdfs.mkdirs(path);
+for (int j = 1; j <= 4; j++) {
+  final Path file = new Path(path, "file" + j);
+  DFSTestUtil.createFile(hdfs, file, BLOCKSIZE, REPLICATION, SEED);
+}
+SnapshotTestHelper.createSnapshot(hdfs, root, "s0");
+path = new Path(root, "dir1");
+for (int j = 1; j <= 4; j++) {
+  final Path file = new Path(path, "file" + j);
+  hdfs.delete(file, false);
+}
+for (int j = 5; j <= 10; j++) {
+  final Path file = new Path(path, "file" + j);
+  DFSTestUtil.createFile(hdfs, file, BLOCKSIZE, REPLICATION, SEED);
+}
+
+SnapshotTestHelper.createSnapshot(hdfs, root, "s1");
+verifyDiffReport(root, "s0", "s1",
+new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
+new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1")),
+new DiffReportEntry(DiffType.CREATE,
+DFSUtil.string2Bytes("dir1/file5")),
+new DiffReportEntry(DiffType.CREATE,
+DFSUtil.string2Bytes("dir1/file6")),
+new DiffReportEntry(DiffType.CREATE,
+DFSUtil.string2Bytes("dir1/file7")),
+new DiffReportEntry(DiffType.CREATE,
+DFSUtil.string2Bytes("dir1/file8")),
+new DiffReportEntry(DiffType.CREATE,
+DFSUtil.string2Bytes("dir1/file9")),
+new DiffReportEntry(DiffType.CREATE,
+DFSUtil.string2Bytes("dir1/file10")),
+new DiffReportEntry(DiffT

[hadoop] branch branch-3.3 updated: HDFS-16121. Iterative snapshot diff report can generate duplicate records for creates, deletes and Renames. (#3188)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new ab93981  HDFS-16121. Iterative snapshot diff report can generate 
duplicate records for creates, deletes and Renames. (#3188)
ab93981 is described below

commit ab939814a312064866fb0774317e8af068cb88b5
Author: bshashikant 
AuthorDate: Fri Jul 9 10:26:42 2021 +0530

HDFS-16121. Iterative snapshot diff report can generate duplicate records 
for creates, deletes and Renames. (#3188)

(cherry picked from commit 6450c1ea9ab884015290876798a734cbe5c0038b)
---
 .../namenode/snapshot/SnapshotDiffListingInfo.java |  6 ++-
 .../namenode/snapshot/TestSnapshotDiffReport.java  | 52 ++
 2 files changed, 57 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java
index 8861a1f0..4b03c4f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java
@@ -116,7 +116,7 @@ class SnapshotDiffListingInfo {
 
 if (lastIndex == -1 || lastIndex >= clist.size()) {
   final List dlist =  diff.getDeletedUnmodifiable();
-  int size = dlist.size();
+  int size = clist.size();
   ListIterator iterator = lastIndex != -1 ?
   dlist.listIterator(lastIndex - size): dlist.listIterator();
   while (iterator.hasNext()) {
@@ -130,6 +130,10 @@ class SnapshotDiffListingInfo {
   deletedList.add(e);
 } else {
   setLastPath(parent);
+  // the offset will be set to created list + iterator index in the
+  // deleted list so that it points to the exact entry in the deleted
+  // list post checking the created list in the next iteration of rpc
+  // call
   setLastIndex(size + iterator.nextIndex());
   return false;
 }
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
index 18ec3c5..f6fdec8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
@@ -1400,6 +1400,58 @@ public class TestSnapshotDiffReport {
 DFSUtil.string2Bytes("dir3/file3")));
   }
 
+  /**
+   * Tests to verify the diff report with maximum SnapsdiffReportEntries limit
+   * over an rpc being set to 3.
+   * @throws Exception
+   */
+  @Test
+  public void testDiffReportWithRpcLimit3() throws Exception {
+final Path root = new Path("/");
+hdfs.mkdirs(root);
+Path path = new Path(root, "dir1");
+hdfs.mkdirs(path);
+for (int j = 1; j <= 4; j++) {
+  final Path file = new Path(path, "file" + j);
+  DFSTestUtil.createFile(hdfs, file, BLOCKSIZE, REPLICATION, SEED);
+}
+SnapshotTestHelper.createSnapshot(hdfs, root, "s0");
+path = new Path(root, "dir1");
+for (int j = 1; j <= 4; j++) {
+  final Path file = new Path(path, "file" + j);
+  hdfs.delete(file, false);
+}
+for (int j = 5; j <= 10; j++) {
+  final Path file = new Path(path, "file" + j);
+  DFSTestUtil.createFile(hdfs, file, BLOCKSIZE, REPLICATION, SEED);
+}
+
+SnapshotTestHelper.createSnapshot(hdfs, root, "s1");
+verifyDiffReport(root, "s0", "s1",
+new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
+new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1")),
+new DiffReportEntry(DiffType.CREATE,
+DFSUtil.string2Bytes("dir1/file5")),
+new DiffReportEntry(DiffType.CREATE,
+DFSUtil.string2Bytes("dir1/file6")),
+new DiffReportEntry(DiffType.CREATE,
+DFSUtil.string2Bytes("dir1/file7")),
+new DiffReportEntry(DiffType.CREATE,
+DFSUtil.string2Bytes("dir1/file8")),
+new DiffReportEntry(DiffType.CREATE,
+DFSUtil.string2Bytes("dir1/file9")),
+new DiffReportEntry(DiffType.CREATE,
+DFSUtil.string2Bytes("dir1/file10")),
+new DiffReportEntry(DiffT

[hadoop] branch branch-2.10 updated: HDFS-15150. Introduce read write lock to Datanode. Contributed by Stephen O'Donnell and Ahmed Hussein

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-2.10 by this push:
 new caa59d7  HDFS-15150. Introduce read write lock to Datanode. 
Contributed by Stephen O'Donnell and Ahmed Hussein
caa59d7 is described below

commit caa59d7a582cbe7c3471866f25a7d13ef0b200bb
Author: S O'Donnell 
AuthorDate: Tue Jun 22 16:24:19 2021 +0100

HDFS-15150. Introduce read write lock to Datanode. Contributed by Stephen 
O'Donnell and Ahmed Hussein
---
 .../hadoop/util/InstrumentedReadWriteLock.java |   2 +-
 .../java/org/apache/hadoop/hdfs/DFSConfigKeys.java |   7 ++
 .../server/datanode/fsdataset/FsDatasetSpi.java|   6 +
 .../datanode/fsdataset/impl/BlockPoolSlice.java|   4 +-
 .../datanode/fsdataset/impl/FsDatasetImpl.java | 140 -
 .../server/datanode/fsdataset/impl/ReplicaMap.java |  31 +++--
 .../src/main/resources/hdfs-default.xml|  21 
 .../hdfs/server/datanode/SimulatedFSDataset.java   |   6 +
 .../datanode/extdataset/ExternalDatasetImpl.java   |   5 +
 .../fsdataset/impl/FsDatasetImplTestUtils.java |   2 +-
 .../datanode/fsdataset/impl/TestFsVolumeList.java  |   4 +-
 .../fsdataset/impl/TestInterDatanodeProtocol.java  |   4 +-
 .../datanode/fsdataset/impl/TestReplicaMap.java|   9 +-
 .../fsdataset/impl/TestWriteToReplica.java |   4 +-
 14 files changed, 156 insertions(+), 89 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadWriteLock.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadWriteLock.java
index a410524..758f1ff 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadWriteLock.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadWriteLock.java
@@ -37,7 +37,7 @@ public class InstrumentedReadWriteLock implements 
ReadWriteLock {
   private final Lock readLock;
   private final Lock writeLock;
 
-  InstrumentedReadWriteLock(boolean fair, String name, Logger logger,
+  public InstrumentedReadWriteLock(boolean fair, String name, Logger logger,
   long minLoggingGapMs, long lockWarningThresholdMs) {
 ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(fair);
 readLock = new InstrumentedReadLock(name, logger, readWriteLock,
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index e71ed95..980ce9b 100755
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -452,6 +452,13 @@ public class DFSConfigKeys extends CommonConfigurationKeys 
{
   "dfs.lock.suppress.warning.interval";
   public static final long DFS_LOCK_SUPPRESS_WARNING_INTERVAL_DEFAULT =
   1; //ms
+  public static final String DFS_DATANODE_LOCK_FAIR_KEY =
+  "dfs.datanode.lock.fair";
+  public static final boolean DFS_DATANODE_LOCK_FAIR_DEFAULT = true;
+  public static final String  DFS_DATANODE_LOCK_REPORTING_THRESHOLD_MS_KEY =
+  "dfs.datanode.lock-reporting-threshold-ms";
+  public static final long
+  DFS_DATANODE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT = 300L;
 
   public static final String  DFS_UPGRADE_DOMAIN_FACTOR = 
"dfs.namenode.upgrade.domain.factor";
   public static final int DFS_UPGRADE_DOMAIN_FACTOR_DEFAULT = 
DFS_REPLICATION_DEFAULT;
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index 7953d45..d29d772 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@ -662,6 +662,12 @@ public interface FsDatasetSpi 
extends FSDatasetMBean {
*/
   AutoCloseableLock acquireDatasetLock();
 
+  /***
+   * Acquire the read lock of the data set.
+   * @return The AutoClosable read lock instance.
+   */
+  AutoCloseableLock acquireDatasetReadLock();
+
   /**
* Deep copy the replica info belonging to given block pool.
* @param bpid Specified block pool id.
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index fca01bb..72bfa9a 100644
--

[hadoop] branch branch-3.3 updated: HADOOP-16954. Add -S option in "Count" command to show only Snapshot Counts. Contributed by hemanthboyina.

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 65bf544  HADOOP-16954. Add -S option in "Count" command to show only 
Snapshot Counts. Contributed by hemanthboyina.
65bf544 is described below

commit 65bf5441182a39589d37dfe7ed707ae03a693409
Author: Takanobu Asanuma 
AuthorDate: Mon Apr 6 11:03:10 2020 +0900

HADOOP-16954. Add -S option in "Count" command to show only Snapshot 
Counts. Contributed by hemanthboyina.

(cherry picked from commit b89d875f7b1db4a98d37f13040eecc5afdf1a485)
---
 .../java/org/apache/hadoop/fs/ContentSummary.java  | 34 ++--
 .../java/org/apache/hadoop/fs/shell/Count.java | 17 --
 .../src/site/markdown/FileSystemShell.md   |  5 ++-
 .../org/apache/hadoop/fs/TestContentSummary.java   | 36 ++
 .../java/org/apache/hadoop/fs/shell/TestCount.java | 25 +--
 .../hadoop-common/src/test/resources/testConf.xml  |  2 +-
 6 files changed, 110 insertions(+), 9 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
index cdbd10f..20e205a 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
@@ -281,6 +281,21 @@ public class ContentSummary extends QuotaUsage implements 
Writable{
 
   private static final String ALL_HEADER = QUOTA_HEADER + SUMMARY_HEADER;
 
+  /**
+   * Output format:<---18---> <--24-->
+   * <--24-->. <-28> SNAPSHOT_LENGTH
+   * SNAPSHOT_FILE_COUNT SNAPSHOT_DIR_COUNT SNAPSHOT_SPACE_CONSUMED
+   */
+  private static final String SNAPSHOT_FORMAT = "%18s %24s %24s %28s ";
+
+  private static final String[] SNAPSHOT_HEADER_FIELDS =
+  new String[] {"SNAPSHOT_LENGTH", "SNAPSHOT_FILE_COUNT",
+  "SNAPSHOT_DIR_COUNT", "SNAPSHOT_SPACE_CONSUMED"};
+
+  /** The header string. */
+  private static final String SNAPSHOT_HEADER =
+  String.format(SNAPSHOT_FORMAT, (Object[]) SNAPSHOT_HEADER_FIELDS);
+
 
   /** Return the header of the output.
* if qOption is false, output directory count, file count, and content size;
@@ -293,7 +308,9 @@ public class ContentSummary extends QuotaUsage implements 
Writable{
 return qOption ? ALL_HEADER : SUMMARY_HEADER;
   }
 
-
+  public static String getSnapshotHeader() {
+return SNAPSHOT_HEADER;
+  }
 
   /**
* Returns the names of the fields from the summary header.
@@ -416,7 +433,7 @@ public class ContentSummary extends QuotaUsage implements 
Writable{
   }
 
   /**
-   * Formats a size to be human readable or in bytes
+   * Formats a size to be human readable or in bytes.
* @param size value to be formatted
* @param humanReadable flag indicating human readable or not
* @return String representation of the size
@@ -426,4 +443,17 @@ public class ContentSummary extends QuotaUsage implements 
Writable{
   ? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1)
   : String.valueOf(size);
   }
+
+  /**
+   * Return the string representation of the snapshot counts in the output
+   * format.
+   * @param hOption flag indicating human readable or not
+   * @return String representation of the snapshot counts
+   */
+  public String toSnapshot(boolean hOption) {
+return String.format(SNAPSHOT_FORMAT, formatSize(snapshotLength, hOption),
+formatSize(snapshotFileCount, hOption),
+formatSize(snapshotDirectoryCount, hOption),
+formatSize(snapshotSpaceConsumed, hOption));
+  }
 }
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
index 22d8be5..ab7e195 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
@@ -56,13 +56,14 @@ public class Count extends FsCommand {
   //return the quota, namespace count and disk space usage.
   private static final String OPTION_QUOTA_AND_USAGE = "u";
   private static final String OPTION_ECPOLICY = "e";
+  private static final String OPTION_SNAPSHOT_COUNT = "s";
 
   public static final String NAME = "count";
   public static final String USAGE =
   "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
   + "

[hadoop] branch branch-3.3 updated: HADOOP-17675. LdapGroupsMapping$LdapSslSocketFactory ClassNotFoundException (#2965)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 00ba886  HADOOP-17675. LdapGroupsMapping$LdapSslSocketFactory 
ClassNotFoundException (#2965)
00ba886 is described below

commit 00ba886a223e091a8887a5c272fa036d37142cfc
Author: Istvan Fajth 
AuthorDate: Tue May 4 12:33:59 2021 +0200

HADOOP-17675. LdapGroupsMapping$LdapSslSocketFactory ClassNotFoundException 
(#2965)

(cherry picked from commit 0d78d73973cf8643c4120678ebeea9cde473a2c4)
---
 .../apache/hadoop/security/LdapGroupsMapping.java  | 24 +-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
index 3e89c27..c51821f 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
@@ -677,7 +677,29 @@ public class LdapGroupsMapping
   env.put("com.sun.jndi.ldap.read.timeout", conf.get(READ_TIMEOUT,
   String.valueOf(READ_TIMEOUT_DEFAULT)));
 
-  ctx = new InitialDirContext(env);
+  // See HADOOP-17675 for details TLDR:
+  // From a native thread the thread's context classloader is null.
+  // jndi internally in the InitialDirContext specifies the context
+  // classloader for Class.forName, and as it is null, jndi will use the
+  // bootstrap classloader in this case to laod the socket factory
+  // implementation.
+  // BUT
+  // Bootstrap classloader does not have it in its classpath, so throws a
+  // ClassNotFoundException.
+  // This affects Impala for example when it uses LdapGroupsMapping.
+  ClassLoader currentContextLoader =
+  Thread.currentThread().getContextClassLoader();
+  if (currentContextLoader == null) {
+try {
+  Thread.currentThread().setContextClassLoader(
+  this.getClass().getClassLoader());
+  ctx = new InitialDirContext(env);
+} finally {
+  Thread.currentThread().setContextClassLoader(null);
+}
+  } else {
+ctx = new InitialDirContext(env);
+  }
 }
 return ctx;
   }

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: HADOOP-17675. LdapGroupsMapping$LdapSslSocketFactory ClassNotFoundException (#2965)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 0d78d73  HADOOP-17675. LdapGroupsMapping$LdapSslSocketFactory 
ClassNotFoundException (#2965)
0d78d73 is described below

commit 0d78d73973cf8643c4120678ebeea9cde473a2c4
Author: Istvan Fajth 
AuthorDate: Tue May 4 12:33:59 2021 +0200

HADOOP-17675. LdapGroupsMapping$LdapSslSocketFactory ClassNotFoundException 
(#2965)
---
 .../apache/hadoop/security/LdapGroupsMapping.java  | 24 +-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
index 10483a2..17934ad 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
@@ -643,7 +643,29 @@ public class LdapGroupsMapping
   env.put("com.sun.jndi.ldap.read.timeout", conf.get(READ_TIMEOUT,
   String.valueOf(READ_TIMEOUT_DEFAULT)));
 
-  ctx = new InitialDirContext(env);
+  // See HADOOP-17675 for details TLDR:
+  // From a native thread the thread's context classloader is null.
+  // jndi internally in the InitialDirContext specifies the context
+  // classloader for Class.forName, and as it is null, jndi will use the
+  // bootstrap classloader in this case to laod the socket factory
+  // implementation.
+  // BUT
+  // Bootstrap classloader does not have it in its classpath, so throws a
+  // ClassNotFoundException.
+  // This affects Impala for example when it uses LdapGroupsMapping.
+  ClassLoader currentContextLoader =
+  Thread.currentThread().getContextClassLoader();
+  if (currentContextLoader == null) {
+try {
+  Thread.currentThread().setContextClassLoader(
+  this.getClass().getClassLoader());
+  ctx = new InitialDirContext(env);
+} finally {
+  Thread.currentThread().setContextClassLoader(null);
+}
+  } else {
+ctx = new InitialDirContext(env);
+  }
 }
 return ctx;
   }

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.3 updated: HDFS-15865. Interrupt DataStreamer thread if no ack (#2728)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 00c3f3c  HDFS-15865. Interrupt DataStreamer thread if no ack (#2728)
00c3f3c is described below

commit 00c3f3c42f93a5bbf5c48608619867560cc82b09
Author: Karthik Palanisamy 
AuthorDate: Sat May 1 11:05:31 2021 -0700

HDFS-15865. Interrupt DataStreamer thread if no ack (#2728)

(cherry picked from commit bd3da73a0ff75231340b1168f7805164710bf4fe)
---
 .../src/main/java/org/apache/hadoop/hdfs/DataStreamer.java   | 12 
 1 file changed, 12 insertions(+)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index 3d6d3c5..3e368ae4 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -892,6 +892,8 @@ class DataStreamer extends Daemon {
 try (TraceScope ignored = dfsClient.getTracer().
 newScope("waitForAckedSeqno")) {
   LOG.debug("{} waiting for ack for: {}", this, seqno);
+  int dnodes = nodes != null ? nodes.length : 3;
+  int writeTimeout = dfsClient.getDatanodeWriteTimeout(dnodes);
   long begin = Time.monotonicNow();
   try {
 synchronized (dataQueue) {
@@ -902,6 +904,16 @@ class DataStreamer extends Daemon {
 }
 try {
   dataQueue.wait(1000); // when we receive an ack, we notify on
+  long duration = Time.monotonicNow() - begin;
+  if (duration > writeTimeout) {
+LOG.error("No ack received, took {}ms (threshold={}ms). "
++ "File being written: {}, block: {}, "
++ "Write pipeline datanodes: {}.",
+duration, writeTimeout, src, block, nodes);
+throw new InterruptedIOException("No ack received after " +
+duration / 1000 + "s and a timeout of " +
+writeTimeout / 1000 + "s");
+  }
   // dataQueue
 } catch (InterruptedException ie) {
   throw new InterruptedIOException(

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.1 updated: HDFS-15621. Datanode DirectoryScanner uses excessive memory (#2849). Contributed by Stephen O'Donnell

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.1 by this push:
 new 71a9885  HDFS-15621. Datanode DirectoryScanner uses excessive memory 
(#2849). Contributed by Stephen O'Donnell
71a9885 is described below

commit 71a9885c978c45085997915575bc1ac05729ed65
Author: Stephen O'Donnell 
AuthorDate: Mon Apr 26 11:00:23 2021 +0100

HDFS-15621. Datanode DirectoryScanner uses excessive memory (#2849). 
Contributed by Stephen O'Donnell

(cherry picked from commit 605ed85c291a6250b077da32a49dbb35f3b78bf7)

 Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java

(cherry picked from commit f6efb58b0735fa19a93fa9acae05666cdba8997a)

 Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java

(cherry picked from commit 7a81e50bd2a48d372bd9788aac369078e580e107)
---
 .../hdfs/server/datanode/DirectoryScanner.java |   2 +-
 .../server/datanode/fsdataset/FsVolumeSpi.java | 118 ++---
 .../datanode/fsdataset/impl/FsVolumeImpl.java  |   5 +-
 .../hdfs/server/datanode/TestDirectoryScanner.java |  38 +++
 4 files changed, 74 insertions(+), 89 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index 02aa00b..f36d03b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -515,7 +515,7 @@ public class DirectoryScanner implements Runnable {
  FsVolumeSpi vol) {
 statsRecord.missingBlockFile++;
 statsRecord.missingMetaFile++;
-diffRecord.add(new ScanInfo(blockId, null, null, vol));
+diffRecord.add(new ScanInfo(blockId, null, null, null, vol));
   }
 
   /**
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
index 20a153d..822be0f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
@@ -224,27 +224,27 @@ public interface FsVolumeSpi
*/
   public static class ScanInfo implements Comparable {
 private final long blockId;
-
 /**
- * The block file path, relative to the volume's base directory.
- * If there was no block file found, this may be null. If 'vol'
- * is null, then this is the full path of the block file.
+ * The full path to the folder containing the block / meta files.
  */
-private final String blockSuffix;
-
+private final File basePath;
 /**
- * The suffix of the meta file path relative to the block file.
- * If blockSuffix is null, then this will be the entire path relative
- * to the volume base directory, or an absolute path if vol is also
- * null.
+ * The block file name, with no path
  */
-private final String metaSuffix;
+private final String blockFile;
+/**
+ * Holds the meta file name, with no path, only if blockFile is null.
+ * If blockFile is not null, the meta file will be named identically to
+ * the blockFile, but with a suffix like "_1234.meta". If the blockFile
+ * is present, we store only the meta file suffix.
+ */
+private final String metaFile;
 
 private final FsVolumeSpi volume;
 
 private final FileRegion fileRegion;
 /**
- * Get the file's length in async block scan
+ * Get the file's length in async block scan.
  */
 private final long blockLength;
 
@@ -255,34 +255,18 @@ public interface FsVolumeSpi
 Matcher.quoteReplacement(File.separator);
 
 /**
- * Get the most condensed version of the path.
- *
- * For example, the condensed version of /foo//bar is /foo/bar
- * Unlike {@link File#getCanonicalPath()}, this will never perform I/O
- * on the filesystem.
- *
- * @param path the path to condense
- * @return the condensed path
- */
-private static String getCondensedPath(String path) {
-  return CONDENSED_PATH_REGEX.matcher(path).
-  replaceAll(QUOTED_FILE

[hadoop] branch branch-3.2 updated: HDFS-15621. Datanode DirectoryScanner uses excessive memory (#2849). Contributed by Stephen O'Donnell

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new 7a81e50  HDFS-15621. Datanode DirectoryScanner uses excessive memory 
(#2849). Contributed by Stephen O'Donnell
7a81e50 is described below

commit 7a81e50bd2a48d372bd9788aac369078e580e107
Author: Stephen O'Donnell 
AuthorDate: Mon Apr 26 11:00:23 2021 +0100

HDFS-15621. Datanode DirectoryScanner uses excessive memory (#2849). 
Contributed by Stephen O'Donnell

(cherry picked from commit 605ed85c291a6250b077da32a49dbb35f3b78bf7)

 Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java

(cherry picked from commit f6efb58b0735fa19a93fa9acae05666cdba8997a)

 Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
---
 .../hdfs/server/datanode/DirectoryScanner.java |   2 +-
 .../server/datanode/fsdataset/FsVolumeSpi.java | 118 ++---
 .../datanode/fsdataset/impl/FsVolumeImpl.java  |   5 +-
 .../hdfs/server/datanode/TestDirectoryScanner.java |  38 +++
 4 files changed, 74 insertions(+), 89 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index 29ca868..a488428 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -515,7 +515,7 @@ public class DirectoryScanner implements Runnable {
  FsVolumeSpi vol) {
 statsRecord.missingBlockFile++;
 statsRecord.missingMetaFile++;
-diffRecord.add(new ScanInfo(blockId, null, null, vol));
+diffRecord.add(new ScanInfo(blockId, null, null, null, vol));
   }
 
   /**
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
index 20a153d..822be0f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
@@ -224,27 +224,27 @@ public interface FsVolumeSpi
*/
   public static class ScanInfo implements Comparable {
 private final long blockId;
-
 /**
- * The block file path, relative to the volume's base directory.
- * If there was no block file found, this may be null. If 'vol'
- * is null, then this is the full path of the block file.
+ * The full path to the folder containing the block / meta files.
  */
-private final String blockSuffix;
-
+private final File basePath;
 /**
- * The suffix of the meta file path relative to the block file.
- * If blockSuffix is null, then this will be the entire path relative
- * to the volume base directory, or an absolute path if vol is also
- * null.
+ * The block file name, with no path
  */
-private final String metaSuffix;
+private final String blockFile;
+/**
+ * Holds the meta file name, with no path, only if blockFile is null.
+ * If blockFile is not null, the meta file will be named identically to
+ * the blockFile, but with a suffix like "_1234.meta". If the blockFile
+ * is present, we store only the meta file suffix.
+ */
+private final String metaFile;
 
 private final FsVolumeSpi volume;
 
 private final FileRegion fileRegion;
 /**
- * Get the file's length in async block scan
+ * Get the file's length in async block scan.
  */
 private final long blockLength;
 
@@ -255,34 +255,18 @@ public interface FsVolumeSpi
 Matcher.quoteReplacement(File.separator);
 
 /**
- * Get the most condensed version of the path.
- *
- * For example, the condensed version of /foo//bar is /foo/bar
- * Unlike {@link File#getCanonicalPath()}, this will never perform I/O
- * on the filesystem.
- *
- * @param path the path to condense
- * @return the condensed path
- */
-private static String getCondensedPath(String path) {
-  return CONDENSED_PATH_REGEX.matcher(path).
-  replaceAll(QUOTED_FILE_SEPARATOR);
-}
-
-/**
  * Get a path suffix.
  *
- 

[hadoop] branch branch-3.3 updated: HDFS-15621. Datanode DirectoryScanner uses excessive memory (#2849). Contributed by Stephen O'Donnell

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new f6efb58  HDFS-15621. Datanode DirectoryScanner uses excessive memory 
(#2849). Contributed by Stephen O'Donnell
f6efb58 is described below

commit f6efb58b0735fa19a93fa9acae05666cdba8997a
Author: Stephen O'Donnell 
AuthorDate: Mon Apr 26 11:00:23 2021 +0100

HDFS-15621. Datanode DirectoryScanner uses excessive memory (#2849). 
Contributed by Stephen O'Donnell

(cherry picked from commit 605ed85c291a6250b077da32a49dbb35f3b78bf7)

 Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
---
 .../hdfs/server/datanode/DirectoryScanner.java |   2 +-
 .../server/datanode/fsdataset/FsVolumeSpi.java | 118 ++---
 .../datanode/fsdataset/impl/FsVolumeImpl.java  |   5 +-
 .../hdfs/server/datanode/TestDirectoryScanner.java |  37 +++
 4 files changed, 75 insertions(+), 87 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index be7e47a..b48d6ce 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -582,7 +582,7 @@ public class DirectoryScanner implements Runnable {
   long blockId, FsVolumeSpi vol) {
 statsRecord.missingBlockFile++;
 statsRecord.missingMetaFile++;
-diffRecord.add(new ScanInfo(blockId, null, null, vol));
+diffRecord.add(new ScanInfo(blockId, null, null, null, vol));
   }
 
   /**
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
index be978d7..c1043ae 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
@@ -224,27 +224,27 @@ public interface FsVolumeSpi
*/
   public static class ScanInfo implements Comparable {
 private final long blockId;
-
 /**
- * The block file path, relative to the volume's base directory.
- * If there was no block file found, this may be null. If 'vol'
- * is null, then this is the full path of the block file.
+ * The full path to the folder containing the block / meta files.
  */
-private final String blockSuffix;
-
+private final File basePath;
 /**
- * The suffix of the meta file path relative to the block file.
- * If blockSuffix is null, then this will be the entire path relative
- * to the volume base directory, or an absolute path if vol is also
- * null.
+ * The block file name, with no path
  */
-private final String metaSuffix;
+private final String blockFile;
+/**
+ * Holds the meta file name, with no path, only if blockFile is null.
+ * If blockFile is not null, the meta file will be named identically to
+ * the blockFile, but with a suffix like "_1234.meta". If the blockFile
+ * is present, we store only the meta file suffix.
+ */
+private final String metaFile;
 
 private final FsVolumeSpi volume;
 
 private final FileRegion fileRegion;
 /**
- * Get the file's length in async block scan
+ * Get the file's length in async block scan.
  */
 private final long blockLength;
 
@@ -255,34 +255,18 @@ public interface FsVolumeSpi
 Matcher.quoteReplacement(File.separator);
 
 /**
- * Get the most condensed version of the path.
- *
- * For example, the condensed version of /foo//bar is /foo/bar
- * Unlike {@link File#getCanonicalPath()}, this will never perform I/O
- * on the filesystem.
- *
- * @param path the path to condense
- * @return the condensed path
- */
-private static String getCondensedPath(String path) {
-  return CONDENSED_PATH_REGEX.matcher(path).
-  replaceAll(QUOTED_FILE_SEPARATOR);
-}
-
-/**
  * Get a path suffix.
  *
- * @param fThe file to get the suffix for.
+ * @param fThe string to get the suffix for.
  * @param prefix   The prefix we're stripping off.
  *
- * @return A suffix such that prefix + suffix = path to f
+ * @return A suffix such that prefix + suffix = f
  */
-private static String

[hadoop] branch trunk updated: HDFS-15621. Datanode DirectoryScanner uses excessive memory (#2849). Contributed by Stephen O'Donnell

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 605ed85  HDFS-15621. Datanode DirectoryScanner uses excessive memory 
(#2849). Contributed by Stephen O'Donnell
605ed85 is described below

commit 605ed85c291a6250b077da32a49dbb35f3b78bf7
Author: Stephen O'Donnell 
AuthorDate: Mon Apr 26 11:00:23 2021 +0100

HDFS-15621. Datanode DirectoryScanner uses excessive memory (#2849). 
Contributed by Stephen O'Donnell
---
 .../hdfs/server/datanode/DirectoryScanner.java |   2 +-
 .../server/datanode/fsdataset/FsVolumeSpi.java | 118 ++---
 .../datanode/fsdataset/impl/FsVolumeImpl.java  |   5 +-
 .../hdfs/server/datanode/TestDirectoryScanner.java |  37 +++
 .../datanode/fsdataset/impl/TestFsDatasetImpl.java |   4 +-
 5 files changed, 77 insertions(+), 89 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index 63865f6..a3bceec 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -582,7 +582,7 @@ public class DirectoryScanner implements Runnable {
   long blockId, FsVolumeSpi vol) {
 statsRecord.missingBlockFile++;
 statsRecord.missingMetaFile++;
-diffRecord.add(new ScanInfo(blockId, null, null, vol));
+diffRecord.add(new ScanInfo(blockId, null, null, null, vol));
   }
 
   /**
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
index 68d1a15..8ae2043 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
@@ -227,27 +227,27 @@ public interface FsVolumeSpi
*/
   public static class ScanInfo implements Comparable {
 private final long blockId;
-
 /**
- * The block file path, relative to the volume's base directory.
- * If there was no block file found, this may be null. If 'vol'
- * is null, then this is the full path of the block file.
+ * The full path to the folder containing the block / meta files.
  */
-private final String blockSuffix;
-
+private final File basePath;
 /**
- * The suffix of the meta file path relative to the block file.
- * If blockSuffix is null, then this will be the entire path relative
- * to the volume base directory, or an absolute path if vol is also
- * null.
+ * The block file name, with no path
  */
-private final String metaSuffix;
+private final String blockFile;
+/**
+ * Holds the meta file name, with no path, only if blockFile is null.
+ * If blockFile is not null, the meta file will be named identically to
+ * the blockFile, but with a suffix like "_1234.meta". If the blockFile
+ * is present, we store only the meta file suffix.
+ */
+private final String metaFile;
 
 private final FsVolumeSpi volume;
 
 private final FileRegion fileRegion;
 /**
- * Get the file's length in async block scan
+ * Get the file's length in async block scan.
  */
 private final long blockLength;
 
@@ -258,34 +258,18 @@ public interface FsVolumeSpi
 Matcher.quoteReplacement(File.separator);
 
 /**
- * Get the most condensed version of the path.
- *
- * For example, the condensed version of /foo//bar is /foo/bar
- * Unlike {@link File#getCanonicalPath()}, this will never perform I/O
- * on the filesystem.
- *
- * @param path the path to condense
- * @return the condensed path
- */
-private static String getCondensedPath(String path) {
-  return CONDENSED_PATH_REGEX.matcher(path).
-  replaceAll(QUOTED_FILE_SEPARATOR);
-}
-
-/**
  * Get a path suffix.
  *
- * @param fThe file to get the suffix for.
+ * @param fThe string to get the suffix for.
  * @param prefix   The prefix we're stripping off.
  *
- * @return A suffix such that prefix + suffix = path to f
+ * @return A suffix such that prefix + suffix = f
  */
-private static String getSuffix(File f, String prefix) {
-  String fullPath = getCondensedPath(f.getAbsolutePath());
-  if (fullPath.startsWith(prefix)) {
-return fullPath.substring(pre

[hadoop] branch branch-3.3 updated: HDFS-15569. Speed up the Storage#doRecover during datanode rolling upgrade. Contributed by Hemanth Boyina.

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 90f63cb  HDFS-15569. Speed up the Storage#doRecover during datanode 
rolling upgrade. Contributed by Hemanth Boyina.
90f63cb is described below

commit 90f63cb12ccdeba6e423874aa7162dcf76200315
Author: hemanthboyina 
AuthorDate: Tue Dec 22 17:27:31 2020 +0530

HDFS-15569. Speed up the Storage#doRecover during datanode rolling upgrade. 
Contributed by Hemanth Boyina.

(cherry picked from commit 16a20503cacf12c3d8e27ba90820384f58bed06f)
---
 .../apache/hadoop/hdfs/server/common/Storage.java  | 36 +-
 1 file changed, 28 insertions(+), 8 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
index ea10f01..83a8256 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
@@ -801,8 +801,7 @@ public abstract class Storage extends StorageInfo {
   case RECOVER_UPGRADE:   // mv previous.tmp -> current
 LOG.info("Recovering storage directory {} from previous upgrade",
 rootPath);
-if (curDir.exists())
-  deleteDir(curDir);
+deleteAsync(curDir);
 rename(getPreviousTmp(), curDir);
 return;
   case COMPLETE_ROLLBACK: // rm removed.tmp
@@ -818,21 +817,19 @@ public abstract class Storage extends StorageInfo {
   case COMPLETE_FINALIZE: // rm finalized.tmp
 LOG.info("Completing previous finalize for storage directory {}",
 rootPath);
-deleteDir(getFinalizedTmp());
+deleteAsync(getFinalizedTmp());
 return;
   case COMPLETE_CHECKPOINT: // mv lastcheckpoint.tmp -> previous.checkpoint
 LOG.info("Completing previous checkpoint for storage directory {}",
 rootPath);
 File prevCkptDir = getPreviousCheckpoint();
-if (prevCkptDir.exists())
-  deleteDir(prevCkptDir);
+deleteAsync(prevCkptDir);
 rename(getLastCheckpointTmp(), prevCkptDir);
 return;
   case RECOVER_CHECKPOINT:  // mv lastcheckpoint.tmp -> current
 LOG.info("Recovering storage directory {} from failed checkpoint",
 rootPath);
-if (curDir.exists())
-  deleteDir(curDir);
+deleteAsync(curDir);
 rename(getLastCheckpointTmp(), curDir);
 return;
   default:
@@ -840,7 +837,30 @@ public abstract class Storage extends StorageInfo {
 + " for storage directory: " + rootPath);
   }
 }
-
+
+/**
+ * Rename the curDir to curDir.tmp and delete the curDir.tmp parallely.
+ * @throws IOException
+ */
+private void deleteAsync(File curDir) throws IOException {
+  if (curDir.exists()) {
+File curTmp = new File(curDir.getParent(), curDir.getName() + ".tmp");
+if (curTmp.exists()) {
+  deleteDir(curTmp);
+}
+rename(curDir, curTmp);
+new Thread("Async Delete Current.tmp") {
+  public void run() {
+try {
+  deleteDir(curTmp);
+} catch (IOException e) {
+  LOG.warn("Deleting storage directory {} failed", curTmp);
+}
+  }
+}.start();
+  }
+}
+
 /**
  * @return true if the storage directory should prompt the user prior
  * to formatting (i.e if the directory appears to contain some data)

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.3 updated: HDFS-14999. Avoid Potential Infinite Loop in DFSNetworkTopology. Contributed by Ayush Saxena.

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new c93a2f6  HDFS-14999. Avoid Potential Infinite Loop in 
DFSNetworkTopology. Contributed by Ayush Saxena.
c93a2f6 is described below

commit c93a2f67899d4ee08bf55af1e9f2174cf16d90e9
Author: Ayush Saxena 
AuthorDate: Mon May 18 21:06:46 2020 +0530

HDFS-14999. Avoid Potential Infinite Loop in DFSNetworkTopology. 
Contributed by Ayush Saxena.

(cherry picked from commit c84e6beada4e604175f7f138c9878a29665a8c47)
---
 .../apache/hadoop/hdfs/net/DFSNetworkTopology.java | 51 ++
 1 file changed, 32 insertions(+), 19 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
index 02d2c43..c18fdc5 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
@@ -249,17 +249,10 @@ public class DFSNetworkTopology extends NetworkTopology {
   return null;
 }
 // to this point, it is guaranteed that there is at least one node
-// that satisfies the requirement, keep trying until we found one.
-Node chosen;
-do {
-  chosen = chooseRandomWithStorageTypeAndExcludeRoot(root, excludeRoot,
-  type);
-  if (excludedNodes == null || !excludedNodes.contains(chosen)) {
-break;
-  } else {
-LOG.debug("Node {} is excluded, continuing.", chosen);
-  }
-} while (true);
+// that satisfies the requirement.
+Node chosen =
+chooseRandomWithStorageTypeAndExcludeRoot(root, excludeRoot, type,
+excludedNodes);
 LOG.debug("chooseRandom returning {}", chosen);
 return chosen;
   }
@@ -277,23 +270,23 @@ public class DFSNetworkTopology extends NetworkTopology {
* Choose a random node that has the required storage type, under the given
* root, with an excluded subtree root (could also just be a leaf node).
*
-   * Note that excludedNode is checked after a random node, so it is not being
-   * handled here.
-   *
* @param root the root node where we start searching for a datanode
* @param excludeRoot the root of the subtree what should be excluded
* @param type the expected storage type
+   * @param excludedNodes the list of nodes to be excluded
* @return a random datanode, with the storage type, and is not in excluded
* scope
*/
   private Node chooseRandomWithStorageTypeAndExcludeRoot(
-  DFSTopologyNodeImpl root, Node excludeRoot, StorageType type) {
+  DFSTopologyNodeImpl root, Node excludeRoot, StorageType type,
+  Collection excludedNodes) {
 Node chosenNode;
 if (root.isRack()) {
   // children are datanode descriptor
   ArrayList candidates = new ArrayList<>();
   for (Node node : root.getChildren()) {
-if (node.equals(excludeRoot)) {
+if (node.equals(excludeRoot) || (excludedNodes != null && excludedNodes
+.contains(node))) {
   continue;
 }
 DatanodeDescriptor dnDescriptor = (DatanodeDescriptor)node;
@@ -310,7 +303,7 @@ public class DFSNetworkTopology extends NetworkTopology {
 } else {
   // the children are inner nodes
   ArrayList candidates =
-  getEligibleChildren(root, excludeRoot, type);
+  getEligibleChildren(root, excludeRoot, type, excludedNodes);
   if (candidates.size() == 0) {
 return null;
   }
@@ -339,7 +332,7 @@ public class DFSNetworkTopology extends NetworkTopology {
   }
   DFSTopologyNodeImpl nextRoot = candidates.get(idxChosen);
   chosenNode = chooseRandomWithStorageTypeAndExcludeRoot(
-  nextRoot, excludeRoot, type);
+  nextRoot, excludeRoot, type, excludedNodes);
 }
 return chosenNode;
   }
@@ -352,11 +345,13 @@ public class DFSNetworkTopology extends NetworkTopology {
* @param root the subtree root we check.
* @param excludeRoot the root of the subtree that should be excluded.
* @param type the storage type we look for.
+   * @param excludedNodes the list of excluded nodes.
* @return a list of possible nodes, each of them is eligible as the next
* level root we search.
*/
   private ArrayList getEligibleChildren(
-  DFSTopologyNodeImpl root, Node excludeRoot, StorageType type) {
+  DFSTopologyNodeImpl root, Node excludeRoot, StorageType type,
+  Collection excludedNodes) {
 ArrayList candidates = new ArrayList<>();
 int excludeCount = 0;
 if (excludeRoot != null && root.isAncestor(excludeRoot)) {
@@ -383,6 +3

[hadoop] branch branch-3.3 updated: HDFS-15160. ReplicaMap, Disk Balancer, Directory Scanner and various FsDatasetImpl methods should use datanode readlock. Contributed by Stephen O'Donnell. (#2813)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 0e6cd35  HDFS-15160. ReplicaMap, Disk Balancer, Directory Scanner and 
various FsDatasetImpl methods should use datanode readlock. Contributed by 
Stephen O'Donnell. (#2813)
0e6cd35 is described below

commit 0e6cd352b530d81a734bf07274b22e3abd4845f0
Author: Stephen O'Donnell 
AuthorDate: Fri Apr 9 10:25:14 2021 +0100

HDFS-15160. ReplicaMap, Disk Balancer, Directory Scanner and various 
FsDatasetImpl methods should use datanode readlock. Contributed by Stephen 
O'Donnell. (#2813)

Signed-off-by: Wei-Chiu Chuang 
(cherry picked from commit 2a67e2b1a0e3a5f91056f5b977ef9c4c07ba6718)

 Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
---
 .../java/org/apache/hadoop/hdfs/DFSConfigKeys.java |   4 +
 .../hadoop/hdfs/server/datanode/BlockSender.java   |   2 +-
 .../hadoop/hdfs/server/datanode/DataNode.java  |   2 +-
 .../hadoop/hdfs/server/datanode/DiskBalancer.java  |   2 +-
 .../server/datanode/fsdataset/FsDatasetSpi.java|   8 +-
 .../datanode/fsdataset/impl/FsDatasetImpl.java |  64 -
 .../server/datanode/fsdataset/impl/ReplicaMap.java |  31 +--
 .../src/main/resources/hdfs-default.xml|  13 +++
 .../datanode/fsdataset/impl/TestFsDatasetImpl.java | 101 -
 9 files changed, 186 insertions(+), 41 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b964114..ca72dfc 100755
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -609,6 +609,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys 
{
   public static final String DFS_DATANODE_LOCK_FAIR_KEY =
   "dfs.datanode.lock.fair";
   public static final boolean DFS_DATANODE_LOCK_FAIR_DEFAULT = true;
+  public static final String DFS_DATANODE_LOCK_READ_WRITE_ENABLED_KEY =
+  "dfs.datanode.lock.read.write.enabled";
+  public static final Boolean DFS_DATANODE_LOCK_READ_WRITE_ENABLED_DEFAULT =
+  true;
   public static final String  DFS_DATANODE_LOCK_REPORTING_THRESHOLD_MS_KEY =
   "dfs.datanode.lock-reporting-threshold-ms";
   public static final long
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
index 6ea8650..99b9d64 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
@@ -255,7 +255,7 @@ class BlockSender implements java.io.Closeable {
   // the append write.
   ChunkChecksum chunkChecksum = null;
   final long replicaVisibleLength;
-  try(AutoCloseableLock lock = datanode.data.acquireDatasetLock()) {
+  try(AutoCloseableLock lock = datanode.data.acquireDatasetReadLock()) {
 replica = getReplica(block, datanode);
 replicaVisibleLength = replica.getVisibleLength();
   }
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 4d2fba4..cc476d8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -3062,7 +3062,7 @@ public class DataNode extends ReconfigurableBase
 final BlockConstructionStage stage;
 
 //get replica information
-try(AutoCloseableLock lock = data.acquireDatasetLock()) {
+try(AutoCloseableLock lock = data.acquireDatasetReadLock()) {
   Block storedBlock = data.getStoredBlock(b.getBlockPoolId(),
   b.getBlockId());
   if (null == storedBlock) {
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
index 1c57e38..1d49f24 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
+++ 
b/hadoop-hdfs-pro

[hadoop] branch branch-3.3 updated: HDFS-15942. Increase Quota initialization threads. Contributed by Stephen O'Donnell.

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new df99ac0  HDFS-15942. Increase Quota initialization threads. 
Contributed by Stephen O'Donnell.
df99ac0 is described below

commit df99ac0399723844c15fd3a8fad08d8e89058359
Author: He Xiaoqiao 
AuthorDate: Tue Apr 6 21:22:48 2021 +0800

HDFS-15942. Increase Quota initialization threads. Contributed by Stephen 
O'Donnell.

(cherry picked from commit 459df4199791a9093324e089b4a5cc880be8d6ff)
---
 .../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java   | 2 +-
 hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index c516d47..b964114 100755
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -374,7 +374,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String  DFS_NAMENODE_EDITS_DIR_MINIMUM_KEY = 
"dfs.namenode.edits.dir.minimum";
   public static final int DFS_NAMENODE_EDITS_DIR_MINIMUM_DEFAULT = 1;
   public static final String  DFS_NAMENODE_QUOTA_INIT_THREADS_KEY = 
"dfs.namenode.quota.init-threads";
-  public static final int DFS_NAMENODE_QUOTA_INIT_THREADS_DEFAULT = 4;
+  public static final int DFS_NAMENODE_QUOTA_INIT_THREADS_DEFAULT = 12;
 
   public static final String  
DFS_NAMENODE_EDIT_LOG_AUTOROLL_MULTIPLIER_THRESHOLD = 
"dfs.namenode.edit.log.autoroll.multiplier.threshold";
   public static final float
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 15828d9..594f80e 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -3632,11 +3632,11 @@
 
 
   dfs.namenode.quota.init-threads
-  4
+  12
   
 The number of concurrent threads to be used in quota initialization. The
 speed of quota initialization also affects the namenode fail-over latency.
-If the size of name space is big, try increasing this.
+If the size of name space is big, try increasing this to 16 or higher.
   
 
 

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.1 updated: HDFS-15937. Reduce memory used during datanode layout upgrade. Contributed by Stephen O'Donnell (#2838)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.1 by this push:
 new 40ed9e1  HDFS-15937. Reduce memory used during datanode layout 
upgrade. Contributed by Stephen O'Donnell (#2838)
40ed9e1 is described below

commit 40ed9e1dc0824a5c4bd0b4e8ba05e42046c3d3d1
Author: Stephen O'Donnell 
AuthorDate: Thu Apr 8 11:59:02 2021 +0100

HDFS-15937. Reduce memory used during datanode layout upgrade. Contributed 
by Stephen O'Donnell (#2838)

(cherry picked from commit 4c567fcff7af45c75117ee4a75c087aa454a89e5)
---
 .../hadoop/hdfs/server/datanode/DataStorage.java   | 98 ++
 1 file changed, 62 insertions(+), 36 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index 2645e07..450b58c 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -1062,12 +1062,26 @@ public class DataStorage extends Storage {
   }
 
   private static class LinkArgs {
-File src;
-File dst;
+private File srcDir;
+private File dstDir;
+private String blockFile;
+
+LinkArgs(File srcDir, File dstDir, String blockFile) {
+  this.srcDir = srcDir;
+  this.dstDir = dstDir;
+  this.blockFile = blockFile;
+}
+
+public File src() {
+  return new File(srcDir, blockFile);
+}
 
-LinkArgs(File src, File dst) {
-  this.src = src;
-  this.dst = dst;
+public File dst() {
+  return new File(dstDir, blockFile);
+}
+
+public String blockFile() {
+  return blockFile;
 }
   }
 
@@ -1093,8 +1107,9 @@ public class DataStorage extends Storage {
 }
 
 final ArrayList idBasedLayoutSingleLinks = Lists.newArrayList();
-linkBlocksHelper(from, to, oldLV, hl, upgradeToIdBasedLayout, to,
-idBasedLayoutSingleLinks);
+final Map pathCache = new HashMap<>();
+linkBlocksHelper(from, to, hl, upgradeToIdBasedLayout, to,
+idBasedLayoutSingleLinks, pathCache);
 
 // Detect and remove duplicate entries.
 final ArrayList duplicates =
@@ -1120,7 +1135,7 @@ public class DataStorage extends Storage {
   idBasedLayoutSingleLinks.size());
   for (int j = iCopy; j < upperBound; j++) {
 LinkArgs cur = idBasedLayoutSingleLinks.get(j);
-HardLink.createHardLink(cur.src, cur.dst);
+HardLink.createHardLink(cur.src(), cur.dst());
   }
   return null;
 }
@@ -1153,9 +1168,9 @@ public class DataStorage extends Storage {
   @Override
   public int compare(LinkArgs a, LinkArgs b) {
 return ComparisonChain.start().
-compare(a.src.getName(), b.src.getName()).
-compare(a.src, b.src).
-compare(a.dst, b.dst).
+compare(a.blockFile(), b.blockFile()).
+compare(a.src(), b.src()).
+compare(a.dst(), b.dst()).
 result();
   }
 });
@@ -1165,8 +1180,8 @@ public class DataStorage extends Storage {
 boolean addedPrev = false;
 for (int i = 0; i < all.size(); i++) {
   LinkArgs args = all.get(i);
-  long blockId = Block.getBlockId(args.src.getName());
-  boolean isMeta = Block.isMetaFilename(args.src.getName());
+  long blockId = Block.getBlockId(args.blockFile());
+  boolean isMeta = Block.isMetaFilename(args.blockFile());
   if ((prevBlockId == null) ||
   (prevBlockId.longValue() != blockId)) {
 prevBlockId = blockId;
@@ -1205,10 +1220,10 @@ public class DataStorage extends Storage {
 TreeMap> highestGenstamps =
 new TreeMap>();
 for (LinkArgs duplicate : duplicates) {
-  if (!Block.isMetaFilename(duplicate.src.getName())) {
+  if (!Block.isMetaFilename(duplicate.blockFile())) {
 continue;
   }
-  long blockId = Block.getBlockId(duplicate.src.getName());
+  long blockId = Block.getBlockId(duplicate.blockFile());
   List prevHighest = highestGenstamps.get(blockId);
   if (prevHighest == null) {
 List highest = new LinkedList();
@@ -1217,8 +1232,8 @@ public class DataStorage extends Storage {
 continue;
   }
   long prevGenstamp =
-  Block.getGenerationStamp(prevHighest.get(0).src.getName());
-  long genstamp = Block.getGenerationStamp(duplicate.src.getName());
+  Block.getGenerationStamp(prevHighest.get(0).blockFile());
+  long genstamp = Block.getGenerationStamp(duplicate.blockFile());
   if (genstamp < prevGenstamp) {
 continue;
   }
@@ -1232,19 +12

[hadoop] branch branch-3.2 updated: HDFS-15937. Reduce memory used during datanode layout upgrade. Contributed by Stephen O'Donnell (#2838)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new 3422691  HDFS-15937. Reduce memory used during datanode layout 
upgrade. Contributed by Stephen O'Donnell (#2838)
3422691 is described below

commit 3422691fb261f2fe518e7c5bb39fb8d4a3047edb
Author: Stephen O'Donnell 
AuthorDate: Thu Apr 8 11:59:02 2021 +0100

HDFS-15937. Reduce memory used during datanode layout upgrade. Contributed 
by Stephen O'Donnell (#2838)

(cherry picked from commit 4c567fcff7af45c75117ee4a75c087aa454a89e5)
---
 .../hadoop/hdfs/server/datanode/DataStorage.java   | 98 ++
 1 file changed, 62 insertions(+), 36 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index 2645e07..450b58c 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -1062,12 +1062,26 @@ public class DataStorage extends Storage {
   }
 
   private static class LinkArgs {
-File src;
-File dst;
+private File srcDir;
+private File dstDir;
+private String blockFile;
+
+LinkArgs(File srcDir, File dstDir, String blockFile) {
+  this.srcDir = srcDir;
+  this.dstDir = dstDir;
+  this.blockFile = blockFile;
+}
+
+public File src() {
+  return new File(srcDir, blockFile);
+}
 
-LinkArgs(File src, File dst) {
-  this.src = src;
-  this.dst = dst;
+public File dst() {
+  return new File(dstDir, blockFile);
+}
+
+public String blockFile() {
+  return blockFile;
 }
   }
 
@@ -1093,8 +1107,9 @@ public class DataStorage extends Storage {
 }
 
 final ArrayList idBasedLayoutSingleLinks = Lists.newArrayList();
-linkBlocksHelper(from, to, oldLV, hl, upgradeToIdBasedLayout, to,
-idBasedLayoutSingleLinks);
+final Map pathCache = new HashMap<>();
+linkBlocksHelper(from, to, hl, upgradeToIdBasedLayout, to,
+idBasedLayoutSingleLinks, pathCache);
 
 // Detect and remove duplicate entries.
 final ArrayList duplicates =
@@ -1120,7 +1135,7 @@ public class DataStorage extends Storage {
   idBasedLayoutSingleLinks.size());
   for (int j = iCopy; j < upperBound; j++) {
 LinkArgs cur = idBasedLayoutSingleLinks.get(j);
-HardLink.createHardLink(cur.src, cur.dst);
+HardLink.createHardLink(cur.src(), cur.dst());
   }
   return null;
 }
@@ -1153,9 +1168,9 @@ public class DataStorage extends Storage {
   @Override
   public int compare(LinkArgs a, LinkArgs b) {
 return ComparisonChain.start().
-compare(a.src.getName(), b.src.getName()).
-compare(a.src, b.src).
-compare(a.dst, b.dst).
+compare(a.blockFile(), b.blockFile()).
+compare(a.src(), b.src()).
+compare(a.dst(), b.dst()).
 result();
   }
 });
@@ -1165,8 +1180,8 @@ public class DataStorage extends Storage {
 boolean addedPrev = false;
 for (int i = 0; i < all.size(); i++) {
   LinkArgs args = all.get(i);
-  long blockId = Block.getBlockId(args.src.getName());
-  boolean isMeta = Block.isMetaFilename(args.src.getName());
+  long blockId = Block.getBlockId(args.blockFile());
+  boolean isMeta = Block.isMetaFilename(args.blockFile());
   if ((prevBlockId == null) ||
   (prevBlockId.longValue() != blockId)) {
 prevBlockId = blockId;
@@ -1205,10 +1220,10 @@ public class DataStorage extends Storage {
 TreeMap> highestGenstamps =
 new TreeMap>();
 for (LinkArgs duplicate : duplicates) {
-  if (!Block.isMetaFilename(duplicate.src.getName())) {
+  if (!Block.isMetaFilename(duplicate.blockFile())) {
 continue;
   }
-  long blockId = Block.getBlockId(duplicate.src.getName());
+  long blockId = Block.getBlockId(duplicate.blockFile());
   List prevHighest = highestGenstamps.get(blockId);
   if (prevHighest == null) {
 List highest = new LinkedList();
@@ -1217,8 +1232,8 @@ public class DataStorage extends Storage {
 continue;
   }
   long prevGenstamp =
-  Block.getGenerationStamp(prevHighest.get(0).src.getName());
-  long genstamp = Block.getGenerationStamp(duplicate.src.getName());
+  Block.getGenerationStamp(prevHighest.get(0).blockFile());
+  long genstamp = Block.getGenerationStamp(duplicate.blockFile());
   if (genstamp < prevGenstamp) {
 continue;
   }
@@ -1232,19 +12

[hadoop] branch branch-3.3 updated: HDFS-15937. Reduce memory used during datanode layout upgrade. Contributed by Stephen O'Donnell (#2838)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new ef95f7a  HDFS-15937. Reduce memory used during datanode layout 
upgrade. Contributed by Stephen O'Donnell (#2838)
ef95f7a is described below

commit ef95f7a96355e93d69eb9a7c9d9dd672974ff22e
Author: Stephen O'Donnell 
AuthorDate: Thu Apr 8 11:59:02 2021 +0100

HDFS-15937. Reduce memory used during datanode layout upgrade. Contributed 
by Stephen O'Donnell (#2838)

(cherry picked from commit 4c567fcff7af45c75117ee4a75c087aa454a89e5)
---
 .../hadoop/hdfs/server/datanode/DataStorage.java   | 98 ++
 1 file changed, 62 insertions(+), 36 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index e917b77..1d2f10f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -1064,12 +1064,26 @@ public class DataStorage extends Storage {
   }
 
   private static class LinkArgs {
-File src;
-File dst;
+private File srcDir;
+private File dstDir;
+private String blockFile;
+
+LinkArgs(File srcDir, File dstDir, String blockFile) {
+  this.srcDir = srcDir;
+  this.dstDir = dstDir;
+  this.blockFile = blockFile;
+}
+
+public File src() {
+  return new File(srcDir, blockFile);
+}
 
-LinkArgs(File src, File dst) {
-  this.src = src;
-  this.dst = dst;
+public File dst() {
+  return new File(dstDir, blockFile);
+}
+
+public String blockFile() {
+  return blockFile;
 }
   }
 
@@ -1095,8 +1109,9 @@ public class DataStorage extends Storage {
 }
 
 final ArrayList idBasedLayoutSingleLinks = Lists.newArrayList();
-linkBlocksHelper(from, to, oldLV, hl, upgradeToIdBasedLayout, to,
-idBasedLayoutSingleLinks);
+final Map pathCache = new HashMap<>();
+linkBlocksHelper(from, to, hl, upgradeToIdBasedLayout, to,
+idBasedLayoutSingleLinks, pathCache);
 
 // Detect and remove duplicate entries.
 final ArrayList duplicates =
@@ -1122,7 +1137,7 @@ public class DataStorage extends Storage {
   idBasedLayoutSingleLinks.size());
   for (int j = iCopy; j < upperBound; j++) {
 LinkArgs cur = idBasedLayoutSingleLinks.get(j);
-HardLink.createHardLink(cur.src, cur.dst);
+HardLink.createHardLink(cur.src(), cur.dst());
   }
   return null;
 }
@@ -1155,9 +1170,9 @@ public class DataStorage extends Storage {
   @Override
   public int compare(LinkArgs a, LinkArgs b) {
 return ComparisonChain.start().
-compare(a.src.getName(), b.src.getName()).
-compare(a.src, b.src).
-compare(a.dst, b.dst).
+compare(a.blockFile(), b.blockFile()).
+compare(a.src(), b.src()).
+compare(a.dst(), b.dst()).
 result();
   }
 });
@@ -1167,8 +1182,8 @@ public class DataStorage extends Storage {
 boolean addedPrev = false;
 for (int i = 0; i < all.size(); i++) {
   LinkArgs args = all.get(i);
-  long blockId = Block.getBlockId(args.src.getName());
-  boolean isMeta = Block.isMetaFilename(args.src.getName());
+  long blockId = Block.getBlockId(args.blockFile());
+  boolean isMeta = Block.isMetaFilename(args.blockFile());
   if ((prevBlockId == null) ||
   (prevBlockId.longValue() != blockId)) {
 prevBlockId = blockId;
@@ -1207,10 +1222,10 @@ public class DataStorage extends Storage {
 TreeMap> highestGenstamps =
 new TreeMap>();
 for (LinkArgs duplicate : duplicates) {
-  if (!Block.isMetaFilename(duplicate.src.getName())) {
+  if (!Block.isMetaFilename(duplicate.blockFile())) {
 continue;
   }
-  long blockId = Block.getBlockId(duplicate.src.getName());
+  long blockId = Block.getBlockId(duplicate.blockFile());
   List prevHighest = highestGenstamps.get(blockId);
   if (prevHighest == null) {
 List highest = new LinkedList();
@@ -1219,8 +1234,8 @@ public class DataStorage extends Storage {
 continue;
   }
   long prevGenstamp =
-  Block.getGenerationStamp(prevHighest.get(0).src.getName());
-  long genstamp = Block.getGenerationStamp(duplicate.src.getName());
+  Block.getGenerationStamp(prevHighest.get(0).blockFile());
+  long genstamp = Block.getGenerationStamp(duplicate.blockFile());
   if (genstamp < prevGenstamp) {
 continue;
   }
@@ -1234,19 +12

[hadoop] branch trunk updated (ca9aa91 -> 4c567fc)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git.


from ca9aa91  YARN-10564. Support Auto Queue Creation template 
configurations. Contributed by Andras Gyori.
 add 4c567fc  HDFS-15937. Reduce memory used during datanode layout 
upgrade. Contributed by Stephen O'Donnell (#2838)

No new revisions were added by this update.

Summary of changes:
 .../hadoop/hdfs/server/datanode/DataStorage.java   | 98 ++
 1 file changed, 62 insertions(+), 36 deletions(-)

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.3 updated: HADOOP-17222. Create socket address leveraging URI cache (#2817)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 56ef164  HADOOP-17222. Create socket address leveraging URI cache 
(#2817)
56ef164 is described below

commit 56ef16468adfa830422e64a0d8311fa476505d0c
Author: Stephen O'Donnell 
AuthorDate: Tue Mar 30 11:59:44 2021 +0100

HADOOP-17222. Create socket address leveraging URI cache (#2817)

Contributed by fanrui.

Signed-off-by: Mingliang Liu 
Signed-off-by: He Xiaoqiao 
---
 .../main/java/org/apache/hadoop/net/NetUtils.java  | 77 ++
 .../java/org/apache/hadoop/net/TestNetUtils.java   | 45 -
 .../apache/hadoop/security/TestSecurityUtil.java   | 10 +++
 .../org/apache/hadoop/hdfs/DFSInputStream.java |  4 +-
 .../hadoop/hdfs/client/HdfsClientConfigKeys.java   |  3 +
 .../hadoop/hdfs/client/impl/DfsClientConf.java | 52 ++-
 .../src/main/resources/hdfs-default.xml|  9 +++
 .../apache/hadoop/tools/TestHdfsConfigFields.java  |  1 +
 8 files changed, 168 insertions(+), 33 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
index 3715e3c..5ded4b4 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
@@ -39,6 +39,7 @@ import java.net.ConnectException;
 import java.nio.channels.SocketChannel;
 import java.nio.channels.UnresolvedAddressException;
 import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
@@ -46,6 +47,8 @@ import java.util.concurrent.ConcurrentHashMap;
 import javax.net.SocketFactory;
 
 import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.thirdparty.com.google.common.cache.Cache;
+import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
 import org.apache.commons.net.util.SubnetUtils;
 import org.apache.commons.net.util.SubnetUtils.SubnetInfo;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -178,11 +181,33 @@ public class NetUtils {
*include a port number
* @param configName the name of the configuration from which
*   target was loaded. This is used in the
-   *   exception message in the case that parsing fails. 
+   *   exception message in the case that parsing fails.
*/
   public static InetSocketAddress createSocketAddr(String target,
int defaultPort,
String configName) {
+return createSocketAddr(target, defaultPort, configName, false);
+  }
+
+  /**
+   * Create an InetSocketAddress from the given target string and
+   * default port. If the string cannot be parsed correctly, the
+   * configName parameter is used as part of the
+   * exception message, allowing the user to better diagnose
+   * the misconfiguration.
+   *
+   * @param target a string of either "host" or "host:port"
+   * @param defaultPort the default port if target does not
+   *include a port number
+   * @param configName the name of the configuration from which
+   *   target was loaded. This is used in the
+   *   exception message in the case that parsing fails.
+   * @param useCacheIfPresent Whether use cache when create URI
+   */
+  public static InetSocketAddress createSocketAddr(String target,
+   int defaultPort,
+   String configName,
+   boolean useCacheIfPresent) {
 String helpText = "";
 if (configName != null) {
   helpText = " (configuration property '" + configName + "')";
@@ -192,15 +217,8 @@ public class NetUtils {
   helpText);
 }
 target = target.trim();
-boolean hasScheme = target.contains("://");
-URI uri = null;
-try {
-  uri = hasScheme ? URI.create(target) : 
URI.create("dummyscheme://"+target);
-} catch (IllegalArgumentException e) {
-  throw new IllegalArgumentException(
-  "Does not contain a valid host:port authority: " + target + helpText
-  );
-}
+boolean hasScheme = target.contains("://");
+URI uri = createURI(target, hasScheme, helpText, useCacheIfPresent);
 
 String host = uri.getHost();
 int port = uri.getPort();
@@ -208,10 +226,9 @

[hadoop] branch branch-3.3 updated: HADOOP-16828. Zookeeper Delegation Token Manager fetch sequence number by batch. Contributed by Fengnan Li.

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 67d52af  HADOOP-16828. Zookeeper Delegation Token Manager fetch 
sequence number by batch. Contributed by Fengnan Li.
67d52af is described below

commit 67d52af225753207c63218e6da734980a1c3a453
Author: Xiaoyu Yao 
AuthorDate: Tue Jun 2 11:53:08 2020 -0700

HADOOP-16828. Zookeeper Delegation Token Manager fetch sequence number by 
batch. Contributed by Fengnan Li.

(cherry picked from commit 6288e15118fab65a9a1452898e639313c6996769)
---
 .../delegation/ZKDelegationTokenSecretManager.java | 54 --
 .../TestZKDelegationTokenSecretManager.java| 52 +
 2 files changed, 93 insertions(+), 13 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
index 6f65574..4013f5d 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
@@ -98,12 +98,16 @@ public abstract class 
ZKDelegationTokenSecretManager versionedValue = sharedCount.getVersionedValue();
-  if (sharedCount.trySetCount(versionedValue, versionedValue.getValue() + 
1)) {
-break;
+  if (sharedCount.trySetCount(
+  versionedValue, versionedValue.getValue() + batchSize)) {
+return versionedValue.getValue();
   }
 }
   }
 
   @Override
   protected int incrementDelegationTokenSeqNum() {
-try {
-  incrSharedCount(delTokSeqCounter);
-} catch (InterruptedException e) {
-  // The ExpirationThread is just finishing.. so dont do anything..
-  LOG.debug("Thread interrupted while performing token counter increment", 
e);
-  Thread.currentThread().interrupt();
-} catch (Exception e) {
-  throw new RuntimeException("Could not increment shared counter !!", e);
+// The secret manager will keep a local range of seq num which won't be
+// seen by peers, so only when the range is exhausted it will ask zk for
+// another range again
+if (currentSeqNum >= currentMaxSeqNum) {
+  try {
+// after a successful batch request, we can get the range starting 
point
+currentSeqNum = incrSharedCount(delTokSeqCounter, seqNumBatchSize);
+currentMaxSeqNum = currentSeqNum + seqNumBatchSize;
+LOG.info("Fetched new range of seq num, from {} to {} ",
+currentSeqNum+1, currentMaxSeqNum);
+  } catch (InterruptedException e) {
+// The ExpirationThread is just finishing.. so dont do anything..
+LOG.debug(
+"Thread interrupted while performing token counter increment", e);
+Thread.currentThread().interrupt();
+  } catch (Exception e) {
+throw new RuntimeException("Could not increment shared counter !!", e);
+  }
 }
-return delTokSeqCounter.getCount();
+
+return ++currentSeqNum;
   }
 
   @Override
@@ -603,7 +631,7 @@ public abstract class 
ZKDelegationTokenSecretManager token1 =
+(Token) tm1.createToken(
+UserGroupInformation.getCurrentUser(), "foo");
+Assert.assertNotNull(token1);
+AbstractDelegationTokenIdentifier id1 =
+tm1.getDelegationTokenSecretManager().decodeTokenIdentifier(token1);
+Assert.assertEquals(
+"Token seq should be the same", 1, id1.getSequenceNumber());
+Token token2 =
+(Token) tm1.createToken(
+UserGroupInformation.getCurrentUser(), "foo");
+Assert.assertNotNull(token2);
+AbstractDelegationTokenIdentifier id2 =
+tm1.getDelegationTokenSecretManager().decodeTokenIdentifier(token2);
+Assert.assertEquals(
+"Token seq should be the same", 2, id2.getSequenceNumber());
+
+tm2 = new DelegationTokenManager(conf, new Text("bla"));
+tm2.init();
+
+Token token3 =
+(Token) tm2.createToken(
+UserGroupInformation.getCurrentUser(), "foo");
+Assert.assertNotNull(token3);
+AbstractDelegationTokenIdentifier id3 =
+tm2.getDelegationTokenSecretManager().decodeTokenIdentifier(token3);
+Assert.assertEquals(
+"Token seq should be the same", 1001, id3.getSequenceNumber());
+Token token4 =
+(Token) tm2.createToken(
+UserGroupInformation.getCurrentUser(), "foo");
+Assert.assertNotNull(token4);
+AbstractDelegationTokenIde

[hadoop] branch branch-3.3 updated: HDFS-13639. SlotReleaser is not fast enough (#1885)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new b3e968d  HDFS-13639. SlotReleaser is not fast enough (#1885)
b3e968d is described below

commit b3e968d21eb4a597497507a5222cc81fa61fd5a5
Author: leosunli 
AuthorDate: Fri May 22 04:21:17 2020 +0800

HDFS-13639. SlotReleaser is not fast enough (#1885)

(cherry picked from commit be374faf429d28561dd9c582f5c55451213d89a4)
---
 .../hdfs/shortcircuit/DfsClientShmManager.java | 10 +++
 .../hdfs/shortcircuit/ShortCircuitCache.java   | 61 +++---
 .../hdfs/server/datanode/ShortCircuitRegistry.java |  5 ++
 .../hdfs/shortcircuit/TestShortCircuitCache.java   | 92 ++
 4 files changed, 155 insertions(+), 13 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java
index 4bd2b9b..7cb0c38 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java
@@ -497,4 +497,14 @@ public class DfsClientShmManager implements Closeable {
   public DomainSocketWatcher getDomainSocketWatcher() {
 return domainSocketWatcher;
   }
+
+  @VisibleForTesting
+  public int getShmNum() {
+int segments = 0;
+for (EndpointShmManager endpointShmManager : datanodes.values()) {
+  segments +=
+  endpointShmManager.notFull.size() + endpointShmManager.full.size();
+}
+return segments;
+  }
 }
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
index 40e687c..222cfe2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
@@ -22,6 +22,7 @@ import java.io.Closeable;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.net.SocketException;
 import java.nio.MappedByteBuffer;
 import java.util.HashMap;
 import java.util.Map;
@@ -181,25 +182,52 @@ public class ShortCircuitCache implements Closeable {
 
 @Override
 public void run() {
+  if (slot == null) {
+return;
+  }
   LOG.trace("{}: about to release {}", ShortCircuitCache.this, slot);
   final DfsClientShm shm = (DfsClientShm)slot.getShm();
   final DomainSocket shmSock = shm.getPeer().getDomainSocket();
   final String path = shmSock.getPath();
+  DataOutputStream out = null;
   boolean success = false;
-  try (DomainSocket sock = DomainSocket.connect(path);
-   DataOutputStream out = new DataOutputStream(
-   new BufferedOutputStream(sock.getOutputStream( {
-new Sender(out).releaseShortCircuitFds(slot.getSlotId());
-DataInputStream in = new DataInputStream(sock.getInputStream());
-ReleaseShortCircuitAccessResponseProto resp =
-ReleaseShortCircuitAccessResponseProto.parseFrom(
-PBHelperClient.vintPrefixed(in));
-if (resp.getStatus() != Status.SUCCESS) {
-  String error = resp.hasError() ? resp.getError() : "(unknown)";
-  throw new IOException(resp.getStatus().toString() + ": " + error);
+  int retries = 2;
+  try {
+while (retries > 0) {
+  try {
+if (domainSocket == null || !domainSocket.isOpen()) {
+  // we are running in single thread mode, no protection needed for
+  // domainSocket
+  domainSocket = DomainSocket.connect(path);
+}
+
+out = new DataOutputStream(
+new BufferedOutputStream(domainSocket.getOutputStream()));
+new Sender(out).releaseShortCircuitFds(slot.getSlotId());
+DataInputStream in =
+new DataInputStream(domainSocket.getInputStream());
+ReleaseShortCircuitAccessResponseProto resp =
+ReleaseShortCircuitAccessResponseProto
+.parseFrom(PBHelperClient.vintPrefixed(in));
+if (resp.getStatus() != Status.SUCCESS) {
+  String error = resp.hasError() ? resp.getError() : "(unknown)";
+  throw new IOException(resp.getStatus().toString() + ": " + 
error);
+}
+
+LOG.trace("

[hadoop] 02/02: HDFS-15093. RENAME.TO_TRASH is ignored When RENAME.OVERWRITE is specified. Contributed by Ayush Saxena.

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit b26d75ce37128dcfda37c52c8bfc8b465ce71d48
Author: Ayush Saxena 
AuthorDate: Sat May 23 18:38:32 2020 +0530

HDFS-15093. RENAME.TO_TRASH is ignored When RENAME.OVERWRITE is specified. 
Contributed by Ayush Saxena.

(cherry picked from commit e0ae232f669b2e2a6654cfacff22a090c462effc)
---
 .../ClientNamenodeProtocolTranslatorPB.java |  3 ++-
 ...lientNamenodeProtocolServerSideTranslatorPB.java |  3 ++-
 .../java/org/apache/hadoop/hdfs/TestDFSRename.java  | 21 +
 3 files changed, 25 insertions(+), 2 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index 35b52b5..6cdce59 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -609,7 +609,8 @@ public class ClientNamenodeProtocolTranslatorPB implements
   for (Rename option : options) {
 if (option == Rename.OVERWRITE) {
   overwrite = true;
-} else if (option == Rename.TO_TRASH) {
+}
+if (option == Rename.TO_TRASH) {
   toTrash = true;
 }
   }
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 9bd8248..e0afe00 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -688,7 +688,8 @@ public class ClientNamenodeProtocolServerSideTranslatorPB 
implements
 ArrayList optionList = new ArrayList();
 if(req.getOverwriteDest()) {
   optionList.add(Rename.OVERWRITE);
-} else if(req.hasMoveToTrash() && req.getMoveToTrash()) {
+}
+if (req.hasMoveToTrash() && req.getMoveToTrash()) {
   optionList.add(Rename.TO_TRASH);
 }
 
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRename.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRename.java
index e7002c3..fe2eee2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRename.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRename.java
@@ -30,7 +30,9 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.Options.Rename;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Test;
 
 public class TestDFSRename {
@@ -175,4 +177,23 @@ public class TestDFSRename {
   }
 }
   }
+
+  @Test
+  public void testRename2Options() throws Exception {
+try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(
+new HdfsConfiguration()).build()) {
+  cluster.waitActive();
+  final DistributedFileSystem dfs = cluster.getFileSystem();
+  Path path = new Path("/test");
+  dfs.mkdirs(path);
+  GenericTestUtils.LogCapturer auditLog =
+  GenericTestUtils.LogCapturer.captureLogs(FSNamesystem.auditLog);
+  dfs.rename(path, new Path("/dir1"),
+  new Rename[] {Rename.OVERWRITE, Rename.TO_TRASH});
+  String auditOut = auditLog.getOutput();
+  assertTrue("Rename should have both OVERWRITE and TO_TRASH "
+  + "flags at namenode but had only " + auditOut,
+  auditOut.contains("options=[OVERWRITE, TO_TRASH]"));
+}
+  }
 }

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.3 updated (91d229b -> b26d75c)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a change to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git.


from 91d229b  MAPREDUCE-7325. Intermediate data encryption is broken in 
LocalJobRunner. Contributed by Ahmed Hussein
 new 968c95b  HDFS-10792. RedundantEditLogInputStream should log caught 
exceptions. Contributed by Wei-Chiu Chuang.
 new b26d75c  HDFS-15093. RENAME.TO_TRASH is ignored When RENAME.OVERWRITE 
is specified. Contributed by Ayush Saxena.

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../ClientNamenodeProtocolTranslatorPB.java |  3 ++-
 ...lientNamenodeProtocolServerSideTranslatorPB.java |  3 ++-
 .../namenode/RedundantEditLogInputStream.java   |  4 +++-
 .../java/org/apache/hadoop/hdfs/TestDFSRename.java  | 21 +
 4 files changed, 28 insertions(+), 3 deletions(-)

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] 01/02: HDFS-10792. RedundantEditLogInputStream should log caught exceptions. Contributed by Wei-Chiu Chuang.

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit 968c95bfba3a6932ad68a2050fc7a8b6454a7010
Author: Ayush Saxena 
AuthorDate: Sun May 31 17:06:44 2020 +0530

HDFS-10792. RedundantEditLogInputStream should log caught exceptions. 
Contributed by Wei-Chiu Chuang.

(cherry picked from commit ae13a5ccbea10fe86481adbbff574c528e03c7f6)
---
 .../hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java
index 80e6b39..15f799a 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java
@@ -170,6 +170,7 @@ class RedundantEditLogInputStream extends 
EditLogInputStream {
   }
   return nextOp();
 } catch (IOException e) {
+  LOG.warn("encountered an exception", e);
   return null;
 }
   }
@@ -228,7 +229,8 @@ class RedundantEditLogInputStream extends 
EditLogInputStream {
   "streams are shorter than the current one!  The best " +
   "remaining edit log ends at transaction " +
   newLast + ", but we thought we could read up to transaction " +
-  oldLast + ".  If you continue, metadata will be lost forever!");
+  oldLast + ".  If you continue, metadata will be lost forever!",
+  prevException);
 }
 LOG.error("Got error reading edit log input stream " +
   streams[curIdx].getName() + "; failing over to edit log " +

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org