This is an automated email from the ASF dual-hosted git repository.
kihwal pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new 2861936 HDFS-14758. Make lease hard limit configurable and reduce the
default. Contributed by hemanthboyina.
2861936 is described below
commit 28619362288fbaf9c3757ee3ef770f575f7a6916
Author: Kihwal Lee <kih...@apache.org>
AuthorDate: Tue Feb 11 15:28:42 2020 -0600
HDFS-14758. Make lease hard limit configurable and reduce the default.
Contributed by hemanthboyina.
(cherry picked from commit 9b8a78d97bfd825ce840c6033371c7f10e49a5b8)
(cherry picked from commit f1840669cfc92c033521bf95989479523e2a649d)
(cherry picked from commit bf09bfa2db7c0790f72d6d7d357e0b80c9eff1f2)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
---
.../src/main/java/org/apache/hadoop/hdfs/DFSClient.java | 4 ++--
.../org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java | 3 +++
.../org/apache/hadoop/hdfs/client/impl/DfsClientConf.java | 12 ++++++++++++
.../java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java | 11 -----------
.../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +++++
.../hadoop/hdfs/server/common/HdfsServerConstants.java | 5 ++---
.../org/apache/hadoop/hdfs/server/namenode/LeaseManager.java | 7 ++++++-
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 8 ++++++++
.../test/java/org/apache/hadoop/hdfs/TestFileAppend4.java | 5 +++--
.../src/test/java/org/apache/hadoop/hdfs/TestLease.java | 5 +++--
.../test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java | 6 +++---
.../apache/hadoop/hdfs/server/namenode/TestFileTruncate.java | 3 ++-
.../snapshot/TestINodeFileUnderConstructionWithSnapshot.java | 5 +++--
13 files changed, 52 insertions(+), 27 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
old mode 100644
new mode 100755
index d5c8294..77ee893
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -565,10 +565,10 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
} catch (IOException e) {
// Abort if the lease has already expired.
final long elapsed = Time.monotonicNow() - getLastLeaseRenewal();
- if (elapsed > HdfsConstants.LEASE_HARDLIMIT_PERIOD) {
+ if (elapsed > dfsClientConf.getleaseHardLimitPeriod()) {
LOG.warn("Failed to renew lease for " + clientName + " for "
+ (elapsed/1000) + " seconds (>= hard-limit ="
- + (HdfsConstants.LEASE_HARDLIMIT_PERIOD / 1000) + " seconds.) "
+ + (dfsClientConf.getleaseHardLimitPeriod() / 1000) + " seconds.)
"
+ "Closing all files being written ...", e);
closeAllFilesBeingWritten(true);
} else {
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
old mode 100644
new mode 100755
index 44662d5..8122693
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -206,6 +206,9 @@ public interface HdfsClientConfigKeys {
"dfs.namenode.snapshot.capture.openfiles";
boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT = false;
+ String DFS_LEASE_HARDLIMIT_KEY = "dfs.namenode.lease-hard-limit-sec";
+ long DFS_LEASE_HARDLIMIT_DEFAULT = 20 * 60;
+
/**
* These are deprecated config keys to client code.
*/
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
index da97be5..df2fd48 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
@@ -141,6 +141,7 @@ public class DfsClientConf {
replicaAccessorBuilderClasses;
private final boolean dataTransferTcpNoDelay;
+ private final long leaseHardLimitPeriod;
public DfsClientConf(Configuration conf) {
// The hdfsTimeout is currently the same as the ipc timeout
@@ -264,6 +265,10 @@ public class DfsClientConf {
HdfsClientConfigKeys.HedgedRead.THREADPOOL_SIZE_DEFAULT);
replicaAccessorBuilderClasses = loadReplicaAccessorBuilderClasses(conf);
+
+ leaseHardLimitPeriod =
+ conf.getLong(HdfsClientConfigKeys.DFS_LEASE_HARDLIMIT_KEY,
+ HdfsClientConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000;
}
@SuppressWarnings("unchecked")
@@ -579,6 +584,13 @@ public class DfsClientConf {
}
/**
+ * @return the leaseHardLimitPeriod
+ */
+ public long getleaseHardLimitPeriod() {
+ return leaseHardLimitPeriod;
+ }
+
+ /**
* @return the replicaAccessorBuilderClasses
*/
public List<Class<? extends ReplicaAccessorBuilder>>
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
old mode 100644
new mode 100755
index bf64427..24d56805
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -104,17 +104,6 @@ public class HdfsConstants {
* lease, another client can preempt the lease.
*/
public static final long LEASE_SOFTLIMIT_PERIOD = 60 * 1000;
- /**
- * For a HDFS client to write to a file, a lease is granted; During the lease
- * period, no other client can write to the file. The writing client can
- * periodically renew the lease. When the file is closed, the lease is
- * revoked. The lease duration is bound by a
- * {@link HdfsConstants#LEASE_SOFTLIMIT_PERIOD soft limit} and this hard
- * limit. If after the hard limit expires and the client has failed to renew
- * the lease, HDFS assumes that the client has quit and will automatically
- * close the file on behalf of the writer, and recover the lease.
- */
- public static final long LEASE_HARDLIMIT_PERIOD = 60 *
LEASE_SOFTLIMIT_PERIOD;
// SafeMode actions
public enum SafeModeAction {
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
old mode 100644
new mode 100755
index e0b3a77..bde8677
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -1455,4 +1455,9 @@ public class DFSConfigKeys extends
CommonConfigurationKeys {
@Deprecated
public static final long DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT =
HdfsClientConfigKeys.DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT;
+
+ public static final String DFS_LEASE_HARDLIMIT_KEY =
+ HdfsClientConfigKeys.DFS_LEASE_HARDLIMIT_KEY;
+ public static final long DFS_LEASE_HARDLIMIT_DEFAULT =
+ HdfsClientConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT;
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
index 69a07f9..c33c0dd 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
@@ -45,11 +45,10 @@ public interface HdfsServerConstants {
int MIN_BLOCKS_FOR_WRITE = 1;
/**
- * Please see {@link HdfsConstants#LEASE_SOFTLIMIT_PERIOD} and
- * {@link HdfsConstants#LEASE_HARDLIMIT_PERIOD} for more information.
+ * Please see {@link HdfsConstants#LEASE_SOFTLIMIT_PERIOD}
+ * for more information.
*/
long LEASE_SOFTLIMIT_PERIOD = HdfsConstants.LEASE_SOFTLIMIT_PERIOD;
- long LEASE_HARDLIMIT_PERIOD = HdfsConstants.LEASE_HARDLIMIT_PERIOD;
long LEASE_RECOVER_PERIOD = 10 * 1000; // in ms
// We need to limit the length and depth of a path in the filesystem.
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index 1685a4f..9b851f8 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -40,7 +40,9 @@ import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
@@ -81,7 +83,7 @@ public class LeaseManager {
private final FSNamesystem fsnamesystem;
private long softLimit = HdfsConstants.LEASE_SOFTLIMIT_PERIOD;
- private long hardLimit = HdfsConstants.LEASE_HARDLIMIT_PERIOD;
+ private long hardLimit;
private long lastHolderUpdateTime;
private String internalLeaseHolder;
static final int INODE_FILTER_WORKER_COUNT_MAX = 4;
@@ -111,7 +113,10 @@ public class LeaseManager {
private volatile boolean shouldRunMonitor;
LeaseManager(FSNamesystem fsnamesystem) {
+ Configuration conf = new Configuration();
this.fsnamesystem = fsnamesystem;
+ this.hardLimit = conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY,
+ DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000;
updateInternalLeaseHolder();
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 2fce9a5..06f48f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4681,4 +4681,12 @@
inter-DN QOP.
</description>
</property>
+
+ <property>
+ <name>dfs.namenode.lease-hard-limit-sec</name>
+ <value>1200</value>
+ <description>
+ Determines the namenode automatic lease recovery interval in seconds.
+ </description>
+ </property>
</configuration>
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
old mode 100644
new mode 100755
index 01141ef..8e9fd00
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
@@ -41,7 +41,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
@@ -108,7 +107,9 @@ public class TestFileAppend4 {
// set the soft limit to be 1 second so that the
// namenode triggers lease recovery upon append request
- cluster.setLeasePeriod(1000, HdfsConstants.LEASE_HARDLIMIT_PERIOD);
+ cluster.setLeasePeriod(1,
+ conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY,
+ DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT));
// Trying recovery
int tries = 60;
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
old mode 100644
new mode 100755
index bacdc99..6fc2c66
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
@@ -114,9 +114,10 @@ public class TestLease {
Assert.fail("Write failed.");
}
+ long hardlimit = conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY,
+ DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000;
// make it look like the hard limit has been exceeded.
- dfs.lastLeaseRenewal = Time.monotonicNow()
- - HdfsConstants.LEASE_HARDLIMIT_PERIOD - 1000;
+ dfs.lastLeaseRenewal = Time.monotonicNow() - hardlimit - 1000;
dfs.renewLease();
// this should not work.
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
old mode 100644
new mode 100755
index a807bbb..85eaf2b
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
@@ -341,10 +341,10 @@ public class TestLeaseRecovery2 {
Map<String, String []> u2g_map = new HashMap<String, String []>(1);
u2g_map.put(fakeUsername, new String[] {fakeGroup});
DFSTestUtil.updateConfWithFakeGroupMapping(conf, u2g_map);
-
+ long hardlimit = conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY,
+ DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000;
// Reset default lease periods
- cluster.setLeasePeriod(HdfsConstants.LEASE_SOFTLIMIT_PERIOD,
- HdfsConstants.LEASE_HARDLIMIT_PERIOD);
+ cluster.setLeasePeriod(HdfsConstants.LEASE_SOFTLIMIT_PERIOD, hardlimit);
//create a file
// create a random file name
String filestr = "/foo" + AppendTestUtil.nextInt();
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
old mode 100644
new mode 100755
index 7e3e1d6..9e963cf
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -658,7 +658,8 @@ public class TestFileTruncate {
NameNodeAdapter.getLeaseManager(cluster.getNamesystem())
.setLeasePeriod(HdfsConstants.LEASE_SOFTLIMIT_PERIOD,
- HdfsConstants.LEASE_HARDLIMIT_PERIOD);
+ conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY,
+ DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000);
checkFullFile(p, newLength, contents);
fs.delete(p, false);
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java
old mode 100644
new mode 100755
index 65b2f8c..be67c68
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java
@@ -304,7 +304,8 @@ public class TestINodeFileUnderConstructionWithSnapshot {
}
} finally {
NameNodeAdapter.setLeasePeriod(fsn, HdfsConstants.LEASE_SOFTLIMIT_PERIOD,
- HdfsConstants.LEASE_HARDLIMIT_PERIOD);
+ conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY,
+ DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000);
}
}
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
