This is an automated email from the ASF dual-hosted git repository.
kihwal pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9b8a78d HDFS-14758. Make lease hard limit configurable and reduce the
default. Contributed by hemanthboyina.
9b8a78d is described below
commit 9b8a78d97bfd825ce840c6033371c7f10e49a5b8
Author: Kihwal Lee <kih...@apache.org>
AuthorDate: Tue Feb 11 12:40:00 2020 -0600
HDFS-14758. Make lease hard limit configurable and reduce the default.
Contributed by hemanthboyina.
---
.../src/main/java/org/apache/hadoop/hdfs/DFSClient.java | 4 ++--
.../org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java | 3 +++
.../org/apache/hadoop/hdfs/client/impl/DfsClientConf.java | 12 ++++++++++++
.../java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java | 11 -----------
.../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 ++++++
.../org/apache/hadoop/hdfs/server/namenode/LeaseManager.java | 7 ++++++-
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 8 ++++++++
.../test/java/org/apache/hadoop/hdfs/TestFileAppend4.java | 5 +++--
.../src/test/java/org/apache/hadoop/hdfs/TestLease.java | 5 +++--
.../test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java | 6 +++---
.../apache/hadoop/hdfs/server/namenode/TestFileTruncate.java | 3 ++-
.../snapshot/TestINodeFileUnderConstructionWithSnapshot.java | 3 ++-
12 files changed, 50 insertions(+), 23 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
old mode 100644
new mode 100755
index e0eaa19..0309cbd
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -577,10 +577,10 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
} catch (IOException e) {
// Abort if the lease has already expired.
final long elapsed = Time.monotonicNow() - getLastLeaseRenewal();
- if (elapsed > HdfsConstants.LEASE_HARDLIMIT_PERIOD) {
+ if (elapsed > dfsClientConf.getleaseHardLimitPeriod()) {
LOG.warn("Failed to renew lease for " + clientName + " for "
+ (elapsed/1000) + " seconds (>= hard-limit ="
- + (HdfsConstants.LEASE_HARDLIMIT_PERIOD / 1000) + " seconds.) "
+ + (dfsClientConf.getleaseHardLimitPeriod() / 1000) + " seconds.)
"
+ "Closing all files being written ...", e);
closeAllFilesBeingWritten(true);
} else {
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
old mode 100644
new mode 100755
index 38f0016..228d7ac
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -242,6 +242,9 @@ public interface HdfsClientConfigKeys {
String DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS =
"dfs.provided.aliasmap.inmemory.dnrpc-address";
+ String DFS_LEASE_HARDLIMIT_KEY = "dfs.namenode.lease-hard-limit-sec";
+ long DFS_LEASE_HARDLIMIT_DEFAULT = 20 * 60;
+
/**
* These are deprecated config keys to client code.
*/
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
index d6194f4..07f0eee 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
@@ -151,6 +151,7 @@ public class DfsClientConf {
private final boolean dataTransferTcpNoDelay;
private final boolean deadNodeDetectionEnabled;
+ private final long leaseHardLimitPeriod;
public DfsClientConf(Configuration conf) {
// The hdfsTimeout is currently the same as the ipc timeout
@@ -285,6 +286,10 @@ public class DfsClientConf {
HdfsClientConfigKeys.StripedRead.THREADPOOL_SIZE_KEY +
" must be greater than 0.");
replicaAccessorBuilderClasses = loadReplicaAccessorBuilderClasses(conf);
+
+ leaseHardLimitPeriod =
+ conf.getLong(HdfsClientConfigKeys.DFS_LEASE_HARDLIMIT_KEY,
+ HdfsClientConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000;
}
@SuppressWarnings("unchecked")
@@ -619,6 +624,13 @@ public class DfsClientConf {
}
/**
+ * @return the leaseHardLimitPeriod
+ */
+ public long getleaseHardLimitPeriod() {
+ return leaseHardLimitPeriod;
+ }
+
+ /**
* @return the replicaAccessorBuilderClasses
*/
public List<Class<? extends ReplicaAccessorBuilder>>
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
old mode 100644
new mode 100755
index c449a2e..eb3dc4e
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -116,17 +116,6 @@ public final class HdfsConstants {
* lease, another client can preempt the lease.
*/
public static final long LEASE_SOFTLIMIT_PERIOD = 60 * 1000;
- /**
- * For a HDFS client to write to a file, a lease is granted; During the lease
- * period, no other client can write to the file. The writing client can
- * periodically renew the lease. When the file is closed, the lease is
- * revoked. The lease duration is bound by a
- * {@link HdfsConstants#LEASE_SOFTLIMIT_PERIOD soft limit} and this hard
- * limit. If after the hard limit expires and the client has failed to renew
- * the lease, HDFS assumes that the client has quit and will automatically
- * close the file on behalf of the writer, and recover the lease.
- */
- public static final long LEASE_HARDLIMIT_PERIOD = 60 *
LEASE_SOFTLIMIT_PERIOD;
// SafeMode actions
public enum SafeModeAction {
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
old mode 100644
new mode 100755
index f6105c2..f0f8f3e
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -1741,4 +1741,10 @@ public class DFSConfigKeys extends
CommonConfigurationKeys {
@Deprecated
public static final long DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT =
HdfsClientConfigKeys.DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT;
+
+ public static final String DFS_LEASE_HARDLIMIT_KEY =
+ HdfsClientConfigKeys.DFS_LEASE_HARDLIMIT_KEY;
+ public static final long DFS_LEASE_HARDLIMIT_DEFAULT =
+ HdfsClientConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT;
+
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index 2efadfc..c3624ce 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -41,7 +41,9 @@ import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
@@ -84,7 +86,7 @@ public class LeaseManager {
.getName());
private final FSNamesystem fsnamesystem;
private long softLimit = HdfsConstants.LEASE_SOFTLIMIT_PERIOD;
- private long hardLimit = HdfsConstants.LEASE_HARDLIMIT_PERIOD;
+ private long hardLimit;
static final int INODE_FILTER_WORKER_COUNT_MAX = 4;
static final int INODE_FILTER_WORKER_TASK_MIN = 512;
private long lastHolderUpdateTime;
@@ -112,7 +114,10 @@ public class LeaseManager {
private volatile boolean shouldRunMonitor;
LeaseManager(FSNamesystem fsnamesystem) {
+ Configuration conf = new Configuration();
this.fsnamesystem = fsnamesystem;
+ this.hardLimit = conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY,
+ DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000;
updateInternalLeaseHolder();
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 0897939..5daaffb 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -5745,4 +5745,12 @@
octal or symbolic.
</description>
</property>
+
+ <property>
+ <name>dfs.namenode.lease-hard-limit-sec</name>
+ <value>1200</value>
+ <description>
+ Determines the namenode automatic lease recovery interval in seconds.
+ </description>
+ </property>
</configuration>
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
old mode 100644
new mode 100755
index 5df7c7f..63e7eb6
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
@@ -107,7 +106,9 @@ public class TestFileAppend4 {
// set the soft limit to be 1 second so that the
// namenode triggers lease recovery upon append request
- cluster.setLeasePeriod(1000, HdfsConstants.LEASE_HARDLIMIT_PERIOD);
+ cluster.setLeasePeriod(1,
+ conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY,
+ DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT));
// Trying recovery
int tries = 60;
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
old mode 100644
new mode 100755
index 909a072..5d7b62a
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
@@ -111,9 +111,10 @@ public class TestLease {
Assert.fail("Write failed.");
}
+ long hardlimit = conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY,
+ DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000;
// make it look like the hard limit has been exceeded.
- dfs.lastLeaseRenewal = Time.monotonicNow()
- - HdfsConstants.LEASE_HARDLIMIT_PERIOD - 1000;
+ dfs.lastLeaseRenewal = Time.monotonicNow() - hardlimit - 1000;
dfs.renewLease();
// this should not work.
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
old mode 100644
new mode 100755
index bcd3cdc..662b0e6
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
@@ -407,10 +407,10 @@ public class TestLeaseRecovery2 {
Map<String, String []> u2g_map = new HashMap<String, String []>(1);
u2g_map.put(fakeUsername, new String[] {fakeGroup});
DFSTestUtil.updateConfWithFakeGroupMapping(conf, u2g_map);
-
+ long hardlimit = conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY,
+ DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000;
// Reset default lease periods
- cluster.setLeasePeriod(HdfsConstants.LEASE_SOFTLIMIT_PERIOD,
- HdfsConstants.LEASE_HARDLIMIT_PERIOD);
+ cluster.setLeasePeriod(HdfsConstants.LEASE_SOFTLIMIT_PERIOD, hardlimit);
//create a file
// create a random file name
String filestr = "/foo" + AppendTestUtil.nextInt();
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
old mode 100644
new mode 100755
index f108387..a1197fb9
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -659,7 +659,8 @@ public class TestFileTruncate {
NameNodeAdapter.getLeaseManager(cluster.getNamesystem())
.setLeasePeriod(HdfsConstants.LEASE_SOFTLIMIT_PERIOD,
- HdfsConstants.LEASE_HARDLIMIT_PERIOD);
+ conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY,
+ DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000);
checkFullFile(p, newLength, contents);
fs.delete(p, false);
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java
old mode 100644
new mode 100755
index 77c9ecc..4f32af9
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java
@@ -304,7 +304,8 @@ public class TestINodeFileUnderConstructionWithSnapshot {
}
} finally {
NameNodeAdapter.setLeasePeriod(fsn, HdfsConstants.LEASE_SOFTLIMIT_PERIOD,
- HdfsConstants.LEASE_HARDLIMIT_PERIOD);
+ conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY,
+ DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
