Author: todd
Date: Thu Jan 30 21:21:15 2014
New Revision: 1562970
URL: http://svn.apache.org/r1562970
Log:
HDFS-5790. LeaseManager.findPath is very slow when many leases need recovery.
Contributed by Todd Lipcon.
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1562970&r1=1562969&r2=1562970&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Jan 30
21:21:15 2014
@@ -299,6 +299,9 @@ Release 2.4.0 - UNRELEASED
OPTIMIZATIONS
+ HDFS-5790. LeaseManager.findPath is very slow when many leases need
recovery
+ (todd)
+
BUG FIXES
HDFS-5492. Port HDFS-2069 (Incorrect default trash interval in the
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1562970&r1=1562969&r2=1562970&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Thu Jan 30 21:21:15 2014
@@ -3800,7 +3800,7 @@ public class FSNamesystem implements Nam
final long diff = fileINode.getPreferredBlockSize() -
commitBlock.getNumBytes();
if (diff > 0) {
try {
- String path = leaseManager.findPath(fileINode);
+ String path = fileINode.getFullPathName();
dir.updateSpaceConsumed(path, 0, -diff*fileINode.getFileReplication());
} catch (IOException e) {
LOG.warn("Unexpected exception while updating disk space.", e);
@@ -4002,7 +4002,7 @@ public class FSNamesystem implements Nam
@VisibleForTesting
String closeFileCommitBlocks(INodeFile pendingFile, BlockInfo storedBlock)
throws IOException {
- String src = leaseManager.findPath(pendingFile);
+ String src = pendingFile.getFullPathName();
// commit the last block and complete it if it has minimum replicas
commitOrCompleteLastBlock(pendingFile, storedBlock);
@@ -4024,7 +4024,7 @@ public class FSNamesystem implements Nam
@VisibleForTesting
String persistBlocks(INodeFile pendingFile, boolean logRetryCache)
throws IOException {
- String src = leaseManager.findPath(pendingFile);
+ String src = pendingFile.getFullPathName();
dir.persistBlocks(src, pendingFile, logRetryCache);
return src;
}
@@ -5952,7 +5952,7 @@ public class FSNamesystem implements Nam
.getDatanodeStorageInfos(newNodes, newStorageIDs);
blockinfo.setExpectedLocations(storages);
- String src = leaseManager.findPath(pendingFile);
+ String src = pendingFile.getFullPathName();
dir.persistBlocks(src, pendingFile, logRetryCache);
}
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java?rev=1562970&r1=1562969&r2=1562970&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
Thu Jan 30 21:21:15 2014
@@ -180,24 +180,6 @@ public class LeaseManager {
}
/**
- * Finds the pathname for the specified pendingFile
- */
- public synchronized String findPath(INodeFile pendingFile)
- throws IOException {
- FileUnderConstructionFeature uc =
pendingFile.getFileUnderConstructionFeature();
- Preconditions.checkArgument(uc != null);
- Lease lease = getLease(uc.getClientName());
- if (lease != null) {
- String src = lease.findPath(pendingFile);
- if (src != null) {
- return src;
- }
- }
- throw new IOException("pendingFile (=" + pendingFile + ") not found."
- + "(lease=" + lease + ")");
- }
-
- /**
* Renew the lease(s) held by the given client
*/
synchronized void renewLease(String holder) {
@@ -252,24 +234,6 @@ public class LeaseManager {
return now() - lastUpdate > softLimit;
}
- /**
- * @return the path associated with the pendingFile and null if not found.
- */
- private String findPath(INodeFile pendingFile) {
- try {
- for (String src : paths) {
- INode node = fsnamesystem.dir.getINode(src);
- if (node == pendingFile
- || (node.isFile() && node.asFile() == pendingFile)) {
- return src;
- }
- }
- } catch (UnresolvedLinkException e) {
- throw new AssertionError("Lease files should reside on this FS");
- }
- return null;
- }
-
/** Does this lease contain any path? */
boolean hasPath() {return !paths.isEmpty();}
