Author: bobby
Date: Wed Nov 14 19:21:25 2012
New Revision: 1409311
URL: http://svn.apache.org/viewvc?rev=1409311&view=rev
Log:
HDFS-4182. SecondaryNameNode leaks NameCache entries (bobby)
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameCache.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameCache.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1409311&r1=1409310&r2=1409311&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Nov 14
19:21:25 2012
@@ -2006,6 +2006,8 @@ Release 0.23.5 - UNRELEASED
HDFS-4172. namenode does not URI-encode parameters when building URI for
datanode request (Derek Dagit via bobby)
+ HDFS-4182. SecondaryNameNode leaks NameCache entries (bobby)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1409311&r1=1409310&r2=1409311&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Wed Nov 14 19:21:25 2012
@@ -61,6 +61,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory.INodesInPath;
import org.apache.hadoop.hdfs.util.ByteArray;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
/*************************************************
@@ -175,6 +176,12 @@ public class FSDirectory implements Clos
writeUnlock();
}
}
+
+ //This is for testing purposes only
+ @VisibleForTesting
+ boolean isReady() {
+ return ready;
+ }
// exposed for unit tests
protected void setReady(boolean flag) {
@@ -1981,9 +1988,16 @@ public class FSDirectory implements Clos
* Reset the entire namespace tree.
*/
void reset() {
- rootDir = new INodeDirectoryWithQuota(INodeDirectory.ROOT_NAME,
- getFSNamesystem().createFsOwnerPermissions(new
FsPermission((short)0755)),
- Integer.MAX_VALUE, -1);
+ writeLock();
+ try {
+ setReady(false);
+ rootDir = new INodeDirectoryWithQuota(INodeDirectory.ROOT_NAME,
+ getFSNamesystem().createFsOwnerPermissions(new
FsPermission((short)0755)),
+ Integer.MAX_VALUE, -1);
+ nameCache.reset();
+ } finally {
+ writeUnlock();
+ }
}
/**
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameCache.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameCache.java?rev=1409311&r1=1409310&r2=1409311&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameCache.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameCache.java
Wed Nov 14 19:21:25 2012
@@ -152,4 +152,14 @@ class NameCache<K> {
cache.put(name, name);
lookups += useThreshold;
}
+
+ public void reset() {
+ initialized = false;
+ cache.clear();
+ if (transientMap == null) {
+ transientMap = new HashMap<K, UseCount>();
+ } else {
+ transientMap.clear();
+ }
+ }
}
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1409311&r1=1409310&r2=1409311&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
Wed Nov 14 19:21:25 2012
@@ -886,6 +886,7 @@ public class SecondaryNameNode implement
"just been downloaded");
}
dstImage.reloadFromImageFile(file, dstNamesystem);
+ dstNamesystem.dir.imageLoadComplete();
}
Checkpointer.rollForwardByApplyingLogs(manifest, dstImage, dstNamesystem);
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java?rev=1409311&r1=1409310&r2=1409311&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java
Wed Nov 14 19:21:25 2012
@@ -81,7 +81,6 @@ public class TestFSDirectory {
DFSTestUtil.createFile(hdfs, file5, 1024, REPLICATION, seed);
hdfs.mkdirs(sub2);
-
}
@After
@@ -130,6 +129,16 @@ public class TestFSDirectory {
Assert.assertTrue(diff.contains(file4.getName()));
}
+ @Test
+ public void testReset() throws Exception {
+ fsdir.reset();
+ Assert.assertFalse(fsdir.isReady());
+ final INodeDirectory root = (INodeDirectory) fsdir.getINode("/");
+ Assert.assertNull(root.getChildren());
+ fsdir.imageLoadComplete();
+ Assert.assertTrue(fsdir.isReady());
+ }
+
static void checkClassName(String line) {
int i = line.lastIndexOf('(');
int j = line.lastIndexOf('@');
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameCache.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameCache.java?rev=1409311&r1=1409310&r2=1409311&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameCache.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameCache.java
Wed Nov 14 19:21:25 2012
@@ -58,6 +58,17 @@ public class TestNameCache {
for (String s : notMatching) {
verifyNameReuse(cache, s, false);
}
+
+ cache.reset();
+ cache.initialized();
+
+ for (String s : matching) {
+ verifyNameReuse(cache, s, false);
+ }
+
+ for (String s : notMatching) {
+ verifyNameReuse(cache, s, false);
+ }
}
private void verifyNameReuse(NameCache<String> cache, String s, boolean
reused) {
