Author: kihwal
Date: Mon Apr 7 18:28:30 2014
New Revision: 1585546
URL: http://svn.apache.org/r1585546
Log:
HDFS-6191. Disable quota checks when replaying edit log. Contributed by Kihwal
Lee.
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1585546&r1=1585545&r2=1585546&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Mon Apr 7 18:28:30 2014
@@ -11,6 +11,8 @@ Release 0.23.11 - UNRELEASED
HDFS-4461. DirectoryScanner: volume path prefix takes up memory for every
block that is scanned (Colin Patrick McCabe)
+ HDFS-6191. Disable quota checks when replaying edit log. (kihwal)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java?rev=1585546&r1=1585545&r2=1585546&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
Mon Apr 7 18:28:30 2014
@@ -123,6 +123,7 @@ public class BackupNode extends NameNode
protected void loadNamesystem(Configuration conf) throws IOException {
BackupImage bnImage = new BackupImage(conf);
this.namesystem = new FSNamesystem(conf, bnImage);
+ namesystem.dir.disableQuotaChecks();
bnImage.recoverCreateRead();
}
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1585546&r1=1585545&r2=1585546&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Mon Apr 7 18:28:30 2014
@@ -77,6 +77,7 @@ public class FSDirectory implements Clos
INodeDirectoryWithQuota rootDir;
FSImage fsImage;
private volatile boolean ready = false;
+ private volatile boolean skipQuotaCheck = false; // disable while consuming
edits
private static final long UNKNOWN_DISK_SPACE = -1;
private final int maxComponentLength;
private final int maxDirItems;
@@ -267,6 +268,16 @@ public class FSDirectory implements Clos
}
}
+ /** Enable quota verification */
+ void enableQuotaChecks() {
+ skipQuotaCheck = false;
+ }
+
+ /** Disable quota verification */
+ void disableQuotaChecks() {
+ skipQuotaCheck = true;
+ }
+
/**
* Add the given filename to the fs.
* @throws QuotaExceededException
@@ -1753,7 +1764,7 @@ public class FSDirectory implements Clos
*/
private void verifyQuota(INode[] inodes, int pos, long nsDelta, long dsDelta,
INode commonAncestor) throws QuotaExceededException {
- if (!ready) {
+ if (!ready || skipQuotaCheck) {
// Do not check quota if edits log is still being processed
return;
}
@@ -1794,7 +1805,7 @@ public class FSDirectory implements Clos
*/
private void verifyQuotaForRename(INode[] srcInodes, INode[]dstInodes)
throws QuotaExceededException {
- if (!ready) {
+ if (!ready || skipQuotaCheck) {
// Do not check quota if edits log is still being processed
return;
}
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1585546&r1=1585545&r2=1585546&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Mon Apr 7 18:28:30 2014
@@ -370,6 +370,7 @@ public class FSNamesystem implements Nam
} else {
this.dir = new FSDirectory(fsImage, this, conf);
}
+ dir.enableQuotaChecks();
this.safeMode = new SafeModeInfo(conf);
}
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1585546&r1=1585545&r2=1585546&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
Mon Apr 7 18:28:30 2014
@@ -812,6 +812,7 @@ public class SecondaryNameNode implement
Collection<URI> editsDirs) throws IOException {
super(conf, (FSNamesystem)null, imageDirs, editsDirs);
setFSNamesystem(new FSNamesystem(this, conf));
+ getFSNamesystem().dir.disableQuotaChecks();
// the 2NN never writes edits -- it only downloads them. So
// we shouldn't have any editLog instance. Setting to null
