Author: suresh
Date: Wed Nov 11 22:46:06 2009
New Revision: 835110
URL: http://svn.apache.org/viewvc?rev=835110&view=rev
Log:
HDFS-761. Fix failure to process rename operation from edits log due to quota
verification. Contributed by Suresh Srinivas.
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=835110&r1=835109&r2=835110&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Wed Nov 11 22:46:06 2009
@@ -539,3 +539,6 @@
HDFS-525. The SimpleDateFormat object in ListPathsServlet is not thread
safe. (Suresh Srinivas and cdouglas)
+
+ HDFS-761. Fix failure to process rename operation from edits log due to
+ quota verification. (suresh)
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=835110&r1=835109&r2=835110&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Wed Nov 11 22:46:06 2009
@@ -1355,6 +1355,10 @@
*/
private void verifyQuota(INode[] inodes, int pos, long nsDelta, long dsDelta,
INode commonAncestor) throws QuotaExceededException {
+ if (!ready) {
+ // Do not check quota if edits log is still being processed
+ return;
+ }
if (nsDelta <= 0 && dsDelta <= 0) {
// if quota is being freed or not being consumed
return;
@@ -1392,6 +1396,10 @@
*/
private void verifyQuotaForRename(INode[] srcInodes, INode[]dstInodes)
throws QuotaExceededException {
+ if (!ready) {
+ // Do not check quota if edits log is still being processed
+ return;
+ }
INode srcInode = srcInodes[srcInodes.length - 1];
INode commonAncestor = null;
for(int i =0;srcInodes[i] == dstInodes[i]; i++) {
Modified:
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java?rev=835110&r1=835109&r2=835110&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
(original)
+++
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
Wed Nov 11 22:46:06 2009
@@ -23,7 +23,6 @@
import javax.security.auth.login.LoginException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -42,18 +41,31 @@
FileContextMainOperationsBaseTest {
private static MiniDFSCluster cluster;
private static Path defaultWorkingDirectory;
+ private static HdfsConfiguration CONF = new HdfsConfiguration();
@BeforeClass
public static void clusterSetupAtBegining() throws IOException,
LoginException, URISyntaxException {
- Configuration conf = new HdfsConfiguration();
- cluster = new MiniDFSCluster(conf, 2, true, null);
- fc = FileContext.getFileContext(cluster.getURI(), conf);
+ cluster = new MiniDFSCluster(CONF, 2, true, null);
+ cluster.waitClusterUp();
+ fc = FileContext.getFileContext(cluster.getURI(), CONF);
defaultWorkingDirectory = fc.makeQualified( new Path("/user/" +
UnixUserGroupInformation.login().getUserName()));
fc.mkdir(defaultWorkingDirectory, FileContext.DEFAULT_PERM, true);
}
+ private static void restartCluster() throws IOException, LoginException {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ cluster = new MiniDFSCluster(CONF, 1, false, null);
+ cluster.waitClusterUp();
+ fc = FileContext.getFileContext(cluster.getURI(), CONF);
+ defaultWorkingDirectory = fc.makeQualified( new Path("/user/" +
+ UnixUserGroupInformation.login().getUserName()));
+ fc.mkdir(defaultWorkingDirectory, FileContext.DEFAULT_PERM, true);
+ }
@AfterClass
public static void ClusterShutdownAtEnd() throws Exception {
@@ -179,6 +191,64 @@
rename(dst, src, true, false, true, Rename.OVERWRITE);
}
+ /**
+ * Perform operations such as setting quota, deletion of files, rename and
+ * ensure system can apply edits log during startup.
+ */
+ @Test
+ public void testEditsLogOldRename() throws Exception {
+ DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem();
+ Path src1 = getTestRootPath("testEditsLogOldRename/srcdir/src1");
+ Path dst1 = getTestRootPath("testEditsLogOldRename/dstdir/dst1");
+ createFile(src1);
+ fs.mkdirs(dst1.getParent());
+ createFile(dst1);
+
+ // Set quota so that dst1 parent cannot allow under it new
files/directories
+ fs.setQuota(dst1.getParent(), 2, FSConstants.QUOTA_DONT_SET);
+ // Free up quota for a subsequent rename
+ fs.delete(dst1, true);
+ oldRename(src1, dst1, true, false);
+
+ // Restart the cluster and ensure the above operations can be
+ // loaded from the edits log
+ restartCluster();
+ fs = (DistributedFileSystem)cluster.getFileSystem();
+ src1 = getTestRootPath("testEditsLogOldRename/srcdir/src1");
+ dst1 = getTestRootPath("testEditsLogOldRename/dstdir/dst1");
+ Assert.assertFalse(fs.exists(src1)); // ensure src1 is already renamed
+ Assert.assertTrue(fs.exists(dst1)); // ensure rename dst exists
+ }
+
+ /**
+ * Perform operations such as setting quota, deletion of files, rename and
+ * ensure system can apply edits log during startup.
+ */
+ @Test
+ public void testEditsLogRename() throws Exception {
+ DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem();
+ Path src1 = getTestRootPath("testEditsLogRename/srcdir/src1");
+ Path dst1 = getTestRootPath("testEditsLogRename/dstdir/dst1");
+ createFile(src1);
+ fs.mkdirs(dst1.getParent());
+ createFile(dst1);
+
+ // Set quota so that dst1 parent cannot allow under it new
files/directories
+ fs.setQuota(dst1.getParent(), 2, FSConstants.QUOTA_DONT_SET);
+ // Free up quota for a subsequent rename
+ fs.delete(dst1, true);
+ rename(src1, dst1, true, true, false, Rename.OVERWRITE);
+
+ // Restart the cluster and ensure the above operations can be
+ // loaded from the edits log
+ restartCluster();
+ fs = (DistributedFileSystem)cluster.getFileSystem();
+ src1 = getTestRootPath("testEditsLogRename/srcdir/src1");
+ dst1 = getTestRootPath("testEditsLogRename/dstdir/dst1");
+ Assert.assertFalse(fs.exists(src1)); // ensure src1 is already renamed
+ Assert.assertTrue(fs.exists(dst1)); // ensure rename dst exists
+ }
+
private void oldRename(Path src, Path dst, boolean renameSucceeds,
boolean exception) throws Exception {
DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem();
