Author: brandonli
Date: Tue Oct 15 21:36:51 2013
New Revision: 1532546
URL: http://svn.apache.org/r1532546
Log:
HDFS-5330. Merging change 1532543 from branch-2
Added:
hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java
- copied unchanged from r1532543,
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java
Modified:
hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java
hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Modified:
hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java?rev=1532546&r1=1532545&r2=1532546&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
(original)
+++
hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
Tue Oct 15 21:36:51 2013
@@ -30,6 +30,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem.Statistics;
import org.apache.hadoop.fs.FileUtil;
@@ -44,6 +45,7 @@ import org.apache.hadoop.hdfs.protocol.D
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.nfs.AccessPrivilege;
import org.apache.hadoop.nfs.NfsExports;
import org.apache.hadoop.nfs.NfsFileType;
@@ -1258,6 +1260,29 @@ public class RpcProgramNfs3 extends RpcP
return new READDIR3Response(Nfs3Status.NFS3ERR_NOTSUPP);
}
+ /**
+ * Used by readdir and readdirplus to get dirents. It retries the listing if
+ * the startAfter can't be found anymore.
+ */
+ private DirectoryListing listPaths(DFSClient dfsClient, String dirFileIdPath,
+ byte[] startAfter) throws IOException {
+ DirectoryListing dlisting = null;
+ try {
+ dlisting = dfsClient.listPaths(dirFileIdPath, startAfter);
+ } catch (RemoteException e) {
+ IOException io = e.unwrapRemoteException();
+ if (!(io instanceof DirectoryListingStartAfterNotFoundException)) {
+ throw io;
+ }
+ // This happens when startAfter was just deleted
+ LOG.info("Cookie cound't be found: " + new String(startAfter)
+ + ", do listing from beginning");
+ dlisting = dfsClient
+ .listPaths(dirFileIdPath, HdfsFileStatus.EMPTY_NAME);
+ }
+ return dlisting;
+ }
+
@Override
public READDIR3Response readdir(XDR xdr, SecurityHandler securityHandler,
InetAddress client) {
@@ -1298,7 +1323,7 @@ public class RpcProgramNfs3 extends RpcP
+ cookie + " count: " + count);
}
- HdfsFileStatus dirStatus;
+ HdfsFileStatus dirStatus = null;
DirectoryListing dlisting = null;
Nfs3FileAttributes postOpAttr = null;
long dotdotFileId = 0;
@@ -1342,8 +1367,8 @@ public class RpcProgramNfs3 extends RpcP
String inodeIdPath = Nfs3Utils.getFileIdPath(cookie);
startAfter = inodeIdPath.getBytes();
}
- dlisting = dfsClient.listPaths(dirFileIdPath, startAfter);
-
+
+ dlisting = listPaths(dfsClient, dirFileIdPath, startAfter);
postOpAttr = Nfs3Utils.getFileAttr(dfsClient, dirFileIdPath, iug);
if (postOpAttr == null) {
LOG.error("Can't get path for fileId:" + handle.getFileId());
@@ -1426,11 +1451,15 @@ public class RpcProgramNfs3 extends RpcP
}
long dirCount = request.getDirCount();
if (dirCount <= 0) {
- LOG.info("Nonpositive count in invalid READDIRPLUS request:" + dirCount);
- return new READDIRPLUS3Response(Nfs3Status.NFS3_OK);
+ LOG.info("Nonpositive dircount in invalid READDIRPLUS request:" +
dirCount);
+ return new READDIRPLUS3Response(Nfs3Status.NFS3ERR_INVAL);
}
int maxCount = request.getMaxCount();
-
+ if (maxCount <= 0) {
+ LOG.info("Nonpositive maxcount in invalid READDIRPLUS request:" +
maxCount);
+ return new READDIRPLUS3Response(Nfs3Status.NFS3ERR_INVAL);
+ }
+
if (LOG.isDebugEnabled()) {
LOG.debug("NFS READDIRPLUS fileId: " + handle.getFileId() + " cookie: "
+ cookie + " dirCount: " + dirCount + " maxCount: " + maxCount);
@@ -1480,8 +1509,8 @@ public class RpcProgramNfs3 extends RpcP
String inodeIdPath = Nfs3Utils.getFileIdPath(cookie);
startAfter = inodeIdPath.getBytes();
}
- dlisting = dfsClient.listPaths(dirFileIdPath, startAfter);
-
+
+ dlisting = listPaths(dfsClient, dirFileIdPath, startAfter);
postOpDirAttr = Nfs3Utils.getFileAttr(dfsClient, dirFileIdPath, iug);
if (postOpDirAttr == null) {
LOG.info("Can't get path for fileId:" + handle.getFileId());
Modified:
hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java?rev=1532546&r1=1532545&r2=1532546&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
(original)
+++
hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
Tue Oct 15 21:36:51 2013
@@ -195,6 +195,7 @@ public class WriteManager {
COMMIT_STATUS ret = openFileCtx.checkCommit(dfsClient, commitOffset,
channel, xid, preOpAttr);
switch (ret) {
+ case COMMIT_DO_SYNC:
case COMMIT_FINISHED:
case COMMIT_INACTIVE_CTX:
status = Nfs3Status.NFS3_OK;
@@ -207,7 +208,8 @@ public class WriteManager {
// Do nothing. Commit is async now.
return;
default:
- throw new RuntimeException("Wring error code:" + ret.name());
+ throw new RuntimeException("Should not get commit return code:"
+ + ret.name());
}
}
Modified:
hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java?rev=1532546&r1=1532545&r2=1532546&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java
(original)
+++
hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java
Tue Oct 15 21:36:51 2013
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.nfs.nfs3;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Map.Entry;
import java.util.concurrent.ConcurrentNavigableMap;
import junit.framework.Assert;
@@ -33,7 +32,6 @@ import org.apache.hadoop.nfs.nfs3.IdUser
import org.apache.hadoop.nfs.nfs3.Nfs3Constant.WriteStableHow;
import org.apache.hadoop.nfs.nfs3.Nfs3FileAttributes;
import org.apache.hadoop.nfs.nfs3.request.WRITE3Request;
-import org.jboss.netty.channel.Channel;
import org.junit.Test;
import org.mockito.Mockito;
Modified:
hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1532546&r1=1532545&r2=1532546&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Tue Oct 15 21:36:51 2013
@@ -39,6 +39,8 @@ Release 2.2.1 - UNRELEASED
HDFS-5329. Update FSNamesystem#getListing() to handle inode path in
startAfter
token. (brandonli)
+ HDFS-5330. fix readdir and readdirplus for large directories (brandonli)
+
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
