Author: umamahesh
Date: Sat May 10 19:21:50 2014
New Revision: 1593717
URL: http://svn.apache.org/r1593717
Log:
HDFS-6357. SetXattr should persist rpcIDs for handling retrycache with Namenode
restart and HA. Contributed by Uma Maheswara Rao G.
Modified:
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-2006.txt
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
Modified:
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-2006.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-2006.txt?rev=1593717&r1=1593716&r2=1593717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-2006.txt
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-2006.txt
Sat May 10 19:21:50 2014
@@ -42,3 +42,6 @@ HDFS-2006 (Unreleased)
HDFS-6343. fix TestNamenodeRetryCache and TestRetryCacheWithHA failures.
(umamahesh)
HDFS-6366. FsImage loading failed with RemoveXattr op (umamahesh)
+
+ HDFS-6357. SetXattr should persist rpcIDs for handling retrycache with
Namenode restart and HA
+ (umamahesh)
Modified:
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1593717&r1=1593716&r2=1593717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Sat May 10 19:21:50 2014
@@ -2935,12 +2935,12 @@ public class FSDirectory implements Clos
return xAttrs;
}
- void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
- throws IOException {
+ void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag,
+ boolean logRetryCache) throws IOException {
writeLock();
try {
unprotectedSetXAttr(src, xAttr, flag);
- fsImage.getEditLog().logSetXAttr(src, xAttr);
+ fsImage.getEditLog().logSetXAttr(src, xAttr, logRetryCache);
} finally {
writeUnlock();
}
Modified:
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1593717&r1=1593716&r2=1593717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
Sat May 10 19:21:50 2014
@@ -1054,10 +1054,11 @@ public class FSEditLog implements LogsPu
logEdit(op);
}
- void logSetXAttr(String src, XAttr xAttr) {
+ void logSetXAttr(String src, XAttr xAttr, boolean toLogRpcIds) {
final SetXAttrOp op = SetXAttrOp.getInstance();
op.src = src;
op.xAttr = xAttr;
+ logRpcIds(op, toLogRpcIds);
logEdit(op);
}
Modified:
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1593717&r1=1593716&r2=1593717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
Sat May 10 19:21:50 2014
@@ -806,6 +806,9 @@ public class FSEditLogLoader {
SetXAttrOp setXAttrOp = (SetXAttrOp) op;
fsDir.unprotectedSetXAttr(setXAttrOp.src, setXAttrOp.xAttr,
EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
+ if (toAddRetryCache) {
+ fsNamesys.addCacheEntry(setXAttrOp.rpcClientId, setXAttrOp.rpcCallId);
+ }
break;
}
case OP_REMOVE_XATTR: {
Modified:
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java?rev=1593717&r1=1593716&r2=1593717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
Sat May 10 19:21:50 2014
@@ -3557,6 +3557,7 @@ public abstract class FSEditLogOp {
XAttrEditLogProto p = XAttrEditLogProto.parseDelimitedFrom(in);
src = p.getSrc();
xAttr = PBHelper.convertXAttr(p.getXAttr());
+ readRpcIds(in, logVersion);
}
@Override
@@ -3567,6 +3568,8 @@ public abstract class FSEditLogOp {
}
b.setXAttr(PBHelper.convertXAttrProto(xAttr));
b.build().writeDelimitedTo(out);
+ // clientId and callId
+ writeRpcIds(rpcClientId, rpcCallId, out);
}
@Override
Modified:
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1593717&r1=1593716&r2=1593717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Sat May 10 19:21:50 2014
@@ -7731,7 +7731,7 @@ public class FSNamesystem implements Nam
}
boolean success = false;
try {
- setXAttrInt(src, xAttr, flag);
+ setXAttrInt(src, xAttr, flag, cacheEntry != null);
success = true;
} catch (AccessControlException e) {
logAuditEvent(false, "setXAttr", src);
@@ -7741,8 +7741,8 @@ public class FSNamesystem implements Nam
}
}
- private void setXAttrInt(String src, XAttr xAttr, EnumSet<XAttrSetFlag>
flag)
- throws IOException {
+ private void setXAttrInt(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag,
+ boolean logRetryCache) throws IOException {
nnConf.checkXAttrsConfigFlag();
HdfsFileStatus resultingStat = null;
FSPermissionChecker pc = getPermissionChecker();
@@ -7757,7 +7757,7 @@ public class FSNamesystem implements Nam
if (isPermissionEnabled) {
checkPathAccess(pc, src, FsAction.WRITE);
}
- dir.setXAttr(src, xAttr, flag);
+ dir.setXAttr(src, xAttr, flag, logRetryCache);
resultingStat = getAuditFileInfo(src, false);
} finally {
writeUnlock();
Modified:
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java?rev=1593717&r1=1593716&r2=1593717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
Sat May 10 19:21:50 2014
@@ -415,7 +415,7 @@ public class TestNamenodeRetryCache {
LightWeightCache<CacheEntry, CacheEntry> cacheSet =
(LightWeightCache<CacheEntry, CacheEntry>)
namesystem.getRetryCache().getCacheSet();
- assertEquals(20, cacheSet.size());
+ assertEquals(22, cacheSet.size());
Map<CacheEntry, CacheEntry> oldEntries =
new HashMap<CacheEntry, CacheEntry>();
