Author: szetszwo
Date: Mon Oct 22 00:11:25 2012
New Revision: 1400743
URL: http://svn.apache.org/viewvc?rev=1400743&view=rev
Log:
HDFS-4078. Handle replication in snapshots.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt?rev=1400743&r1=1400742&r2=1400743&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt
Mon Oct 22 00:11:25 2012
@@ -18,3 +18,5 @@ Branch-2802 Snapshot (Unreleased)
HDFS-4079. Add SnapshotManager which maintains a list for all the
snapshottable directories and supports snapshot methods such as setting a
directory to snapshottable and creating a snapshot. (szetszwo)
+
+ HDFS-4078. Handle replication in snapshots. (szetszwo)
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1400743&r1=1400742&r2=1400743&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Mon Oct 22 00:11:25 2012
@@ -315,9 +315,19 @@ public class FSDirectory implements Clos
//add destination snaplink
snapshot = addNode(dstPath, snapshot, UNKNOWN_DISK_SPACE);
- if (snapshot != null && src.getClass() == INodeFile.class) {
- //created a snapshot and the source is an INodeFile, replace the
source.
- replaceNode(srcPath, src, new INodeFileWithLink(src));
+ final INodeFileWithLink srcWithLink;
+ if (snapshot != null) {
+ //added snapshot node successfully, check source type,
+ if (src instanceof INodeFileWithLink) {
+ srcWithLink = (INodeFileWithLink)src;
+ } else {
+ //source is an INodeFile, replace the source.
+ srcWithLink = new INodeFileWithLink(src);
+ replaceNode(srcPath, src, srcWithLink);
+ }
+
+ //insert the snapshot to src's linked list.
+ srcWithLink.insert(snapshot);
}
} finally {
writeUnlock();
@@ -384,13 +394,13 @@ public class FSDirectory implements Clos
// check quota limits and updated space consumed
updateCount(inodes, inodes.length-1, 0,
- fileINode.getPreferredBlockSize()*fileINode.getBlockReplication(),
true);
+ fileINode.getPreferredBlockSize()*fileINode.getFileReplication(),
true);
// associate new last block for the file
BlockInfoUnderConstruction blockInfo =
new BlockInfoUnderConstruction(
block,
- fileINode.getBlockReplication(),
+ fileINode.getFileReplication(),
BlockUCState.UNDER_CONSTRUCTION,
targets);
getBlockManager().addBlockCollection(blockInfo, fileINode);
@@ -481,7 +491,7 @@ public class FSDirectory implements Clos
// update space consumed
INode[] pathINodes = getExistingPathINodes(path);
updateCount(pathINodes, pathINodes.length-1, 0,
- -fileNode.getPreferredBlockSize()*fileNode.getBlockReplication(),
true);
+ -fileNode.getPreferredBlockSize()*fileNode.getFileReplication(), true);
}
/**
@@ -860,13 +870,13 @@ public class FSDirectory implements Clos
return null;
}
INodeFile fileNode = (INodeFile)inode;
- final short oldRepl = fileNode.getBlockReplication();
+ final short oldRepl = fileNode.getFileReplication();
// check disk quota
long dsDelta = (replication - oldRepl) *
(fileNode.diskspaceConsumed()/oldRepl);
updateCount(inodes, inodes.length-1, 0, dsDelta, true);
- fileNode.setReplication(replication);
+ fileNode.setFileReplication(replication);
if (oldReplication != null) {
oldReplication[0] = oldRepl;
@@ -2124,7 +2134,7 @@ public class FSDirectory implements Clos
if (node instanceof INodeFile) {
INodeFile fileNode = (INodeFile)node;
size = fileNode.computeFileSize(true);
- replication = fileNode.getBlockReplication();
+ replication = fileNode.getFileReplication();
blocksize = fileNode.getPreferredBlockSize();
}
return new HdfsFileStatus(
@@ -2154,7 +2164,7 @@ public class FSDirectory implements Clos
if (node instanceof INodeFile) {
INodeFile fileNode = (INodeFile)node;
size = fileNode.computeFileSize(true);
- replication = fileNode.getBlockReplication();
+ replication = fileNode.getFileReplication();
blocksize = fileNode.getPreferredBlockSize();
loc = getFSNamesystem().getBlockManager().createLocatedBlocks(
fileNode.getBlocks(), fileNode.computeFileSize(false),
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1400743&r1=1400742&r2=1400743&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
Mon Oct 22 00:11:25 2012
@@ -661,7 +661,7 @@ public class FSEditLog implements LogsPu
public void logOpenFile(String path, INodeFileUnderConstruction newNode) {
AddOp op = AddOp.getInstance(cache.get())
.setPath(path)
- .setReplication(newNode.getBlockReplication())
+ .setReplication(newNode.getFileReplication())
.setModificationTime(newNode.getModificationTime())
.setAccessTime(newNode.getAccessTime())
.setBlockSize(newNode.getPreferredBlockSize())
@@ -679,7 +679,7 @@ public class FSEditLog implements LogsPu
public void logCloseFile(String path, INodeFile newNode) {
CloseOp op = CloseOp.getInstance(cache.get())
.setPath(path)
- .setReplication(newNode.getBlockReplication())
+ .setReplication(newNode.getFileReplication())
.setModificationTime(newNode.getModificationTime())
.setAccessTime(newNode.getAccessTime())
.setBlockSize(newNode.getPreferredBlockSize())
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java?rev=1400743&r1=1400742&r2=1400743&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
Mon Oct 22 00:11:25 2012
@@ -126,7 +126,7 @@ public class FSImageSerialization {
String path)
throws IOException {
writeString(path, out);
- out.writeShort(cons.getBlockReplication());
+ out.writeShort(cons.getFileReplication());
out.writeLong(cons.getModificationTime());
out.writeLong(cons.getPreferredBlockSize());
int nrBlocks = cons.getBlocks().length;
@@ -175,7 +175,7 @@ public class FSImageSerialization {
filePerm);
} else {
INodeFile fileINode = (INodeFile)node;
- out.writeShort(fileINode.getBlockReplication());
+ out.writeShort(fileINode.getFileReplication());
out.writeLong(fileINode.getModificationTime());
out.writeLong(fileINode.getAccessTime());
out.writeLong(fileINode.getPreferredBlockSize());
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1400743&r1=1400742&r2=1400743&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Mon Oct 22 00:11:25 2012
@@ -1414,7 +1414,7 @@ public class FSNamesystem implements Nam
}
si.add(trgInode);
- short repl = trgInode.getBlockReplication();
+ final short repl = trgInode.getFileReplication();
// now check the srcs
boolean endSrc = false; // final src file doesn't have to have full end
block
@@ -1434,10 +1434,10 @@ public class FSNamesystem implements Nam
}
// check replication and blocks size
- if(repl != srcInode.getBlockReplication()) {
+ if(repl != srcInode.getFileReplication()) {
throw new IllegalArgumentException(src + " and " + target + " " +
"should have same replication: "
- + repl + " vs. " + srcInode.getBlockReplication());
+ + repl + " vs. " + srcInode.getFileReplication());
}
//boolean endBlock=false;
@@ -1878,9 +1878,10 @@ public class FSNamesystem implements Nam
LocatedBlock prepareFileForWrite(String src, INodeFile file,
String leaseHolder, String clientMachine, DatanodeDescriptor clientNode,
boolean writeToEditLog) throws IOException {
+ //TODO SNAPSHOT: INodeFileUnderConstruction with link
INodeFileUnderConstruction cons = new INodeFileUnderConstruction(
file.getLocalNameBytes(),
- file.getBlockReplication(),
+ file.getFileReplication(),
file.getModificationTime(),
file.getPreferredBlockSize(),
file.getBlocks(),
@@ -2194,7 +2195,7 @@ public class FSNamesystem implements Nam
fileLength = pendingFile.computeContentSummary().getLength();
blockSize = pendingFile.getPreferredBlockSize();
clientNode = pendingFile.getClientNode();
- replication = pendingFile.getBlockReplication();
+ replication = pendingFile.getFileReplication();
} finally {
writeUnlock();
}
@@ -3157,7 +3158,7 @@ public class FSNamesystem implements Nam
if (diff > 0) {
try {
String path = leaseManager.findPath(fileINode);
- dir.updateSpaceConsumed(path, 0, -diff *
fileINode.getBlockReplication());
+ dir.updateSpaceConsumed(path, 0, -diff*fileINode.getFileReplication());
} catch (IOException e) {
LOG.warn("Unexpected exception while updating disk space.", e);
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1400743&r1=1400742&r2=1400743&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
Mon Oct 22 00:11:25 2012
@@ -49,13 +49,13 @@ public class INodeFile extends INode imp
short replication, long modificationTime,
long atime, long preferredBlockSize) {
super(permissions, modificationTime, atime);
- this.setReplication(replication);
+ this.setFileReplication(replication);
this.setPreferredBlockSize(preferredBlockSize);
blocks = blklist;
}
protected INodeFile(INodeFile f) {
- this(f.getPermissionStatus(), f.getBlocks(), f.getBlockReplication(),
+ this(f.getPermissionStatus(), f.getBlocks(), f.getFileReplication(),
f.getModificationTime(), f.getAccessTime(), f.getPreferredBlockSize());
}
@@ -75,12 +75,16 @@ public class INodeFile extends INode imp
}
/** @return the replication factor of the file. */
+ public final short getFileReplication() {
+ return (short) ((header & HEADERMASK) >> BLOCKBITS);
+ }
+
@Override
public short getBlockReplication() {
- return (short) ((header & HEADERMASK) >> BLOCKBITS);
+ return getFileReplication();
}
- void setReplication(short replication) {
+ void setFileReplication(short replication) {
if(replication <= 0)
throw new IllegalArgumentException("Unexpected value for the
replication");
header = ((long)replication << BLOCKBITS) | (header & ~HEADERMASK);
@@ -220,7 +224,7 @@ public class INodeFile extends INode imp
isUnderConstruction()) {
size += getPreferredBlockSize() - blkArr[blkArr.length-1].getNumBytes();
}
- return size * getBlockReplication();
+ return size * getFileReplication();
}
/**
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java?rev=1400743&r1=1400742&r2=1400743&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
Mon Oct 22 00:11:25 2012
@@ -102,9 +102,10 @@ class INodeFileUnderConstruction extends
assert allBlocksComplete() :
"Can't finalize inode " + this + " since it contains " +
"non-complete blocks! Blocks are: " + blocksAsString();
+ //TODO SNAPSHOT: may convert to INodeFileWithLink
INodeFile obj = new INodeFile(getPermissionStatus(),
getBlocks(),
- getBlockReplication(),
+ getFileReplication(),
getModificationTime(),
getModificationTime(),
getPreferredBlockSize());
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java?rev=1400743&r1=1400742&r2=1400743&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
Mon Oct 22 00:11:25 2012
@@ -834,7 +834,7 @@ class NamenodeJspHelper {
doc.endTag();
doc.startTag("replication");
- doc.pcdata(""+inode.getBlockReplication());
+ doc.pcdata(""+inode.getFileReplication());
doc.endTag();
doc.startTag("disk_space_consumed");
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java?rev=1400743&r1=1400742&r2=1400743&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java
Mon Oct 22 00:11:25 2012
@@ -33,6 +33,7 @@ public class INodeFileWithLink extends I
public INodeFileWithLink(INodeFile f) {
super(f);
+ next = this;
}
void setNext(INodeFileWithLink next) {
@@ -42,4 +43,26 @@ public class INodeFileWithLink extends I
INodeFileWithLink getNext() {
return next;
}
+
+ /** Insert inode to the circular linked list. */
+ public void insert(INodeFileWithLink inode) {
+ inode.setNext(this.getNext());
+ this.setNext(inode);
+ }
+
+ /**
+ * @return the max file replication of the elements
+ * in the circular linked list.
+ */
+ @Override
+ public short getBlockReplication() {
+ short max = getFileReplication();
+ for(INodeFileWithLink i = next; i != this; i = i.getNext()) {
+ final short replication = i.getFileReplication();
+ if (replication > max) {
+ max = replication;
+ }
+ }
+ return max;
+ }
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java?rev=1400743&r1=1400742&r2=1400743&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
Mon Oct 22 00:11:25 2012
@@ -48,7 +48,7 @@ public class TestINodeFile {
FsPermission.getDefault()), null,
replication,
0L, 0L, preferredBlockSize);
assertEquals("True has to be returned in this case", replication,
- inf.getBlockReplication());
+ inf.getFileReplication());
}
/**
