Author: szetszwo
Date: Sat Nov 3 01:38:41 2012
New Revision: 1405253
URL: http://svn.apache.org/viewvc?rev=1405253&view=rev
Log:
Merge r1404624 through r1405251 from trunk.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1404624-1405251
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1405253&r1=1405252&r2=1405253&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Sat Nov 3 01:38:41 2012
@@ -445,6 +445,9 @@ Release 2.0.3-alpha - Unreleased
HDFS-3916. libwebhdfs testing code cleanup. (Jing Zhao via suresh)
+ HDFS-4143. Change blocks to private in INodeFile and renames isLink() to
+ isSymlink() in INode. (szetszwo)
+
OPTIMIZATIONS
BUG FIXES
@@ -536,6 +539,12 @@ Release 2.0.3-alpha - Unreleased
HDFS-3809. Make BKJM use protobufs for all serialization with ZK.
(Ivan Kelly via umamahesh)
+ HDFS-3804. TestHftpFileSystem fails intermittently with JDK7
+ (Trevor Robinson via daryn)
+
+ HDFS-4132. When libwebhdfs is not enabled, nativeMiniDfsClient frees
+ uninitialized memory (Colin Patrick McCabe via todd)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1404624-1405251
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1405253&r1=1405252&r2=1405253&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Sat Nov 3 01:38:41 2012
@@ -313,7 +313,7 @@ public class FSDirectory implements Clos
}
if(newParent == null)
return null;
- if(!newNode.isDirectory() && !newNode.isLink()) {
+ if(!newNode.isDirectory() && !newNode.isSymlink()) {
// Add file->block mapping
INodeFile newF = (INodeFile)newNode;
BlockInfo[] blocks = newF.getBlocks();
@@ -533,7 +533,7 @@ public class FSDirectory implements Clos
if (dst.equals(src)) {
return true;
}
- if (srcInode.isLink() &&
+ if (srcInode.isSymlink() &&
dst.equals(((INodeSymlink)srcInode).getLinkValue())) {
throw new FileAlreadyExistsException(
"Cannot rename symlink "+src+" to its target "+dst);
@@ -655,7 +655,7 @@ public class FSDirectory implements Clos
throw new FileAlreadyExistsException(
"The source "+src+" and destination "+dst+" are the same");
}
- if (srcInode.isLink() &&
+ if (srcInode.isSymlink() &&
dst.equals(((INodeSymlink)srcInode).getLinkValue())) {
throw new FileAlreadyExistsException(
"Cannot rename symlink "+src+" to its target "+dst);
@@ -819,7 +819,7 @@ public class FSDirectory implements Clos
if (inode == null) {
return null;
}
- assert !inode.isLink();
+ assert !inode.isSymlink();
if (inode.isDirectory()) {
return null;
}
@@ -851,7 +851,7 @@ public class FSDirectory implements Clos
if (inode == null) {
throw new FileNotFoundException("File does not exist: " + filename);
}
- if (inode.isDirectory() || inode.isLink()) {
+ if (inode.isDirectory() || inode.isSymlink()) {
throw new IOException("Getting block size of non-file: "+ filename);
}
return ((INodeFile)inode).getPreferredBlockSize();
@@ -868,7 +868,7 @@ public class FSDirectory implements Clos
if (inode == null) {
return false;
}
- return inode.isDirectory() || inode.isLink()
+ return inode.isDirectory() || inode.isSymlink()
? true
: ((INodeFile)inode).getBlocks() != null;
} finally {
@@ -968,7 +968,7 @@ public class FSDirectory implements Clos
for(String src : srcs) {
INodeFile srcInode = (INodeFile)getINode(src);
allSrcInodes[i++] = srcInode;
- totalBlocks += srcInode.blocks.length;
+ totalBlocks += srcInode.numBlocks();
}
trgInode.appendBlocks(allSrcInodes, totalBlocks); // copy the blocks
@@ -977,7 +977,7 @@ public class FSDirectory implements Clos
for(INodeFile nodeToRemove: allSrcInodes) {
if(nodeToRemove == null) continue;
- nodeToRemove.blocks = null;
+ nodeToRemove.setBlocks(null);
trgParent.removeChild(nodeToRemove);
count++;
}
@@ -1232,7 +1232,7 @@ public class FSDirectory implements Clos
return null;
if (targetNode.isDirectory())
return null;
- if (targetNode.isLink())
+ if (targetNode.isSymlink())
return null;
return ((INodeFile)targetNode).getBlocks();
} finally {
@@ -1846,7 +1846,7 @@ public class FSDirectory implements Clos
if (child.isDirectory()) {
updateCountForINodeWithQuota((INodeDirectory)child,
counts, nodesInPath);
- } else if (child.isLink()) {
+ } else if (child.isSymlink()) {
counts.nsCount += 1;
} else { // reduce recursive calls
counts.nsCount += 1;
@@ -2075,7 +2075,7 @@ public class FSDirectory implements Clos
node.getFsPermission(),
node.getUserName(),
node.getGroupName(),
- node.isLink() ? ((INodeSymlink)node).getSymlink() : null,
+ node.isSymlink() ? ((INodeSymlink)node).getSymlink() : null,
path);
}
@@ -2111,7 +2111,7 @@ public class FSDirectory implements Clos
node.getFsPermission(),
node.getUserName(),
node.getGroupName(),
- node.isLink() ? ((INodeSymlink)node).getSymlink() : null,
+ node.isSymlink() ? ((INodeSymlink)node).getSymlink() : null,
path,
loc);
}
@@ -2182,7 +2182,7 @@ public class FSDirectory implements Clos
*/
void cacheName(INode inode) {
// Name is cached only for files
- if (inode.isDirectory() || inode.isLink()) {
+ if (inode.isDirectory() || inode.isSymlink()) {
return;
}
ByteArray name = new ByteArray(inode.getLocalNameBytes());
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java?rev=1405253&r1=1405252&r2=1405253&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
Sat Nov 3 01:38:41 2012
@@ -162,7 +162,7 @@ public class FSImageSerialization {
PermissionStatus.write(out, node.getUserName(),
node.getGroupName(),
filePerm);
- } else if (node.isLink()) {
+ } else if (node.isSymlink()) {
out.writeShort(0); // replication
out.writeLong(0); // modification time
out.writeLong(0); // access time
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1405253&r1=1405252&r2=1405253&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Sat Nov 3 01:38:41 2012
@@ -1394,7 +1394,7 @@ public class FSNamesystem implements Nam
+ target + " is under construction");
}
// per design target shouldn't be empty and all the blocks same size
- if(trgInode.blocks.length == 0) {
+ if(trgInode.numBlocks() == 0) {
throw new HadoopIllegalArgumentException("concat: target file "
+ target + " is empty");
}
@@ -1406,10 +1406,10 @@ public class FSNamesystem implements Nam
long blockSize = trgInode.getPreferredBlockSize();
// check the end block to be full
- if(blockSize != trgInode.blocks[trgInode.blocks.length-1].getNumBytes()) {
+ final BlockInfo last = trgInode.getLastBlock();
+ if(blockSize != last.getNumBytes()) {
throw new HadoopIllegalArgumentException("The last block in " + target
- + " is not full; last block size = "
- + trgInode.blocks[trgInode.blocks.length-1].getNumBytes()
+ + " is not full; last block size = " + last.getNumBytes()
+ " but file block size = " + blockSize);
}
@@ -1426,7 +1426,7 @@ public class FSNamesystem implements Nam
final INodeFile srcInode = INodeFile.valueOf(dir.getINode(src), src);
if(src.isEmpty()
|| srcInode.isUnderConstruction()
- || srcInode.blocks.length == 0) {
+ || srcInode.numBlocks() == 0) {
throw new HadoopIllegalArgumentException("concat: source file " + src
+ " is invalid or empty or underConstruction");
}
@@ -1443,15 +1443,16 @@ public class FSNamesystem implements Nam
//boolean endBlock=false;
// verify that all the blocks are of the same length as target
// should be enough to check the end blocks
- int idx = srcInode.blocks.length-1;
+ final BlockInfo[] srcBlocks = srcInode.getBlocks();
+ int idx = srcBlocks.length-1;
if(endSrc)
- idx = srcInode.blocks.length-2; // end block of endSrc is OK not to be
full
- if(idx >= 0 && srcInode.blocks[idx].getNumBytes() != blockSize) {
+ idx = srcBlocks.length-2; // end block of endSrc is OK not to be full
+ if(idx >= 0 && srcBlocks[idx].getNumBytes() != blockSize) {
throw new HadoopIllegalArgumentException("concat: the soruce file "
+ src + " and the target file " + target
+ " should have the same blocks sizes: target block size is "
+ blockSize + " but the size of source block " + idx + " is "
- + srcInode.blocks[idx].getNumBytes());
+ + srcBlocks[idx].getNumBytes());
}
si.add(srcInode);
@@ -1686,7 +1687,7 @@ public class FSNamesystem implements Nam
if (parentNode == null) {
throw new FileNotFoundException("Parent directory doesn't exist: "
+ parent.toString());
- } else if (!parentNode.isDirectory() && !parentNode.isLink()) {
+ } else if (!parentNode.isDirectory() && !parentNode.isSymlink()) {
throw new ParentNotDirectoryException("Parent path is not a directory:
"
+ parent.toString());
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java?rev=1405253&r1=1405252&r2=1405253&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
Sat Nov 3 01:38:41 2012
@@ -183,7 +183,9 @@ public abstract class INode implements C
/**
* Check whether it's a directory
*/
- abstract boolean isDirectory();
+ public boolean isDirectory() {
+ return false;
+ }
/**
* Collect all the blocks in all children of this INode.
@@ -337,7 +339,7 @@ public abstract class INode implements C
/**
* Check whether it's a symlink
*/
- public boolean isLink() {
+ public boolean isSymlink() {
return false;
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java?rev=1405253&r1=1405252&r2=1405253&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
Sat Nov 3 01:38:41 2012
@@ -81,11 +81,9 @@ public class INodeDirectory extends INod
this.children = other.getChildren();
}
- /**
- * Check whether it's a directory
- */
+ /** @return true unconditionally. */
@Override
- public boolean isDirectory() {
+ public final boolean isDirectory() {
return true;
}
@@ -214,7 +212,7 @@ public class INodeDirectory extends INod
if (index >= 0) {
existing.addNode(curNode);
}
- if (curNode.isLink() && (!lastComp || (lastComp && resolveLink))) {
+ if (curNode.isSymlink() && (!lastComp || (lastComp && resolveLink))) {
final String path = constructPath(components, 0, components.length);
final String preceding = constructPath(components, 0, count);
final String remainder =
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1405253&r1=1405252&r2=1405253&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
Sat Nov 3 01:38:41 2012
@@ -55,7 +55,7 @@ public class INodeFile extends INode imp
private long header;
- protected BlockInfo[] blocks = null;
+ protected BlockInfo[] blocks;
INodeFile(PermissionStatus permissions, BlockInfo[] blklist,
short replication, long modificationTime,
@@ -63,7 +63,7 @@ public class INodeFile extends INode imp
super(permissions, modificationTime, atime);
this.setFileReplication(replication);
this.setPreferredBlockSize(preferredBlockSize);
- blocks = blklist;
+ this.blocks = blklist;
}
protected INodeFile(INodeFile f) {
@@ -82,11 +82,6 @@ public class INodeFile extends INode imp
super.setPermission(permission.applyUMask(UMASK));
}
- @Override
- boolean isDirectory() {
- return false;
- }
-
/** @return the replication factor of the file. */
public final short getFileReplication() {
return (short) ((header & HEADERMASK) >> BLOCKBITS);
@@ -138,7 +133,7 @@ public class INodeFile extends INode imp
for(BlockInfo bi: newlist) {
bi.setBlockCollection(this);
}
- this.blocks = newlist;
+ setBlocks(newlist);
}
/**
@@ -146,14 +141,13 @@ public class INodeFile extends INode imp
*/
void addBlock(BlockInfo newblock) {
if (this.blocks == null) {
- this.blocks = new BlockInfo[1];
- this.blocks[0] = newblock;
+ this.setBlocks(new BlockInfo[]{newblock});
} else {
int size = this.blocks.length;
BlockInfo[] newlist = new BlockInfo[size + 1];
System.arraycopy(this.blocks, 0, newlist, 0, size);
newlist[size] = newblock;
- this.blocks = newlist;
+ this.setBlocks(newlist);
}
}
@@ -162,6 +156,11 @@ public class INodeFile extends INode imp
this.blocks[idx] = blk;
}
+ /** Set the blocks. */
+ public void setBlocks(BlockInfo[] blocks) {
+ this.blocks = blocks;
+ }
+
@Override
protected int collectSubtreeBlocksAndClear(List<Block> v) {
parent = null;
@@ -171,7 +170,7 @@ public class INodeFile extends INode imp
blk.setBlockCollection(null);
}
}
- blocks = null;
+ setBlocks(null);
return 1;
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java?rev=1405253&r1=1405252&r2=1405253&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
Sat Nov 3 01:38:41 2012
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.PermissionStatus;
@@ -28,8 +29,6 @@ import org.apache.hadoop.hdfs.server.blo
import org.apache.hadoop.hdfs.server.blockmanagement.MutableBlockCollection;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
-import com.google.common.base.Joiner;
-
/**
* I-node for file being written.
*/
@@ -109,9 +108,9 @@ public class INodeFileUnderConstruction
// use the modification time as the access time
//
INodeFile convertToInodeFile() {
- assert allBlocksComplete() :
- "Can't finalize inode " + this + " since it contains " +
- "non-complete blocks! Blocks are: " + blocksAsString();
+ assert allBlocksComplete() : "Can't finalize inode " + this
+ + " since it contains non-complete blocks! Blocks are "
+ + Arrays.asList(getBlocks());
//TODO SNAPSHOT: may convert to INodeFileWithLink
INodeFile obj = new INodeFile(getPermissionStatus(),
getBlocks(),
@@ -127,7 +126,7 @@ public class INodeFileUnderConstruction
* @return true if all of the blocks in this file are marked as completed.
*/
private boolean allBlocksComplete() {
- for (BlockInfo b : blocks) {
+ for (BlockInfo b : getBlocks()) {
if (!b.isComplete()) {
return false;
}
@@ -140,6 +139,7 @@ public class INodeFileUnderConstruction
* the last one on the list.
*/
void removeLastBlock(Block oldblock) throws IOException {
+ final BlockInfo[] blocks = getBlocks();
if (blocks == null) {
throw new IOException("Trying to delete non-existant block " + oldblock);
}
@@ -151,7 +151,7 @@ public class INodeFileUnderConstruction
//copy to a new list
BlockInfo[] newlist = new BlockInfo[size_1];
System.arraycopy(blocks, 0, newlist, 0, size_1);
- blocks = newlist;
+ setBlocks(newlist);
}
/**
@@ -160,11 +160,9 @@ public class INodeFileUnderConstruction
*/
@Override
public BlockInfoUnderConstruction setLastBlock(BlockInfo lastBlock,
- DatanodeDescriptor[] targets)
- throws IOException {
- if (blocks == null || blocks.length == 0) {
- throw new IOException("Trying to update non-existant block. " +
- "File is empty.");
+ DatanodeDescriptor[] targets) throws IOException {
+ if (numBlocks() == 0) {
+ throw new IOException("Failed to set last block: File is empty.");
}
BlockInfoUnderConstruction ucBlock =
lastBlock.convertToBlockUnderConstruction(
@@ -173,8 +171,4 @@ public class INodeFileUnderConstruction
setBlock(numBlocks()-1, ucBlock);
return ucBlock;
}
-
- private String blocksAsString() {
- return Joiner.on(",").join(this.blocks);
- }
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java?rev=1405253&r1=1405252&r2=1405253&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
Sat Nov 3 01:38:41 2012
@@ -49,7 +49,7 @@ public class INodeSymlink extends INode
}
@Override
- public boolean isLink() {
+ public boolean isSymlink() {
return true;
}
@@ -81,9 +81,4 @@ public class INodeSymlink extends INode
summary[1]++; // Increment the file count
return summary;
}
-
- @Override
- public boolean isDirectory() {
- return false;
- }
}
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1404624-1405251
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c?rev=1405253&r1=1405252&r2=1405253&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c
Sat Nov 3 01:38:41 2012
@@ -44,11 +44,11 @@ struct NativeMiniDfsCluster {
struct NativeMiniDfsCluster* nmdCreate(struct NativeMiniDfsConf *conf)
{
struct NativeMiniDfsCluster* cl = NULL;
- jobject bld = NULL, bld2 = NULL, cobj = NULL;
+ jobject bld = NULL, cobj = NULL, cluster = NULL;
jvalue val;
JNIEnv *env = getJNIEnv();
jthrowable jthr;
- jstring jconfStr;
+ jstring jconfStr = NULL;
if (!env) {
fprintf(stderr, "nmdCreate: unable to construct JNIEnv.\n");
@@ -63,14 +63,14 @@ struct NativeMiniDfsCluster* nmdCreate(s
if (jthr) {
printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
"nmdCreate: new Configuration");
- goto error_free_cl;
+ goto error;
}
if (conf->webhdfsEnabled) {
jthr = newJavaStr(env, DFS_WEBHDFS_ENABLED_KEY, &jconfStr);
if (jthr) {
printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
"nmdCreate: new String");
- goto error_dlr_cobj;
+ goto error;
}
jthr = invokeMethod(env, NULL, INSTANCE, cobj, HADOOP_CONF,
"setBoolean", "(Ljava/lang/String;Z)V",
@@ -78,7 +78,7 @@ struct NativeMiniDfsCluster* nmdCreate(s
if (jthr) {
printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
"nmdCreate: Configuration::setBoolean");
- goto error_dlr_cobj;
+ goto error;
}
}
jthr = constructNewObjectOfClass(env, &bld, MINIDFS_CLUSTER_BUILDER,
@@ -86,58 +86,53 @@ struct NativeMiniDfsCluster* nmdCreate(s
if (jthr) {
printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
"nmdCreate: NativeMiniDfsCluster#Builder#Builder");
- goto error_dlr_cobj;
+ goto error;
}
jthr = invokeMethod(env, &val, INSTANCE, bld, MINIDFS_CLUSTER_BUILDER,
"format", "(Z)L" MINIDFS_CLUSTER_BUILDER ";", conf->doFormat);
if (jthr) {
printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "nmdCreate: "
"Builder::format");
- goto error_dlr_bld;
+ goto error;
}
- bld2 = val.l;
+ (*env)->DeleteLocalRef(env, val.l);
if (conf->webhdfsEnabled) {
- jthr = invokeMethod(env, &val, INSTANCE, bld2, MINIDFS_CLUSTER_BUILDER,
+ jthr = invokeMethod(env, &val, INSTANCE, bld, MINIDFS_CLUSTER_BUILDER,
"nameNodeHttpPort", "(I)L" MINIDFS_CLUSTER_BUILDER ";",
conf->namenodeHttpPort);
if (jthr) {
printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "nmdCreate: "
"Builder::nameNodeHttpPort");
- goto error_dlr_bld2;
+ goto error;
}
+ (*env)->DeleteLocalRef(env, val.l);
}
jthr = invokeMethod(env, &val, INSTANCE, bld, MINIDFS_CLUSTER_BUILDER,
"build", "()L" MINIDFS_CLUSTER ";");
if (jthr) {
printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
"nmdCreate: Builder#build");
- goto error_dlr_bld2;
+ goto error;
}
- cl->obj = (*env)->NewGlobalRef(env, val.l);
+ cluster = val.l;
+ cl->obj = (*env)->NewGlobalRef(env, val.l);
if (!cl->obj) {
printPendingExceptionAndFree(env, PRINT_EXC_ALL,
"nmdCreate: NewGlobalRef");
- goto error_dlr_val;
+ goto error;
}
- (*env)->DeleteLocalRef(env, val.l);
- (*env)->DeleteLocalRef(env, bld2);
+ (*env)->DeleteLocalRef(env, cluster);
(*env)->DeleteLocalRef(env, bld);
(*env)->DeleteLocalRef(env, cobj);
(*env)->DeleteLocalRef(env, jconfStr);
return cl;
-error_dlr_val:
- (*env)->DeleteLocalRef(env, val.l);
-error_dlr_bld2:
- (*env)->DeleteLocalRef(env, bld2);
-error_dlr_bld:
+error:
+ (*env)->DeleteLocalRef(env, cluster);
(*env)->DeleteLocalRef(env, bld);
-error_dlr_cobj:
(*env)->DeleteLocalRef(env, cobj);
(*env)->DeleteLocalRef(env, jconfStr);
-error_free_cl:
free(cl);
-error:
return NULL;
}
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1404624-1405251
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1404624-1405251
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1404624-1405251
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1404624-1405251
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java?rev=1405253&r1=1405252&r2=1405253&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java
Sat Nov 3 01:38:41 2012
@@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
@@ -42,18 +43,17 @@ import org.apache.hadoop.hdfs.server.dat
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.util.ServletUtil;
import org.apache.log4j.Level;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.*;
public class TestHftpFileSystem {
private static final Random RAN = new Random();
private static Configuration config = null;
private static MiniDFSCluster cluster = null;
- private static FileSystem hdfs = null;
- private static HftpFileSystem hftpFs = null;
private static String blockPoolId = null;
+ private static String hftpUri = null;
+ private FileSystem hdfs = null;
+ private HftpFileSystem hftpFs = null;
private static Path[] TEST_PATHS = new Path[] {
// URI does not encode, Request#getPathInfo returns /foo
@@ -93,26 +93,33 @@ public class TestHftpFileSystem {
config = new Configuration();
cluster = new MiniDFSCluster.Builder(config).numDataNodes(2).build();
- hdfs = cluster.getFileSystem();
blockPoolId = cluster.getNamesystem().getBlockPoolId();
- final String hftpUri =
+ hftpUri =
"hftp://"; + config.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY);
- hftpFs = (HftpFileSystem) new Path(hftpUri).getFileSystem(config);
}
@AfterClass
public static void tearDown() throws IOException {
- if (hdfs != null) {
- hdfs.close();
- }
- if (hftpFs != null) {
- hftpFs.close();
- }
if (cluster != null) {
cluster.shutdown();
}
}
+
+ @Before
+ public void initFileSystems() throws IOException {
+ hdfs = cluster.getFileSystem();
+ hftpFs = (HftpFileSystem) new Path(hftpUri).getFileSystem(config);
+ // clear out the namespace
+ for (FileStatus stat : hdfs.listStatus(new Path("/"))) {
+ hdfs.delete(stat.getPath(), true);
+ }
+ }
+ @After
+ public void resetFileSystems() throws IOException {
+ FileSystem.closeAll();
+ }
+
/**
* Test file creation and access with file names that need encoding.
*/
@@ -280,19 +287,8 @@ public class TestHftpFileSystem {
assertEquals("Stream closed", ioe.getMessage());
}
- public void resetFileSystem() throws IOException {
- // filesystem caching has a quirk/bug that it caches based on the user's
- // given uri. the result is if a filesystem is instantiated with no port,
- // it gets the default port. then if the default port is changed,
- // and another filesystem is instantiated with no port, the prior fs
- // is returned, not a new one using the changed port. so let's flush
- // the cache between tests...
- FileSystem.closeAll();
- }
-
@Test
public void testHftpDefaultPorts() throws IOException {
- resetFileSystem();
Configuration conf = new Configuration();
URI uri = URI.create("hftp://localhost";);
HftpFileSystem fs = (HftpFileSystem) FileSystem.get(uri, conf);
@@ -309,7 +305,6 @@ public class TestHftpFileSystem {
@Test
public void testHftpCustomDefaultPorts() throws IOException {
- resetFileSystem();
Configuration conf = new Configuration();
conf.setInt("dfs.http.port", 123);
conf.setInt("dfs.https.port", 456);
@@ -329,7 +324,6 @@ public class TestHftpFileSystem {
@Test
public void testHftpCustomUriPortWithDefaultPorts() throws IOException {
- resetFileSystem();
Configuration conf = new Configuration();
URI uri = URI.create("hftp://localhost:123";);
HftpFileSystem fs = (HftpFileSystem) FileSystem.get(uri, conf);
@@ -346,7 +340,6 @@ public class TestHftpFileSystem {
@Test
public void testHftpCustomUriPortWithCustomDefaultPorts() throws IOException
{
- resetFileSystem();
Configuration conf = new Configuration();
conf.setInt("dfs.http.port", 123);
conf.setInt("dfs.https.port", 456);
@@ -368,7 +361,6 @@ public class TestHftpFileSystem {
@Test
public void testHsftpDefaultPorts() throws IOException {
- resetFileSystem();
Configuration conf = new Configuration();
URI uri = URI.create("hsftp://localhost";);
HsftpFileSystem fs = (HsftpFileSystem) FileSystem.get(uri, conf);
@@ -385,7 +377,6 @@ public class TestHftpFileSystem {
@Test
public void testHsftpCustomDefaultPorts() throws IOException {
- resetFileSystem();
Configuration conf = new Configuration();
conf.setInt("dfs.http.port", 123);
conf.setInt("dfs.https.port", 456);
@@ -405,7 +396,6 @@ public class TestHftpFileSystem {
@Test
public void testHsftpCustomUriPortWithDefaultPorts() throws IOException {
- resetFileSystem();
Configuration conf = new Configuration();
URI uri = URI.create("hsftp://localhost:123";);
HsftpFileSystem fs = (HsftpFileSystem) FileSystem.get(uri, conf);
@@ -422,7 +412,6 @@ public class TestHftpFileSystem {
@Test
public void testHsftpCustomUriPortWithCustomDefaultPorts() throws
IOException {
- resetFileSystem();
Configuration conf = new Configuration();
conf.setInt("dfs.http.port", 123);
conf.setInt("dfs.https.port", 456);
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java?rev=1405253&r1=1405252&r2=1405253&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
Sat Nov 3 01:38:41 2012
@@ -64,6 +64,7 @@ import org.apache.hadoop.hdfs.protocol.C
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.namenode.NamenodeFsck.Result;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.hdfs.tools.DFSck;
@@ -678,11 +679,11 @@ public class TestFsck {
DFSTestUtil.waitReplication(fs, filePath, (short)1);
// intentionally corrupt NN data structure
- INodeFile node =
- (INodeFile)cluster.getNamesystem().dir.rootDir.getNode(fileName,
- true);
- assertEquals(node.blocks.length, 1);
- node.blocks[0].setNumBytes(-1L); // set the block length to be negative
+ INodeFile node = (INodeFile)cluster.getNamesystem().dir.rootDir.getNode(
+ fileName, true);
+ final BlockInfo[] blocks = node.getBlocks();
+ assertEquals(blocks.length, 1);
+ blocks[0].setNumBytes(-1L); // set the block length to be negative
// run fsck and expect a failure with -1 as the error code
String outStr = runFsck(conf, -1, true, fileName);
