Author: szetszwo
Date: Wed Jan 15 06:06:31 2014
New Revision: 1558303
URL: http://svn.apache.org/r1558303
Log:
Merge r1555021 through r1558254 from trunk.
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
(props changed)
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
Propchange: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1557039-1558254
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1558303&r1=1558302&r2=1558303&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Wed Jan 15 06:06:31 2014
@@ -741,6 +741,17 @@ Release 2.4.0 - UNRELEASED
HDFS-5449. WebHdfs compatibility broken between 2.2 and 1.x / 23.x (kihwal)
+ HDFS-5756. hadoopRzOptionsSetByteBufferPool does not accept NULL argument,
+ contrary to docs. (cmccabe via wang)
+
+ HDFS-5747. Fix NPEs in BlockManager. (Arpit Agarwal)
+
+ HDFS-5710. FSDirectory#getFullPathName should check inodes against null.
+ (Uma Maheswara Rao G via jing9)
+
+ HDFS-5579. Under construction files make DataNode decommission take very
long
+ hours. (zhaoyunjiong via jing9)
+
BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS
HDFS-4985. Add storage type to the protocol and expose it in block report
@@ -913,6 +924,9 @@ Release 2.3.0 - UNRELEASED
HDFS-5675. Add Mkdirs operation to NNThroughputBenchmark.
(Plamen Jeliazkov via shv)
+ HDFS-5677. Need error checking for HA cluster configuration.
+ (Vincent Sheffer via cos)
+
OPTIMIZATIONS
BUG FIXES
@@ -2293,6 +2307,16 @@ Release 2.1.0-beta - 2013-08-22
HDFS-4982. JournalNode should relogin from keytab before fetching logs
from other JNs (todd)
+Release 2.0.6-alpha - 08/22/2013
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
Release 2.0.5-alpha - 06/06/2013
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1557039-1558254
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1558303&r1=1558302&r2=1558303&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
Wed Jan 15 06:06:31 2014
@@ -585,6 +585,12 @@ public class DFSUtil {
String address = getConfValue(defaultValue, suffix, conf, keys);
if (address != null) {
InetSocketAddress isa = NetUtils.createSocketAddr(address);
+ if (isa.isUnresolved()) {
+ LOG.warn("Namenode for " + nsId +
+ " remains unresolved for ID " + nnId +
+ ". Check your hdfs-site.xml file to " +
+ "ensure namenodes are configured properly.");
+ }
ret.put(nnId, isa);
}
}
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java?rev=1558303&r1=1558302&r2=1558303&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
Wed Jan 15 06:06:31 2014
@@ -31,7 +31,7 @@ public interface BlockCollection {
/**
* Get the last block of the collection.
*/
- public BlockInfo getLastBlock() throws IOException;
+ public BlockInfo getLastBlock();
/**
* Get content summary.
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java?rev=1558303&r1=1558302&r2=1558303&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
Wed Jan 15 06:06:31 2014
@@ -324,12 +324,14 @@ public class BlockInfoUnderConstruction
Iterator<ReplicaUnderConstruction> it = replicas.iterator();
while (it.hasNext()) {
ReplicaUnderConstruction r = it.next();
- if(r.getExpectedStorageLocation() == storage) {
+ DatanodeStorageInfo expectedLocation = r.getExpectedStorageLocation();
+ if(expectedLocation == storage) {
// Record the gen stamp from the report
r.setGenerationStamp(block.getGenerationStamp());
return;
- } else if (r.getExpectedStorageLocation().getDatanodeDescriptor() ==
- storage.getDatanodeDescriptor()) {
+ } else if (expectedLocation != null &&
+ expectedLocation.getDatanodeDescriptor() ==
+ storage.getDatanodeDescriptor()) {
// The Datanode reported that the block is on a different storage
// than the one chosen by BlockPlacementPolicy. This can occur as
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1558303&r1=1558302&r2=1558303&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
Wed Jan 15 06:06:31 2014
@@ -1233,8 +1233,10 @@ public class BlockManager {
// block should belong to a file
bc = blocksMap.getBlockCollection(block);
// abandoned block or block reopened for append
- if(bc == null || bc.isUnderConstruction()) {
- neededReplications.remove(block, priority); // remove from
neededReplications
+ if (bc == null
+ || (bc.isUnderConstruction() &&
block.equals(bc.getLastBlock()))) {
+ // remove from neededReplications
+ neededReplications.remove(block, priority);
continue;
}
@@ -1314,7 +1316,7 @@ public class BlockManager {
// block should belong to a file
bc = blocksMap.getBlockCollection(block);
// abandoned block or block reopened for append
- if(bc == null || bc.isUnderConstruction()) {
+ if(bc == null || (bc.isUnderConstruction() &&
block.equals(bc.getLastBlock()))) {
neededReplications.remove(block, priority); // remove from
neededReplications
rw.targets = null;
continue;
@@ -3007,8 +3009,16 @@ assert storedBlock.findDatanode(dn) < 0
NumberReplicas num = countNodes(block);
int curReplicas = num.liveReplicas();
int curExpectedReplicas = getReplication(block);
+
if (isNeededReplication(block, curExpectedReplicas, curReplicas)) {
if (curExpectedReplicas > curReplicas) {
+ if (bc.isUnderConstruction()) {
+ if (block.equals(bc.getLastBlock()) && curReplicas >
minReplication) {
+ continue;
+ }
+ underReplicatedInOpenFiles++;
+ }
+
// Log info about one block for this node which needs replication
if (!status) {
status = true;
@@ -3025,9 +3035,6 @@ assert storedBlock.findDatanode(dn) < 0
if ((curReplicas == 0) && (num.decommissionedReplicas() > 0)) {
decommissionOnlyReplicas++;
}
- if (bc.isUnderConstruction()) {
- underReplicatedInOpenFiles++;
- }
}
if (!neededReplications.contains(block) &&
pendingReplications.getNumReplicas(block) == 0) {
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1558303&r1=1558302&r2=1558303&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Wed Jan 15 06:06:31 2014
@@ -1842,7 +1842,8 @@ public class FSDirectory implements Clos
/** Return the full path name of the specified inode */
static String getFullPathName(INode inode) {
INode[] inodes = getFullPathINodes(inode);
- return getFullPathName(inodes, inodes.length - 1);
+ // inodes can be null only when its called without holding lock
+ return inodes == null ? "" : getFullPathName(inodes, inodes.length - 1);
}
/**
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1558303&r1=1558302&r2=1558303&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
Wed Jan 15 06:06:31 2014
@@ -640,7 +640,7 @@ public class INodeFile extends INodeWith
}
@Override
- public BlockInfo getLastBlock() throws IOException {
+ public BlockInfo getLastBlock() {
return blocks == null || blocks.length == 0? null: blocks[blocks.length-1];
}
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1558303&r1=1558302&r2=1558303&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Wed Jan 15 06:06:31 2014
@@ -547,8 +547,8 @@ public class NameNode implements NameNod
}
private void stopCommonServices() {
- if(namesystem != null) namesystem.close();
if(rpcServer != null) rpcServer.stop();
+ if(namesystem != null) namesystem.close();
if (pauseMonitor != null) pauseMonitor.stop();
if (plugins != null) {
for (ServicePlugin p : plugins) {
Propchange:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1555021-1558254
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c?rev=1558303&r1=1558302&r2=1558303&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
Wed Jan 15 06:06:31 2014
@@ -2174,16 +2174,18 @@ int hadoopRzOptionsSetByteBufferPool(
return -1;
}
- // Note: we don't have to call hadoopRzOptionsClearCached in this
- // function, since the ByteBufferPool is passed separately from the
- // EnumSet of ReadOptions.
-
- jthr = constructNewObjectOfClass(env, &byteBufferPool, className, "()V");
- if (jthr) {
- printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
- "hadoopRzOptionsSetByteBufferPool(className=%s): ", className);
- errno = EINVAL;
- return -1;
+ if (className) {
+ // Note: we don't have to call hadoopRzOptionsClearCached in this
+ // function, since the ByteBufferPool is passed separately from the
+ // EnumSet of ReadOptions.
+
+ jthr = constructNewObjectOfClass(env, &byteBufferPool, className, "()V");
+ if (jthr) {
+ printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "hadoopRzOptionsSetByteBufferPool(className=%s): ", className);
+ errno = EINVAL;
+ return -1;
+ }
}
if (opts->byteBufferPool) {
// Delete any previous ByteBufferPool we had.
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c?rev=1558303&r1=1558302&r2=1558303&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c
Wed Jan 15 06:06:31 2014
@@ -140,6 +140,12 @@ static int doTestZeroCopyReads(hdfsFS fs
EXPECT_NULL(hadoopReadZero(file, opts, TEST_ZEROCOPY_FULL_BLOCK_SIZE));
EXPECT_INT_EQ(EPROTONOSUPPORT, errno);
+ /* Verify that setting a NULL ByteBufferPool class works. */
+ EXPECT_ZERO(hadoopRzOptionsSetByteBufferPool(opts, NULL));
+ EXPECT_ZERO(hadoopRzOptionsSetSkipChecksum(opts, 0));
+ EXPECT_NULL(hadoopReadZero(file, opts, TEST_ZEROCOPY_FULL_BLOCK_SIZE));
+ EXPECT_INT_EQ(EPROTONOSUPPORT, errno);
+
/* Now set a ByteBufferPool and try again. It should succeed this time. */
EXPECT_ZERO(hadoopRzOptionsSetByteBufferPool(opts,
ELASTIC_BYTE_BUFFER_POOL_CLASS));
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java?rev=1558303&r1=1558302&r2=1558303&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
Wed Jan 15 06:06:31 2014
@@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.protocol.D
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
@@ -779,4 +780,53 @@ public class TestDecommission {
Thread.sleep(HEARTBEAT_INTERVAL * 1000);
}
}
+
+ @Test(timeout=120000)
+ public void testDecommissionWithOpenfile() throws IOException,
InterruptedException {
+ LOG.info("Starting test testDecommissionWithOpenfile");
+
+ //At most 4 nodes will be decommissioned
+ startCluster(1, 7, conf);
+
+ FileSystem fileSys = cluster.getFileSystem(0);
+ FSNamesystem ns = cluster.getNamesystem(0);
+
+ String openFile = "/testDecommissionWithOpenfile.dat";
+
+ writeFile(fileSys, new Path(openFile), (short)3);
+ // make sure the file was open for write
+ FSDataOutputStream fdos = fileSys.append(new Path(openFile));
+
+ LocatedBlocks lbs =
NameNodeAdapter.getBlockLocations(cluster.getNameNode(0), openFile, 0,
fileSize);
+
+ DatanodeInfo[] dnInfos4LastBlock =
lbs.getLastLocatedBlock().getLocations();
+ DatanodeInfo[] dnInfos4FirstBlock = lbs.get(0).getLocations();
+
+ ArrayList<String> nodes = new ArrayList<String>();
+ ArrayList<DatanodeInfo> dnInfos = new ArrayList<DatanodeInfo>();
+
+ for (DatanodeInfo datanodeInfo : dnInfos4FirstBlock) {
+ DatanodeInfo found = datanodeInfo;
+ for (DatanodeInfo dif: dnInfos4LastBlock) {
+ if (datanodeInfo.equals(dif)) {
+ found = null;
+ }
+ }
+ if (found != null) {
+ nodes.add(found.getXferAddr());
+ dnInfos.add(found);
+ }
+ }
+ //decommission one of the 3 nodes which have last block
+ nodes.add(dnInfos4LastBlock[0].getXferAddr());
+ dnInfos.add(dnInfos4LastBlock[0]);
+
+ writeConfigFile(excludeFile, nodes);
+ refreshNodes(ns, conf);
+ for (DatanodeInfo dn : dnInfos) {
+ waitNodeState(dn, AdminStates.DECOMMISSIONED);
+ }
+
+ fdos.close();
+ }
}
