Author: szetszwo
Date: Tue Sep 3 19:01:19 2013
New Revision: 1519796
URL: http://svn.apache.org/r1519796
Log:
Merge r1517887 through r1518850 from trunk.
Added:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
- copied unchanged from r1518850,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java
- copied unchanged from r1518850,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
(props changed)
Propchange: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1517887-1518850
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
Tue Sep 3 19:01:19 2013
@@ -126,6 +126,9 @@ class OpenFileCtx {
nonSequentialWriteInMemory = 0;
this.dumpFilePath = dumpFilePath;
enabledDump = dumpFilePath == null ? false: true;
+ nextOffset = latestAttr.getSize();
+ assert(nextOffset == this.fos.getPos());
+
ctxLock = new ReentrantLock(true);
}
@@ -685,12 +688,14 @@ class OpenFileCtx {
try {
fos.write(data, 0, count);
-
- if (fos.getPos() != (offset + count)) {
+
+ long flushedOffset = getFlushedOffset();
+ if (flushedOffset != (offset + count)) {
throw new IOException("output stream is out of sync, pos="
- + fos.getPos() + " and nextOffset should be" + (offset + count));
+ + flushedOffset + " and nextOffset should be"
+ + (offset + count));
}
- nextOffset = fos.getPos();
+ nextOffset = flushedOffset;
// Reduce memory occupation size if request was allowed dumped
if (writeCtx.getDataState() == DataState.ALLOW_DUMP) {
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
Tue Sep 3 19:01:19 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.FsStatus;
import org.apache.hadoop.fs.Options;
@@ -123,7 +124,7 @@ public class RpcProgramNfs3 extends RpcP
private final Configuration config = new Configuration();
private final WriteManager writeManager;
- private final IdUserGroup iug;// = new IdUserGroup();
+ private final IdUserGroup iug;
private final DFSClientCache clientCache;
private final NfsExports exports;
@@ -161,10 +162,14 @@ public class RpcProgramNfs3 extends RpcP
DFSConfigKeys.DFS_REPLICATION_DEFAULT);
blockSize = config.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
- bufferSize = config.getInt("io.file.buffer.size", 4096);
-
- writeDumpDir = config.get("dfs.nfs3.dump.dir", "/tmp/.hdfs-nfs");
- boolean enableDump = config.getBoolean("dfs.nfs3.enableDump", true);
+ bufferSize = config.getInt(
+ CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
+ CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
+
+ writeDumpDir = config.get(Nfs3Constant.FILE_DUMP_DIR_KEY,
+ Nfs3Constant.FILE_DUMP_DIR_DEFAULT);
+ boolean enableDump = config.getBoolean(Nfs3Constant.ENABLE_FILE_DUMP_KEY,
+ Nfs3Constant.ENABLE_FILE_DUMP_DEFAULT);
if (!enableDump) {
writeDumpDir = null;
} else {
@@ -1112,6 +1117,7 @@ public class RpcProgramNfs3 extends RpcP
}
}
+ @Override
public SYMLINK3Response symlink(XDR xdr, RpcAuthSys authSys,
InetAddress client) {
return new SYMLINK3Response(Nfs3Status.NFS3ERR_NOTSUPP);
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
Tue Sep 3 19:01:19 2013
@@ -25,7 +25,9 @@ import java.util.concurrent.ConcurrentMa
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.nfs.NfsFileType;
import org.apache.hadoop.nfs.nfs3.FileHandle;
import org.apache.hadoop.nfs.nfs3.IdUserGroup;
@@ -48,6 +50,7 @@ import com.google.common.collect.Maps;
public class WriteManager {
public static final Log LOG = LogFactory.getLog(WriteManager.class);
+ private final Configuration config;
private final IdUserGroup iug;
private final ConcurrentMap<FileHandle, OpenFileCtx> openFileMap = Maps
.newConcurrentMap();
@@ -76,6 +79,7 @@ public class WriteManager {
WriteManager(IdUserGroup iug, final Configuration config) {
this.iug = iug;
+ this.config = config;
streamTimeout = config.getLong("dfs.nfs3.stream.timeout",
DEFAULT_STREAM_TIMEOUT);
@@ -129,12 +133,41 @@ public class WriteManager {
OpenFileCtx openFileCtx = openFileMap.get(fileHandle);
if (openFileCtx == null) {
LOG.info("No opened stream for fileId:" + fileHandle.getFileId());
- WccData fileWcc = new WccData(Nfs3Utils.getWccAttr(preOpAttr),
preOpAttr);
- WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_IO,
- fileWcc, count, request.getStableHow(),
- Nfs3Constant.WRITE_COMMIT_VERF);
- Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid));
- return;
+
+ String fileIdPath = Nfs3Utils.getFileIdPath(fileHandle.getFileId());
+ HdfsDataOutputStream fos = null;
+ Nfs3FileAttributes latestAttr = null;
+ try {
+ int bufferSize = config.getInt(
+ CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
+ CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
+
+ fos = dfsClient.append(fileIdPath, bufferSize, null, null);
+
+ latestAttr = Nfs3Utils.getFileAttr(dfsClient, fileIdPath, iug);
+ } catch (IOException e) {
+ LOG.error("Can't apapend to file:" + fileIdPath + ", error:" + e);
+ if (fos != null) {
+ fos.close();
+ }
+ WccData fileWcc = new WccData(Nfs3Utils.getWccAttr(preOpAttr),
+ preOpAttr);
+ WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_IO,
+ fileWcc, count, request.getStableHow(),
+ Nfs3Constant.WRITE_COMMIT_VERF);
+ Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid));
+ return;
+ }
+
+ // Add open stream
+ String writeDumpDir = config.get(Nfs3Constant.FILE_DUMP_DIR_KEY,
+ Nfs3Constant.FILE_DUMP_DIR_DEFAULT);
+ openFileCtx = new OpenFileCtx(fos, latestAttr, writeDumpDir + "/"
+ + fileHandle.getFileId());
+ addOpenFileStream(fileHandle, openFileCtx);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("opened stream for file:" + fileHandle.getFileId());
+ }
}
// Add write into the async job queue
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Tue Sep 3 19:01:19 2013
@@ -307,6 +307,9 @@ Release 2.1.1-beta - UNRELEASED
HDFS-4947 Add NFS server export table to control export by hostname or
IP range (Jing Zhao via brandonli)
+ HDFS-5078 Support file append in NFSv3 gateway to enable data streaming
+ to HDFS (brandonli)
+
IMPROVEMENTS
HDFS-4513. Clarify in the WebHDFS REST API that all JSON respsonses may
@@ -333,6 +336,12 @@ Release 2.1.1-beta - UNRELEASED
HDFS-5045. Add more unit tests for retry cache to cover all AtMostOnce
methods. (jing9)
+ HDFS-3245. Add metrics and web UI for cluster version summary. (Ravi
+ Prakash via kihwal)
+
+ HDFS-5128. Allow multiple net interfaces to be used with HA namenode RPC
+ server. (kihwal)
+
OPTIMIZATIONS
BUG FIXES
@@ -389,6 +398,9 @@ Release 2.1.1-beta - UNRELEASED
HDFS-5124. DelegationTokenSecretManager#retrievePassword can cause
deadlock
in NameNode. (Daryn Sharp via jing9)
+ HDFS-5132. Deadlock in NameNode between SafeModeMonitor#run and
+ DatanodeManager#handleHeartbeat. (kihwal)
+
Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1517029-1518850
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
Tue Sep 3 19:01:19 2013
@@ -104,7 +104,9 @@ public class DFSConfigKeys extends Commo
public static final String DFS_NAMENODE_HTTP_ADDRESS_KEY =
"dfs.namenode.http-address";
public static final String DFS_NAMENODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:" +
DFS_NAMENODE_HTTP_PORT_DEFAULT;
public static final String DFS_NAMENODE_RPC_ADDRESS_KEY =
"dfs.namenode.rpc-address";
+ public static final String DFS_NAMENODE_RPC_BIND_HOST_KEY =
"dfs.namenode.rpc-bind-host";
public static final String DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY =
"dfs.namenode.servicerpc-address";
+ public static final String DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY =
"dfs.namenode.servicerpc-bind-host";
public static final String DFS_NAMENODE_MAX_OBJECTS_KEY =
"dfs.namenode.max.objects";
public static final long DFS_NAMENODE_MAX_OBJECTS_DEFAULT = 0;
public static final String DFS_NAMENODE_SAFEMODE_EXTENSION_KEY =
"dfs.namenode.safemode.extension";
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
Tue Sep 3 19:01:19 2013
@@ -47,6 +47,7 @@ public class DatanodeInfo extends Datano
private long lastUpdate;
private int xceiverCount;
private String location = NetworkTopology.DEFAULT_RACK;
+ private String softwareVersion;
// Datanode administrative states
public enum AdminStates {
@@ -383,4 +384,12 @@ public class DatanodeInfo extends Datano
// by DatanodeID
return (this == obj) || super.equals(obj);
}
+
+ public String getSoftwareVersion() {
+ return softwareVersion;
+ }
+
+ public void setSoftwareVersion(String softwareVersion) {
+ this.softwareVersion = softwareVersion;
+ }
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
Tue Sep 3 19:01:19 2013
@@ -26,6 +26,7 @@ import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.NavigableMap;
@@ -165,6 +166,14 @@ public class DatanodeManager {
* according to the NetworkTopology.
*/
private boolean hasClusterEverBeenMultiRack = false;
+
+ /**
+ * The number of datanodes for each software version. This list should change
+ * during rolling upgrades.
+ * Software version -> Number of datanodes with this version
+ */
+ private HashMap<String, Integer> datanodesSoftwareVersions =
+ new HashMap<String, Integer>(4, 0.75f);
DatanodeManager(final BlockManager blockManager, final Namesystem namesystem,
final Configuration conf) throws IOException {
@@ -470,6 +479,7 @@ public class DatanodeManager {
heartbeatManager.removeDatanode(nodeInfo);
blockManager.removeBlocksAssociatedTo(nodeInfo);
networktopology.remove(nodeInfo);
+ decrementVersionCount(nodeInfo.getSoftwareVersion());
if (LOG.isDebugEnabled()) {
LOG.debug("remove datanode " + nodeInfo);
@@ -552,6 +562,61 @@ public class DatanodeManager {
}
}
+ private void incrementVersionCount(String version) {
+ if (version == null) {
+ return;
+ }
+ synchronized(datanodeMap) {
+ Integer count = this.datanodesSoftwareVersions.get(version);
+ count = count == null ? 1 : count + 1;
+ this.datanodesSoftwareVersions.put(version, count);
+ }
+ }
+
+ private void decrementVersionCount(String version) {
+ if (version == null) {
+ return;
+ }
+ synchronized(datanodeMap) {
+ Integer count = this.datanodesSoftwareVersions.get(version);
+ if(count != null) {
+ if(count > 1) {
+ this.datanodesSoftwareVersions.put(version, count-1);
+ } else {
+ this.datanodesSoftwareVersions.remove(version);
+ }
+ }
+ }
+ }
+
+ private boolean shouldCountVersion(DatanodeDescriptor node) {
+ return node.getSoftwareVersion() != null && node.isAlive &&
+ !isDatanodeDead(node);
+ }
+
+ private void countSoftwareVersions() {
+ synchronized(datanodeMap) {
+ HashMap<String, Integer> versionCount = new HashMap<String, Integer>();
+ for(DatanodeDescriptor dn: datanodeMap.values()) {
+ // Check isAlive too because right after removeDatanode(),
+ // isDatanodeDead() is still true
+ if(shouldCountVersion(dn))
+ {
+ Integer num = versionCount.get(dn.getSoftwareVersion());
+ num = num == null ? 1 : num+1;
+ versionCount.put(dn.getSoftwareVersion(), num);
+ }
+ }
+ this.datanodesSoftwareVersions = versionCount;
+ }
+ }
+
+ public HashMap<String, Integer> getDatanodesSoftwareVersions() {
+ synchronized(datanodeMap) {
+ return new HashMap<String, Integer> (this.datanodesSoftwareVersions);
+ }
+ }
+
/* Resolve a node's network location */
private String resolveNetworkLocation (DatanodeID node) {
List<String> names = new ArrayList<String>(1);
@@ -751,21 +816,28 @@ public class DatanodeManager {
try {
// update cluster map
getNetworkTopology().remove(nodeS);
+ if(shouldCountVersion(nodeS)) {
+ decrementVersionCount(nodeS.getSoftwareVersion());
+ }
nodeS.updateRegInfo(nodeReg);
+
+ nodeS.setSoftwareVersion(nodeReg.getSoftwareVersion());
nodeS.setDisallowed(false); // Node is in the include list
-
+
// resolve network location
nodeS.setNetworkLocation(resolveNetworkLocation(nodeS));
getNetworkTopology().add(nodeS);
// also treat the registration message as a heartbeat
heartbeatManager.register(nodeS);
+ incrementVersionCount(nodeS.getSoftwareVersion());
checkDecommissioning(nodeS);
success = true;
} finally {
if (!success) {
removeDatanode(nodeS);
wipeDatanode(nodeS);
+ countSoftwareVersions();
}
}
return;
@@ -789,6 +861,7 @@ public class DatanodeManager {
try {
nodeDescr.setNetworkLocation(resolveNetworkLocation(nodeDescr));
networktopology.add(nodeDescr);
+ nodeDescr.setSoftwareVersion(nodeReg.getSoftwareVersion());
// register new datanode
addDatanode(nodeDescr);
@@ -799,10 +872,12 @@ public class DatanodeManager {
// because its is done when the descriptor is created
heartbeatManager.addDatanode(nodeDescr);
success = true;
+ incrementVersionCount(nodeReg.getSoftwareVersion());
} finally {
if (!success) {
removeDatanode(nodeDescr);
wipeDatanode(nodeDescr);
+ countSoftwareVersions();
}
}
} catch (InvalidTopologyException e) {
@@ -824,6 +899,7 @@ public class DatanodeManager {
namesystem.writeLock();
try {
refreshDatanodes();
+ countSoftwareVersions();
} finally {
namesystem.writeUnlock();
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java
Tue Sep 3 19:01:19 2013
@@ -359,6 +359,7 @@ class ClusterJspHelper {
nn.httpAddress = httpAddress;
getLiveNodeCount(getProperty(props, "LiveNodes").getValueAsText(), nn);
getDeadNodeCount(getProperty(props, "DeadNodes").getValueAsText(), nn);
+ nn.softwareVersion = getProperty(props,
"SoftwareVersion").getTextValue();
return nn;
}
@@ -596,6 +597,7 @@ class ClusterJspHelper {
toXmlItemBlockWithLink(doc, nn.deadDatanodeCount + " (" +
nn.deadDecomCount + ")",
nn.httpAddress+"/dfsnodelist.jsp?whatNodes=DEAD"
, "Dead Datanode (Decommissioned)");
+ toXmlItemBlock(doc, "Software Version", nn.softwareVersion);
doc.endTag(); // node
}
doc.endTag(); // namenodes
@@ -624,6 +626,7 @@ class ClusterJspHelper {
int deadDatanodeCount = 0;
int deadDecomCount = 0;
String httpAddress = null;
+ String softwareVersion = "";
}
/**
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Tue Sep 3 19:01:19 2013
@@ -4804,7 +4804,21 @@ public class FSNamesystem implements Nam
*/
@Override
public void run() {
- while (fsRunning && (safeMode != null && !safeMode.canLeave())) {
+ while (fsRunning) {
+ writeLock();
+ try {
+ if (safeMode == null) { // Not in safe mode.
+ break;
+ }
+ if (safeMode.canLeave()) {
+ // Leave safe mode.
+ safeMode.leave();
+ break;
+ }
+ } finally {
+ writeUnlock();
+ }
+
try {
Thread.sleep(recheckInterval);
} catch (InterruptedException ie) {
@@ -4813,9 +4827,6 @@ public class FSNamesystem implements Nam
}
if (!fsRunning) {
LOG.info("NameNode is being shutdown, exit SafeModeMonitor thread");
- } else {
- // leave safe mode and stop the monitor
- leaveSafeMode();
}
smmthread = null;
}
@@ -6226,6 +6237,7 @@ public class FSNamesystem implements Nam
innerinfo.put("nonDfsUsedSpace", node.getNonDfsUsed());
innerinfo.put("capacity", node.getCapacity());
innerinfo.put("numBlocks", node.numBlocks());
+ innerinfo.put("version", node.getSoftwareVersion());
info.put(node.getHostName(), innerinfo);
}
return JSON.toString(info);
@@ -6437,6 +6449,22 @@ public class FSNamesystem implements Nam
return JSON.toString(list);
}
+ @Override //NameNodeMXBean
+ public int getDistinctVersionCount() {
+ return blockManager.getDatanodeManager().getDatanodesSoftwareVersions()
+ .size();
+ }
+
+ @Override //NameNodeMXBean
+ public Map<String, Integer> getDistinctVersions() {
+ return blockManager.getDatanodeManager().getDatanodesSoftwareVersions();
+ }
+
+ @Override //NameNodeMXBean
+ public String getSoftwareVersion() {
+ return VersionInfo.getVersion();
+ }
+
/**
* Verifies that the given identifier and password are valid and match.
* @param identifier Token identifier.
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Tue Sep 3 19:01:19 2013
@@ -166,12 +166,14 @@ public class NameNode implements NameNod
*/
public static final String[] NAMENODE_SPECIFIC_KEYS = {
DFS_NAMENODE_RPC_ADDRESS_KEY,
+ DFS_NAMENODE_RPC_BIND_HOST_KEY,
DFS_NAMENODE_NAME_DIR_KEY,
DFS_NAMENODE_EDITS_DIR_KEY,
DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
DFS_NAMENODE_CHECKPOINT_DIR_KEY,
DFS_NAMENODE_CHECKPOINT_EDITS_DIR_KEY,
DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
+ DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY,
DFS_NAMENODE_HTTP_ADDRESS_KEY,
DFS_NAMENODE_KEYTAB_FILE_KEY,
DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,
@@ -387,6 +389,28 @@ public class NameNode implements NameNod
return getAddress(conf);
}
+ /** Given a configuration get the bind host of the service rpc server
+ * If the bind host is not configured returns null.
+ */
+ protected String getServiceRpcServerBindHost(Configuration conf) {
+ String addr = conf.getTrimmed(DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY);
+ if (addr == null || addr.isEmpty()) {
+ return null;
+ }
+ return addr;
+ }
+
+ /** Given a configuration get the bind host of the client rpc server
+ * If the bind host is not configured returns null.
+ */
+ protected String getRpcServerBindHost(Configuration conf) {
+ String addr = conf.getTrimmed(DFS_NAMENODE_RPC_BIND_HOST_KEY);
+ if (addr == null || addr.isEmpty()) {
+ return null;
+ }
+ return addr;
+ }
+
/**
* Modifies the configuration passed to contain the service rpc address
setting
*/
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
Tue Sep 3 19:01:19 2013
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import java.util.Map;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -33,7 +35,13 @@ public interface NameNodeMXBean {
* @return the version
*/
public String getVersion();
-
+
+ /**
+ * Get the version of software running on the Namenode
+ * @return a string representing the version
+ */
+ public String getSoftwareVersion();
+
/**
* Gets the used space by data nodes.
*
@@ -215,4 +223,19 @@ public interface NameNodeMXBean {
* @return the list of corrupt files, as a JSON string.
*/
public String getCorruptFiles();
+
+ /**
+ * Get the number of distinct versions of live datanodes
+ *
+ * @return the number of distinct versions of live datanodes
+ */
+ public int getDistinctVersionCount();
+
+ /**
+ * Get the number of live datanodes for each distinct versions
+ *
+ * @return the number of live datanodes for each distinct versions
+ */
+ public Map<String, Integer> getDistinctVersions();
+
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
Tue Sep 3 19:01:19 2013
@@ -137,6 +137,7 @@ import org.apache.hadoop.tools.protocolP
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.util.VersionUtil;
+import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.BlockingService;
/**
@@ -219,6 +220,13 @@ class NameNodeRpcServer implements Namen
InetSocketAddress serviceRpcAddr = nn.getServiceRpcServerAddress(conf);
if (serviceRpcAddr != null) {
+ String bindHost = nn.getServiceRpcServerBindHost(conf);
+ if (bindHost == null) {
+ bindHost = serviceRpcAddr.getHostName();
+ }
+ LOG.info("Service RPC server is binding to " + bindHost + ":" +
+ serviceRpcAddr.getPort());
+
int serviceHandlerCount =
conf.getInt(DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY,
DFS_NAMENODE_SERVICE_HANDLER_COUNT_DEFAULT);
@@ -226,7 +234,7 @@ class NameNodeRpcServer implements Namen
.setProtocol(
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB.class)
.setInstance(clientNNPbService)
- .setBindAddress(serviceRpcAddr.getHostName())
+ .setBindAddress(bindHost)
.setPort(serviceRpcAddr.getPort())
.setNumHandlers(serviceHandlerCount)
.setVerbose(false)
@@ -247,7 +255,10 @@ class NameNodeRpcServer implements Namen
DFSUtil.addPBProtocol(conf, GetUserMappingsProtocolPB.class,
getUserMappingService, serviceRpcServer);
- serviceRPCAddress = serviceRpcServer.getListenerAddress();
+ // Update the address with the correct port
+ InetSocketAddress listenAddr = serviceRpcServer.getListenerAddress();
+ serviceRPCAddress = new InetSocketAddress(
+ serviceRpcAddr.getHostName(), listenAddr.getPort());
nn.setRpcServiceServerAddress(conf, serviceRPCAddress);
} else {
serviceRpcServer = null;
@@ -255,11 +266,17 @@ class NameNodeRpcServer implements Namen
}
InetSocketAddress rpcAddr = nn.getRpcServerAddress(conf);
+ String bindHost = nn.getRpcServerBindHost(conf);
+ if (bindHost == null) {
+ bindHost = rpcAddr.getHostName();
+ }
+ LOG.info("RPC server is binding to " + bindHost + ":" + rpcAddr.getPort());
+
clientRpcServer = new RPC.Builder(conf)
.setProtocol(
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB.class)
.setInstance(clientNNPbService)
- .setBindAddress(rpcAddr.getHostName())
+ .setBindAddress(bindHost)
.setPort(rpcAddr.getPort())
.setNumHandlers(handlerCount)
.setVerbose(false)
@@ -291,7 +308,9 @@ class NameNodeRpcServer implements Namen
}
// The rpc-server port can be ephemeral... ensure we have the correct info
- clientRpcAddress = clientRpcServer.getListenerAddress();
+ InetSocketAddress listenAddr = clientRpcServer.getListenerAddress();
+ clientRpcAddress = new InetSocketAddress(
+ rpcAddr.getHostName(), listenAddr.getPort());
nn.setRpcServerAddress(conf, clientRpcAddress);
minimumDataNodeVersion = conf.get(
@@ -315,6 +334,12 @@ class NameNodeRpcServer implements Namen
NSQuotaExceededException.class,
DSQuotaExceededException.class);
}
+
+ /** Allow access to the client RPC server for testing */
+ @VisibleForTesting
+ RPC.Server getClientRpcServer() {
+ return clientRpcServer;
+ }
/**
* Start client and service RPC servers.
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
Tue Sep 3 19:01:19 2013
@@ -32,6 +32,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
@@ -100,6 +101,20 @@ class NamenodeJspHelper {
}
}
+ static String getRollingUpgradeText(FSNamesystem fsn) {
+ DatanodeManager dm = fsn.getBlockManager().getDatanodeManager();
+ Map<String, Integer> list = dm.getDatanodesSoftwareVersions();
+ if(list.size() > 1) {
+ StringBuffer status = new StringBuffer("Rolling upgrades in progress. " +
+ "There are " + list.size() + " versions of datanodes currently live: ");
+ for(Map.Entry<String, Integer> ver: list.entrySet()) {
+ status.append(ver.getKey() + "(" + ver.getValue() + "), ");
+ }
+ return status.substring(0, status.length()-2);
+ }
+ return "";
+ }
+
static String getInodeLimitText(FSNamesystem fsn) {
if (fsn == null) {
return "";
@@ -803,7 +818,9 @@ class NamenodeJspHelper {
+ "<td align=\"right\" class=\"pcbpused\">"
+ percentBpUsed
+ "<td align=\"right\" class=\"volfails\">"
- + d.getVolumeFailures() + "\n");
+ + d.getVolumeFailures()
+ + "<td align=\"right\" class=\"version\">"
+ + d.getSoftwareVersion() + "\n");
}
void generateNodesList(ServletContext context, JspWriter out,
@@ -901,7 +918,9 @@ class NamenodeJspHelper {
+ nodeHeaderStr("pcbpused")
+ "> Block Pool<br>Used (%)" + " <th "
+ nodeHeaderStr("volfails")
- +"> Failed Volumes\n");
+ +"> Failed Volumes <th "
+ + nodeHeaderStr("versionString")
+ +"> Version\n");
JspHelper.sortNodeList(live, sorterField, sorterOrder);
for (int i = 0; i < live.size(); i++) {
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1513717-1518850
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
Tue Sep 3 19:01:19 2013
@@ -52,6 +52,18 @@
</property>
<property>
+ <name>dfs.namenode.rpc-bind-host</name>
+ <value></value>
+ <description>
+ The actual address the server will bind to. If this optional address is
+ set, the RPC server will bind to this address and the port specified in
+ dfs.namenode.rpc-address for the RPC server. It can also be specified
+ per name node or name service for HA/Federation. This is most useful for
+ making name node listen to all interfaces by setting to 0.0.0.0.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.servicerpc-address</name>
<value></value>
<description>
@@ -65,6 +77,18 @@
</property>
<property>
+ <name>dfs.namenode.servicerpc-bind-host</name>
+ <value></value>
+ <description>
+ The actual address the server will bind to. If this optional address is
+ set, the service RPC server will bind to this address and the port
+ specified in dfs.namenode.servicerpc-address. It can also be specified
+ per name node or name service for HA/Federation. This is most useful for
+ making name node listen to all interfaces by setting to 0.0.0.0.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.secondary.http-address</name>
<value>0.0.0.0:50090</value>
<description>
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1516229-1518850
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1516229-1518850
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp
Tue Sep 3 19:01:19 2013
@@ -65,6 +65,7 @@
<h3>Cluster Summary</h3>
<b> <%= NamenodeJspHelper.getSecurityModeText()%> </b>
<b> <%= NamenodeJspHelper.getSafeModeText(fsn)%> </b>
+<b> <%= NamenodeJspHelper.getRollingUpgradeText(fsn)%> </b>
<b> <%= NamenodeJspHelper.getInodeLimitText(fsn)%> </b>
<%= NamenodeJspHelper.getCorruptFilesWarning(fsn)%>
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1513717-1518850
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1513717-1518850
