Author: arp
Date: Thu Apr 24 17:40:30 2014
New Revision: 1589803
URL: http://svn.apache.org/r1589803
Log:
HDFS-6273. Config options to allow wildcard endpoints for namenode HTTP and
HTTPS servers. (Contributed by Arpit Agarwal)
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1589803&r1=1589802&r2=1589803&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Apr 24
17:40:30 2014
@@ -314,6 +314,9 @@ Release 2.5.0 - UNRELEASED
HDFS-5693. Few NN metrics data points were collected via JMX when NN
is under heavy load. (Ming Ma via jing9)
+ HDFS-6273. Config options to allow wildcard endpoints for namenode HTTP
+ and HTTPS servers. (Arpit Agarwal)
+
OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1589803&r1=1589802&r2=1589803&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
Thu Apr 24 17:40:30 2014
@@ -126,6 +126,7 @@ public class DFSConfigKeys extends Commo
public static final int DFS_NAMENODE_HTTP_PORT_DEFAULT = 50070;
public static final String DFS_NAMENODE_HTTP_ADDRESS_KEY =
"dfs.namenode.http-address";
public static final String DFS_NAMENODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:" +
DFS_NAMENODE_HTTP_PORT_DEFAULT;
+ public static final String DFS_NAMENODE_HTTP_BIND_HOST_KEY =
"dfs.namenode.http-bind-host";
public static final String DFS_NAMENODE_RPC_ADDRESS_KEY =
"dfs.namenode.rpc-address";
public static final String DFS_NAMENODE_RPC_BIND_HOST_KEY =
"dfs.namenode.rpc-bind-host";
public static final String DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY =
"dfs.namenode.servicerpc-address";
@@ -295,6 +296,7 @@ public class DFSConfigKeys extends Commo
public static final String DFS_DATANODE_DATA_DIR_KEY =
"dfs.datanode.data.dir";
public static final int DFS_NAMENODE_HTTPS_PORT_DEFAULT = 50470;
public static final String DFS_NAMENODE_HTTPS_ADDRESS_KEY =
"dfs.namenode.https-address";
+ public static final String DFS_NAMENODE_HTTPS_BIND_HOST_KEY =
"dfs.namenode.https-bind-host";
public static final String DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT = "0.0.0.0:"
+ DFS_NAMENODE_HTTPS_PORT_DEFAULT;
public static final String DFS_NAMENODE_NAME_DIR_KEY =
"dfs.namenode.name.dir";
public static final String DFS_NAMENODE_EDITS_DIR_KEY =
"dfs.namenode.edits.dir";
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1589803&r1=1589802&r2=1589803&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Thu Apr 24 17:40:30 2014
@@ -175,6 +175,8 @@ public class NameNode implements NameNod
DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY,
DFS_NAMENODE_HTTP_ADDRESS_KEY,
DFS_NAMENODE_HTTPS_ADDRESS_KEY,
+ DFS_NAMENODE_HTTP_BIND_HOST_KEY,
+ DFS_NAMENODE_HTTPS_BIND_HOST_KEY,
DFS_NAMENODE_KEYTAB_FILE_KEY,
DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,
DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_KEY,
@@ -444,6 +446,29 @@ public class NameNode implements NameNod
return getHttpAddress(conf);
}
+ /**
+ * HTTP server address for binding the endpoint. This method is
+ * for use by the NameNode and its derivatives. It may return
+ * a different address than the one that should be used by clients to
+ * connect to the NameNode. See
+ * {@link DFSConfigKeys#DFS_NAMENODE_HTTP_BIND_HOST_KEY}
+ *
+ * @param conf
+ * @return
+ */
+ protected InetSocketAddress getHttpServerBindAddress(Configuration conf) {
+ InetSocketAddress bindAddress = getHttpServerAddress(conf);
+
+ // If DFS_NAMENODE_HTTP_BIND_HOST_KEY exists then it overrides the
+ // host name portion of DFS_NAMENODE_HTTP_ADDRESS_KEY.
+ final String bindHost = conf.getTrimmed(DFS_NAMENODE_HTTP_BIND_HOST_KEY);
+ if (bindHost != null && !bindHost.isEmpty()) {
+ bindAddress = new InetSocketAddress(bindHost, bindAddress.getPort());
+ }
+
+ return bindAddress;
+ }
+
/** @return the NameNode HTTP address. */
public static InetSocketAddress getHttpAddress(Configuration conf) {
return NetUtils.createSocketAddr(
@@ -608,7 +633,7 @@ public class NameNode implements NameNod
}
private void startHttpServer(final Configuration conf) throws IOException {
- httpServer = new NameNodeHttpServer(conf, this,
getHttpServerAddress(conf));
+ httpServer = new NameNodeHttpServer(conf, this,
getHttpServerBindAddress(conf));
httpServer.start();
httpServer.setStartupProgress(startupProgress);
}
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java?rev=1589803&r1=1589802&r2=1589803&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
Thu Apr 24 17:40:30 2014
@@ -108,6 +108,16 @@ public class NameNodeHttpServer {
DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT);
InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString);
+ if (httpsAddr != null) {
+ // If DFS_NAMENODE_HTTPS_BIND_HOST_KEY exists then it overrides the
+ // host name portion of DFS_NAMENODE_HTTPS_ADDRESS_KEY.
+ final String bindHost =
+ conf.getTrimmed(DFSConfigKeys.DFS_NAMENODE_HTTPS_BIND_HOST_KEY);
+ if (bindHost != null && !bindHost.isEmpty()) {
+ httpsAddr = new InetSocketAddress(bindHost, httpsAddr.getPort());
+ }
+ }
+
HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
httpAddr, httpsAddr, "hdfs",
DFSConfigKeys.DFS_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY,
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1589803&r1=1589802&r2=1589803&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
Thu Apr 24 17:40:30 2014
@@ -370,6 +370,12 @@ class NameNodeRpcServer implements Namen
return clientRpcServer;
}
+ /** Allow access to the service RPC server for testing */
+ @VisibleForTesting
+ RPC.Server getServiceRpcServer() {
+ return serviceRpcServer;
+ }
+
/**
* Start client and service RPC servers.
*/
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1589803&r1=1589802&r2=1589803&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
Thu Apr 24 17:40:30 2014
@@ -55,11 +55,11 @@
<name>dfs.namenode.rpc-bind-host</name>
<value></value>
<description>
- The actual address the server will bind to. If this optional address is
- set, the RPC server will bind to this address and the port specified in
- dfs.namenode.rpc-address for the RPC server. It can also be specified
- per name node or name service for HA/Federation. This is most useful for
- making name node listen to all interfaces by setting to 0.0.0.0.
+ The actual address the RPC server will bind to. If this optional address is
+ set, it overrides only the hostname portion of dfs.namenode.rpc-address.
+ It can also be specified per name node or name service for HA/Federation.
+ This is useful for making the name node listen on all interfaces by
+ setting it to 0.0.0.0.
</description>
</property>
@@ -80,11 +80,11 @@
<name>dfs.namenode.servicerpc-bind-host</name>
<value></value>
<description>
- The actual address the server will bind to. If this optional address is
- set, the service RPC server will bind to this address and the port
- specified in dfs.namenode.servicerpc-address. It can also be specified
- per name node or name service for HA/Federation. This is most useful for
- making name node listen to all interfaces by setting to 0.0.0.0.
+ The actual address the service RPC server will bind to. If this optional
address is
+ set, it overrides only the hostname portion of
dfs.namenode.servicerpc-address.
+ It can also be specified per name node or name service for HA/Federation.
+ This is useful for making the name node listen on all interfaces by
+ setting it to 0.0.0.0.
</description>
</property>
@@ -143,6 +143,18 @@
</property>
<property>
+ <name>dfs.namenode.http-bind-host</name>
+ <value></value>
+ <description>
+ The actual adress the HTTP server will bind to. If this optional address
+ is set, it overrides only the hostname portion of
dfs.namenode.http-address.
+ It can also be specified per name node or name service for HA/Federation.
+ This is useful for making the name node HTTP server listen on all
+ interfaces by setting it to 0.0.0.0.
+ </description>
+</property>
+
+<property>
<name>dfs.https.enable</name>
<value>false</value>
<description>
@@ -207,6 +219,18 @@
<description>The namenode secure http server address and port.</description>
</property>
+<property>
+ <name>dfs.namenode.https-bind-host</name>
+ <value></value>
+ <description>
+ The actual adress the HTTPS server will bind to. If this optional address
+ is set, it overrides only the hostname portion of
dfs.namenode.https-address.
+ It can also be specified per name node or name service for HA/Federation.
+ This is useful for making the name node HTTPS server listen on all
+ interfaces by setting it to 0.0.0.0.
+ </description>
+</property>
+
<property>
<name>dfs.datanode.dns.interface</name>
<value>default</value>
