Repository: incubator-impala Updated Branches: refs/heads/master 68f32e52b -> 9e7fb830f
IMPALA-4088: Assign fix values to the minicluster server ports The minicluster setup logic assigned fixed port numbers to several but not all listening sockets of the data nodes. This change assigns similar port ranges to all the listening ports that were so far allowed to pick their own port numbers, interfering with other components, e.g. HBase. Change-Id: Iecf312873b7026c52b0ac0e71adbecab181925a0 Reviewed-on: http://gerrit.cloudera.org:8080/6531 Reviewed-by: Michael Brown <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/9e7fb830 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/9e7fb830 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/9e7fb830 Branch: refs/heads/master Commit: 9e7fb830fd047ed6445ea82280993e56ea51e15b Parents: 68f32e5 Author: Laszlo Gaal <[email protected]> Authored: Sun Apr 2 13:22:46 2017 +0200 Committer: Impala Public Jenkins <[email protected]> Committed: Fri Apr 7 22:57:16 2017 +0000 ---------------------------------------------------------------------- testdata/cluster/admin | 29 +++++++++++++++++--- .../cdh5/etc/hadoop/conf/yarn-site.xml.tmpl | 4 +-- .../cdh5/etc/kudu/tserver.conf.tmpl | 2 +- .../common/etc/hadoop/conf/hdfs-site.xml.tmpl | 6 ++-- .../common/etc/hadoop/conf/yarn-site.xml.tmpl | 4 +-- 5 files changed, 33 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9e7fb830/testdata/cluster/admin ---------------------------------------------------------------------- diff --git a/testdata/cluster/admin b/testdata/cluster/admin index e70cc3c..cb33e21 100755 --- a/testdata/cluster/admin +++ b/testdata/cluster/admin @@ -71,8 +71,14 @@ fi # considered a new node. The values below are arbitrary and may conflict with existing # services. Fixed ports were preferred to dynamically chosen free ports for consistency. DATANODE_FREE_PORT_START=31000 +DATANODE_FREE_HTTP_PORT_START=31010 +DATANODE_FREE_IPC_PORT_START=31020 +DATANODE_FREE_HTTPS_PORT_START=31030 NODEMANAGER_FREE_PORT_START=31100 +NODEMANAGER_FREE_LOCALIZER_PORT_START=31120 +NODEMANAGER_FREE_WEBUI_PORT_START=31140 KUDU_TS_RPC_FREE_PORT_START=31200 +KUDU_TS_WEBUI_FREE_PORT_START=31300 # Used to populate config templates later. Any changes made here have no effect on an # existing cluster. @@ -235,13 +241,28 @@ function create_cluster { # Template population DATANODE_PORT=$((DATANODE_FREE_PORT_START++)) + DATANODE_HTTP_PORT=$((DATANODE_FREE_HTTP_PORT_START++)) + DATANODE_IPC_PORT=$((DATANODE_FREE_IPC_PORT_START++)) + DATANODE_HTTPS_PORT=$((DATANODE_FREE_HTTPS_PORT_START++)) NODEMANAGER_PORT=$((NODEMANAGER_FREE_PORT_START++)) + NODEMANAGER_LOCALIZER_PORT=$((NODEMANAGER_FREE_LOCALIZER_PORT_START++)) + NODEMANAGER_WEBUI_PORT=$((NODEMANAGER_FREE_WEBUI_PORT_START++)) KUDU_TS_RPC_PORT=$((KUDU_TS_RPC_FREE_PORT_START++)) + KUDU_TS_WEBUI_PORT=$((KUDU_TS_WEBUI_FREE_PORT_START++)) echo "$NODE will use ports DATANODE_PORT=$DATANODE_PORT," \ - "NODEMANAGER_PORT=$NODEMANAGER_PORT, and KUDU_TS_RPC_PORT=$KUDU_TS_RPC_PORT" - - export NODE NODE_DIR DATANODE_PORT NODEMANAGER_PORT - export KUDU_TS_RPC_PORT + "DATANODE_HTTP_PORT=$DATANODE_HTTP_PORT," \ + "DATANODE_IPC_PORT=$DATANODE_IPC_PORT," \ + "DATANODE_HTTPS_PORT=$DATANODE_HTTPS_PORT," \ + "NODEMANAGER_PORT=$NODEMANAGER_PORT," \ + "NODEMANAGER_LOCALIZER_PORT=$NODEMANAGER_LOCALIZER_PORT," \ + "NODEMANAGER_WEBUI_PORT=$NODEMANAGER_WEBUI_PORT," \ + "KUDU_TS_RPC_PORT=$KUDU_TS_RPC_PORT," \ + "and KUDU_TS_WEBUI_PORT=$KUDU_TS_WEBUI_PORT" + + export NODE NODE_DIR + export DATANODE_PORT DATANODE_HTTP_PORT DATANODE_IPC_PORT DATANODE_HTTPS_PORT + export NODEMANAGER_PORT NODEMANAGER_LOCALIZER_PORT NODEMANAGER_WEBUI_PORT + export KUDU_TS_RPC_PORT KUDU_TS_WEBUI_PORT for TEMPLATE_PATH in $(find "$NODE_DIR" -name "*$TEMPLATE_SUFFIX"); do ACTUAL_PATH="${TEMPLATE_PATH%$TEMPLATE_SUFFIX}" http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9e7fb830/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/yarn-site.xml.tmpl ---------------------------------------------------------------------- diff --git a/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/yarn-site.xml.tmpl b/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/yarn-site.xml.tmpl index 3a01d75..0dd7bec 100644 --- a/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/yarn-site.xml.tmpl +++ b/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/yarn-site.xml.tmpl @@ -50,12 +50,12 @@ <property> <name>yarn.nodemanager.localizer.address</name> - <value>127.0.0.1:0</value> + <value>127.0.0.1:${NODEMANAGER_LOCALIZER_PORT}</value> </property> <property> <name>yarn.nodemanager.webapp.address</name> - <value>127.0.0.1:0</value> + <value>127.0.0.1:${NODEMANAGER_WEBUI_PORT}</value> </property> <property> http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9e7fb830/testdata/cluster/node_templates/cdh5/etc/kudu/tserver.conf.tmpl ---------------------------------------------------------------------- diff --git a/testdata/cluster/node_templates/cdh5/etc/kudu/tserver.conf.tmpl b/testdata/cluster/node_templates/cdh5/etc/kudu/tserver.conf.tmpl index 43e4765..c84bd91 100644 --- a/testdata/cluster/node_templates/cdh5/etc/kudu/tserver.conf.tmpl +++ b/testdata/cluster/node_templates/cdh5/etc/kudu/tserver.conf.tmpl @@ -1,5 +1,5 @@ -rpc_bind_addresses=0.0.0.0:${KUDU_TS_RPC_PORT} --webserver_port=0 +-webserver_port=${KUDU_TS_WEBUI_PORT} -fs_wal_dir=${NODE_DIR}/var/lib/kudu/ts/wal -fs_data_dirs=${NODE_DIR}/var/lib/kudu/ts/data -log_dir=${NODE_DIR}/var/log/kudu/ts http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9e7fb830/testdata/cluster/node_templates/common/etc/hadoop/conf/hdfs-site.xml.tmpl ---------------------------------------------------------------------- diff --git a/testdata/cluster/node_templates/common/etc/hadoop/conf/hdfs-site.xml.tmpl b/testdata/cluster/node_templates/common/etc/hadoop/conf/hdfs-site.xml.tmpl index edc89b3..f72dd97 100644 --- a/testdata/cluster/node_templates/common/etc/hadoop/conf/hdfs-site.xml.tmpl +++ b/testdata/cluster/node_templates/common/etc/hadoop/conf/hdfs-site.xml.tmpl @@ -29,17 +29,17 @@ <property> <name>dfs.datanode.http.address</name> - <value>127.0.0.1:0</value> + <value>127.0.0.1:${DATANODE_HTTP_PORT}</value> </property> <property> <name>dfs.datanode.ipc.address</name> - <value>127.0.0.1:0</value> + <value>127.0.0.1:${DATANODE_IPC_PORT}</value> </property> <property> <name>dfs.datanode.https.address</name> - <value>0.0.0.0:0</value> + <value>127.0.0.1:${DATANODE_HTTPS_PORT}</value> </property> <!-- Configuration to enable disk location metadata --> http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9e7fb830/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.tmpl ---------------------------------------------------------------------- diff --git a/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.tmpl b/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.tmpl index 2adb6fb..001d750 100644 --- a/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.tmpl +++ b/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.tmpl @@ -25,12 +25,12 @@ <property> <name>yarn.nodemanager.localizer.address</name> - <value>127.0.0.1:0</value> + <value>127.0.0.1:${NODEMANAGER_LOCALIZER_PORT}</value> </property> <property> <name>yarn.nodemanager.webapp.address</name> - <value>127.0.0.1:0</value> + <value>127.0.0.1:${NODEMANAGER_WEBUI_PORT}</value> </property> <property>
