This is an automated email from the ASF dual-hosted git repository.
jialiang pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5093e24f2c Initial setup for Router: Included all the changes required
for router installation and start. This PR also takes care of multiple router
configurations (#3804)
5093e24f2c is described below
commit 5093e24f2c7e1bbc15a3a8c369536343f9d6dd2d
Author: Ananya Singh <[email protected]>
AuthorDate: Fri Aug 9 10:55:16 2024 +0530
Initial setup for Router: Included all the changes required for router
installation and start. This PR also takes care of multiple router
configurations (#3804)
Co-authored-by: ananysin <[email protected]>
---
.../BIGTOP/3.2.0/properties/stack_packages.json | 12 +
.../services/HDFS/configuration/hdfs-rbf-site.xml | 771 +++++++++++++++++++++
.../BIGTOP/3.2.0/services/HDFS/kerberos.json | 26 +
.../stacks/BIGTOP/3.2.0/services/HDFS/metainfo.xml | 46 ++
.../services/HDFS/package/scripts/dfsrouter.py | 143 ++++
.../3.2.0/services/HDFS/package/scripts/hdfs.py | 9 +
.../services/HDFS/package/scripts/hdfs_router.py | 91 +++
.../services/HDFS/package/scripts/params_linux.py | 33 +
.../3.2.0/services/HDFS/package/scripts/utils.py | 16 +
9 files changed, 1147 insertions(+)
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json
index a050d4c5ef..3025e61075 100644
---
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json
@@ -83,6 +83,18 @@
"hadoop-hdfs-namenode"
]
},
+ "ROUTER": {
+ "STACK-SELECT-PACKAGE": "hadoop-hdfs-dfsrouter",
+ "INSTALL": [
+ "hadoop-hdfs-dfsrouter"
+ ],
+ "PATCH": [
+ "hadoop-hdfs-dfsrouter"
+ ],
+ "STANDARD": [
+ "hadoop-hdfs-dfsrouter"
+ ]
+ },
"JOURNALNODE": {
"STACK-SELECT-PACKAGE": "hadoop-hdfs-journalnode",
"INSTALL": [
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/configuration/hdfs-rbf-site.xml
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/configuration/hdfs-rbf-site.xml
new file mode 100644
index 0000000000..6088b51112
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/configuration/hdfs-rbf-site.xml
@@ -0,0 +1,771 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Put site-specific property overrides in this file. -->
+<configuration supports_final="true">
+ <property>
+ <name>dfs.federation.router.rpc.enable</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ If true, the RPC service to handle client requests in the router is
+ enabled.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.rpc-address</name>
+ <value>0.0.0.0:20010</value>
+ <description>
+ RPC address that handles all clients requests.
+ The value of this property will take the form of
router-host1:rpc-port.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.rpc-bind-host</name>
+ <value>0.0.0.0</value>
+ <description>
+ The actual address the RPC server will bind to. If this optional
address is
+ set, it overrides only the hostname portion of
+ dfs.federation.router.rpc-address. This is useful for making the
name node
+ listen on all interfaces by setting it to 0.0.0.0.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.handler.count</name>
+ <value>10</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ The number of server threads for the router to handle RPC requests
from
+ clients.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.handler.queue.size</name>
+ <value>100</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ The size of the queue for the number of handlers to handle RPC
client requests.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.reader.count</name>
+ <value>1</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ The number of readers for the router to handle RPC client requests.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.reader.queue.size</name>
+ <value>100</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ The size of the queue for the number of readers for the router to
handle RPC client requests.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.connection.creator.queue-size</name>
+ <value>100</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ Size of async connection creator queue.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.connection.pool-size</name>
+ <value>1</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ Size of the pool of connections from the router to namenodes.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.connection.min-active-ratio</name>
+ <value>0.5f</value>
+ <description>
+ Minimum active ratio of connections from the router to namenodes.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.connection.clean.ms</name>
+ <value>10000</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ Time interval, in milliseconds, to check if the connection pool
should
+ remove unused connections.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.connection.pool.clean.ms</name>
+ <value>60000</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ Time interval, in milliseconds, to check if the connection manager
should
+ remove unused connection pools.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.metrics.enable</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ If the metrics in the router are enabled.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+
+ <property>
+ <name>dfs.federation.router.dn-report.time-out</name>
+ <value>1000</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ Time out, in milliseconds for getDatanodeReport.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.dn-report.cache-expire</name>
+ <value>10s</value>
+ <description>
+ Expiration time in seconds for datanodereport.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.metrics.class</name>
+
<value>org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCPerformanceMonitor</value>
+ <description>
+ Class to monitor the RPC system in the router. It must implement
the
+ RouterRpcMonitor interface.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.admin.enable</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ If true, the RPC admin service to handle client requests in the
router is
+ enabled.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.admin-address</name>
+ <value>0.0.0.0:8111</value>
+ <description>
+ RPC address that handles the admin requests.
+ The value of this property will take the form of
router-host1:rpc-port.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.admin-bind-host</name>
+ <value>0.0.0.0</value>
+ <description>
+ The actual address the RPC admin server will bind to. If this
optional
+ address is set, it overrides only the hostname portion of
+ dfs.federation.router.admin-address. This is useful for making the
name
+ node listen on all interfaces by setting it to 0.0.0.0.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.admin.handler.count</name>
+ <value>1</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ The number of server threads for the router to handle RPC requests
from
+ admin.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.http-address</name>
+ <value>0.0.0.0:50071</value>
+ <description>
+ HTTP address that handles the web requests to the Router.
+ The value of this property will take the form of
router-host1:http-port.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.http-bind-host</name>
+ <value>0.0.0.0</value>
+ <description>
+ The actual address the HTTP server will bind to. If this optional
+ address is set, it overrides only the hostname portion of
+ dfs.federation.router.http-address. This is useful for making the
name
+ node listen on all interfaces by setting it to 0.0.0.0.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.https-address</name>
+ <value>0.0.0.0:50072</value>
+ <description>
+ HTTPS address that handles the web requests to the Router.
+ The value of this property will take the form of
router-host1:https-port.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.https-bind-host</name>
+ <value>0.0.0.0</value>
+ <description>
+ The actual address the HTTPS server will bind to. If this optional
+ address is set, it overrides only the hostname portion of
+ dfs.federation.router.https-address. This is useful for making the
name
+ node listen on all interfaces by setting it to 0.0.0.0.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.http.enable</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ If the HTTP service to handle client requests in the router is
enabled.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.file.resolver.client.class</name>
+
<value>org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver</value>
+ <description>
+ Class to resolve files to subclusters. To enable multiple
subclusters for a mount point,
+ set to
org.apache.hadoop.hdfs.server.federation.resolver.MultipleDestinationMountTableResolver.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.namenode.resolver.client.class</name>
+
<value>org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver</value>
+ <description>
+ Class to resolve the namenode for a subcluster.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.enable</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ If true, the Router connects to the State Store.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.serializer</name>
+
<value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl</value>
+ <description>
+ Class to serialize State Store records.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.driver.class</name>
+
<value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl</value>
+ <description>
+ Class to implement the State Store. There are three implementation
classes currently
+ being supported:
+
org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl,
+
org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl
and
+
org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl.
+ These implementation classes use the local file, filesystem and
ZooKeeper as a backend respectively.
+ By default it uses the ZooKeeper as the default State Store.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.connection.test</name>
+ <value>60000</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ How often to check for the connection to the State Store in
milliseconds.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.cache.ttl</name>
+ <value>1m</value>
+ <description>
+ How often to refresh the State Store caches in milliseconds. This
setting
+ supports multiple time unit suffixes as described in
+ dfs.heartbeat.interval. If no suffix is specified then
milliseconds is
+ assumed.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.membership.expiration</name>
+ <value>300000</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ Expiration time in milliseconds for a membership record.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.membership.expiration.deletion</name>
+ <value>-1</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ Deletion time in milliseconds for a membership record. If an
expired
+ membership record exists beyond this time, it will be deleted. If
this
+ value is negative, the deletion is disabled.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.heartbeat.enable</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ If true, the Router heartbeats into the State Store.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.heartbeat.interval</name>
+ <value>5000</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ How often the Router should heartbeat into the State Store in
milliseconds.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.heartbeat-state.interval</name>
+ <value>5s</value>
+ <description>
+ How often the Router should heartbeat its state into the State
Store in
+ milliseconds. This setting supports multiple time unit suffixes as
+ described in dfs.federation.router.quota-cache.update.interval.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.namenode.heartbeat.enable</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ If true, get namenode heartbeats and send into the State Store.
+ If not explicitly specified takes the same value as for
+ dfs.federation.router.heartbeat.enable.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.router.expiration</name>
+ <value>5m</value>
+ <description>
+ Expiration time in milliseconds for a router state record. This
setting
+ supports multiple time unit suffixes as described in
+ dfs.federation.router.quota-cache.update.interval.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.router.expiration.deletion</name>
+ <value>-1</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ Deletion time in milliseconds for a router state record. If an
expired
+ router state record exists beyond this time, it will be deleted.
If this
+ value is negative, the deletion is disabled.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.safemode.enable</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.safemode.extension</name>
+ <value>30s</value>
+ <description>
+ Time after startup that the Router is in safe mode. This setting
+ supports multiple time unit suffixes as described in
+ dfs.heartbeat.interval. If no suffix is specified then
milliseconds is
+ assumed.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.safemode.expiration</name>
+ <value>3m</value>
+ <description>
+ Time without being able to reach the State Store to enter safe
mode. This
+ setting supports multiple time unit suffixes as described in
+ dfs.heartbeat.interval. If no suffix is specified then
milliseconds is
+ assumed.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.monitor.localnamenode.enable</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ If true, the Router should monitor the namenode in the local
machine.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.mount-table.max-cache-size</name>
+ <value>10000</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ Maximum number of mount table cache entries to have.
+ By default, remove cache entries if we have more than 10k.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.mount-table.cache.enable</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ Set to true to enable mount table cache (Path to Remote Location
cache).
+ Disabling the cache is recommended when a large amount of unique
paths are queried.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.quota.enable</name>
+ <value>false</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ Set to true to enable quota system in Router. When it's enabled,
setting
+ or clearing sub-cluster's quota directly is not recommended since
Router
+ Admin server will override sub-cluster's quota with global quota.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.quota-cache.update.interval</name>
+ <value>60s</value>
+ <description>
+ Interval time for updating quota usage cache in Router.
+ This property is used only if the value of
+ dfs.federation.router.quota.enable is true.
+ This setting supports multiple time unit suffixes as described
+ in dfs.heartbeat.interval. If no suffix is specified then
milliseconds
+ is assumed.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.client.thread-size</name>
+ <value>32</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ Max threads size for the RouterClient to execute concurrent
+ requests.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.client.retry.max.attempts</name>
+ <value>3</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ Max retry attempts for the RouterClient talking to the Router.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.client.reject.overload</name>
+ <value>false</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ Set to true to reject client requests when we run out of RPC client
+ threads.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.client.allow-partial-listing</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ If the Router can return a partial list of files in a
multi-destination mount point when one of the subclusters is unavailable.
+ True may return a partial list of files if a subcluster is down.
+ False will fail the request if one is unavailable.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.client.mount-status.time-out</name>
+ <value>1s</value>
+ <description>
+ Set a timeout for the Router when listing folders containing mount
+ points. In this process, the Router checks the mount table and
then it
+ checks permissions in the subcluster. After the time out, we
return the
+ default values.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.connect.max.retries.on.timeouts</name>
+ <value>0</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ Maximum number of retries for the IPC Client when connecting to the
+ subclusters. By default, it doesn't let the IPC retry and the
Router
+ handles it.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.connect.timeout</name>
+ <value>2s</value>
+ <description>
+ Time out for the IPC client connecting to the subclusters. This
should be
+ short as the Router has knowledge of the state of the Routers.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <!-- <property>
+ <name>dfs.federation.router.keytab.file</name>
+ <value></value>
+ <description>
+ The keytab file used by router to login as its
+ service principal. The principal name is configured with
+ dfs.federation.router.kerberos.principal.
+ </description>
+ </property>
+ <property>
+ <name>dfs.federation.router.kerberos.principal</name>
+ <value></value>
+ <description>
+ The Router service principal. This is typically set to
+ router/[email protected]. Each Router will substitute _HOST with its
+ own fully qualified hostname at startup. The _HOST placeholder
+ allows using the same configuration setting on both Router
+ in an HA setup.
+ </description>
+ </property>
+ <property>
+ <name>dfs.federation.router.kerberos.principal.hostname</name>
+ <value></value>
+ <description>
+ Optional. The hostname for the Router containing this
+ configuration file. Will be different for each machine.
+ Defaults to current hostname.
+ </description>
+ </property>
+ <property>
+ <name>dfs.federation.router.kerberos.internal.spnego.principal</name>
+ <value>${dfs.web.authentication.kerberos.principal}</value>
+ <description>
+ The server principal used by the Router for web UI SPNEGO
+ authentication when Kerberos security is enabled. This is
+ typically set to HTTP/[email protected] The SPNEGO server principal
+ begins with the prefix HTTP/ by convention.
+ If the value is '*', the web server will attempt to login with
+ every principal specified in the keytab file
+ dfs.web.authentication.kerberos.keytab.
+ </description>
+ </property>
+ -->
+ <property>
+ <name>dfs.federation.router.mount-table.cache.update</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>Set true to enable MountTableRefreshService. This service
+ updates mount table cache immediately after adding, modifying or
+ deleting the mount table entries. If this service is not enabled
+ mount table cache are refreshed periodically by
+ StateStoreCacheUpdateService
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.mount-table.cache.update.timeout</name>
+ <value>1m</value>
+ <description>This property defines how long to wait for all the
+ admin servers to finish their mount table cache update. This
setting
+ supports multiple time unit suffixes as described in
+ dfs.federation.router.safemode.extension.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+
<name>dfs.federation.router.mount-table.cache.update.client.max.time</name>
+ <value>5m</value>
+ <description>Remote router mount table cache is updated through
+ RouterClient(RPC call). To improve performance, RouterClient
+ connections are cached but it should not be kept in cache forever.
+ This property defines the max time a connection can be cached. This
+ setting supports multiple time unit suffixes as described in
+ dfs.federation.router.safemode.extension.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.secret.manager.class</name>
+
<value>org.apache.hadoop.hdfs.server.federation.router.security.token.ZKDelegationTokenSecretManagerImpl</value>
+ <description>
+ Class to implement state store to delegation tokens.
+ Default implementation uses zookeeper as the backend to store
delegation tokens.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/kerberos.json
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/kerberos.json
index 2ba6c63664..98b059eb35 100644
---
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/kerberos.json
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/kerberos.json
@@ -146,6 +146,32 @@
}
]
},
+ {
+ "name": "ROUTER",
+ "identities": [
+ {
+ "name": "dfsrouter_dr",
+ "principal": {
+ "value": "router/_HOST@${realm}",
+ "type" : "service",
+ "configuration":
"hdfs-rbf-site/dfs.federation.router.kerberos.principal",
+ "local_username" : "${hadoop-env/hdfs_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/dr.service.keytab",
+ "owner": {
+ "name": "${hadoop-env/hdfs_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration":
"hdfs-rbf-site/dfs.federation.router.keytab.file"
+ }
+ }
+ ]
+ },
{
"name": "SECONDARY_NAMENODE",
"identities": [
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/metainfo.xml
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/metainfo.xml
index a62758ea8a..6c1dbb5639 100644
---
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/metainfo.xml
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/metainfo.xml
@@ -223,6 +223,52 @@
</configFiles>
</component>
+ <component>
+ <name>ROUTER</name>
+ <displayName>Router</displayName>
+ <category>SLAVE</category>
+ <cardinality>0+</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <reassignAllowed>true</reassignAllowed>
+ <timelineAppid>router</timelineAppid>
+ <commandScript>
+ <script>scripts/router.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1800</timeout>
+ </commandScript>
+ <logs>
+ <log>
+ <logId>hdfs_router</logId>
+ <primary>true</primary>
+ </log>
+ <log>
+ <logId>hdfs_audit</logId>
+ </log>
+ </logs>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <configuration-dependencies>
+ <config-type>hdfs-rbf-site</config-type>
+ </configuration-dependencies>
+ <customCommands>
+ <customCommand>
+ <name>DECOMMISSION</name>
+ <commandScript>
+ <script>scripts/router.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ </customCommands>
+ </component>
+
<component>
<name>JOURNALNODE</name>
<displayName>JournalNode</displayName>
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/dfsrouter.py
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/dfsrouter.py
new file mode 100644
index 0000000000..15d04936c9
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/dfsrouter.py
@@ -0,0 +1,143 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import sys
+import os
+import json
+import tempfile
+import hashlib
+from datetime import datetime
+import ambari_simplejson as json # simplejson is much faster comparing to
Python 2.6 json module and has the same functions set.
+
+from ambari_commons import constants
+
+from resource_management.libraries.resources.xml_config import XmlConfig
+
+from resource_management.libraries.script.script import Script
+from resource_management.core.resources.system import Execute, File
+from resource_management.core import shell
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import upgrade_summary
+from resource_management.libraries.functions.constants import Direction
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.resources.execute_hadoop import
ExecuteHadoop
+from resource_management.libraries.functions.security_commons import
build_expectations, \
+ cached_kinit_executor, get_params_from_filesystem,
validate_security_config_properties, \
+ FILE_TYPE_XML
+
+from resource_management.core.exceptions import Fail
+from resource_management.core.shell import as_user
+from resource_management.core.logger import Logger
+
+
+from ambari_commons.os_family_impl import OsFamilyImpl
+from ambari_commons import OSConst
+
+from hdfs_router import router
+
+
+from hdfs import hdfs, reconfig
+import hdfs_rebalance
+from utils import initiate_safe_zkfc_failover, get_hdfs_binary,
get_dfsrouteradmin_base_command
+from resource_management.libraries.functions.namenode_ha_utils import
get_hdfs_cluster_id_from_jmx
+
+# The hash algorithm to use to generate digests/hashes
+HASH_ALGORITHM = hashlib.sha224
+
+class Router(Script):
+
+ def get_hdfs_binary(self):
+ """
+ Get the name or path to the hdfs binary depending on the component name.
+ """
+ return get_hdfs_binary("hadoop-hdfs-dfsrouter")
+
+ def install(self, env):
+ import params
+ env.set_params(params)
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ hdfs("router")
+ hdfs_binary = self.get_hdfs_binary()
+ router(action="configure", hdfs_binary=hdfs_binary, env=env)
+ XmlConfig("hdfs-site.xml",
+ conf_dir=params.hadoop_conf_dir,
+ configurations=params.router_hdfs_site,
+
configuration_attributes=params.config['configurationAttributes']['hdfs-site'],
+ mode=0o644,
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+ XmlConfig("core-site.xml",
+ conf_dir=params.hadoop_conf_dir,
+ configurations=params.router_core_site,
+
configuration_attributes=params.config['configurationAttributes']['core-site'],
+ mode=0o644,
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+
+ def save_configs(self, env):
+ import params
+ env.set_params(params)
+ hdfs()
+
+ def reload_configs(self, env):
+ import params
+ env.set_params(params)
+ Logger.info("RELOAD CONFIGS")
+ reconfig("router", params.router_address)
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ hdfs_binary = self.get_hdfs_binary()
+ router(action="start", hdfs_binary=hdfs_binary, env=env)
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ hdfs_binary = self.get_hdfs_binary()
+ router(action="stop", hdfs_binary=hdfs_binary, env=env)
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ router(action="status", env=env)
+
+ def get_log_folder(self):
+ import params
+ return params.hdfs_log_dir
+
+ def get_user(self):
+ import params
+ return params.hdfs_user
+
+ def get_pid_files(self):
+ import status_params
+ return [status_params.router_pid_file]
+
+def _print(line):
+ sys.stdout.write(line)
+ sys.stdout.flush()
+
+if __name__ == "__main__":
+ Router().execute()
\ No newline at end of file
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs.py
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs.py
index 31664c9744..93bd8f0c9e 100644
---
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs.py
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs.py
@@ -130,6 +130,15 @@ def hdfs(name=None):
group=params.user_group
)
+ XmlConfig("hdfs-rbf-site.xml",
+ conf_dir=params.hadoop_conf_dir,
+ configurations=params.config['configurations']['hdfs-rbf-site'],
+
configuration_attributes=params.config['configurationAttributes']['hdfs-rbf-site'],
+ mode=0o644,
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+
XmlConfig("core-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['core-site'],
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs_router.py
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs_router.py
new file mode 100644
index 0000000000..a4064925a2
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/hdfs_router.py
@@ -0,0 +1,91 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import os.path
+import time
+
+from ambari_commons import constants
+
+from resource_management.core import shell
+from resource_management.core.source import Template
+from resource_management.core.resources.system import File, Execute, Directory
+from resource_management.core.resources.service import Service
+from resource_management.libraries.functions.decorator import retry
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.check_process_status import
check_process_status
+from resource_management.libraries.resources.execute_hadoop import
ExecuteHadoop
+from resource_management.libraries.functions import Direction, upgrade_summary
+from resource_management.libraries.functions.namenode_ha_utils import
get_name_service_by_hostname
+from resource_management.libraries.functions.generate_logfeeder_input_config
import generate_logfeeder_input_config
+from ambari_commons import OSCheck, OSConst
+from ambari_commons.os_family_impl import OsFamilyImpl, OsFamilyFuncImpl
+from utils import get_dfsrouteradmin_base_command
+from utils import set_up_zkfc_security
+
+if OSCheck.is_windows_family():
+ from resource_management.libraries.functions.windows_service_utils import
check_windows_service_status
+
+from resource_management.core.exceptions import Fail
+from resource_management.core.logger import Logger
+
+from utils import service, safe_zkfc_op, is_previous_fs_image
+from setup_ranger_hdfs import setup_ranger_hdfs,
create_ranger_audit_hdfs_directories
+
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def router(action=None, hdfs_binary=None, env=None):
+
+ if action is None:
+ raise Fail('"action" parameter is required for function router().')
+
+ if action in ["start", "stop"] and hdfs_binary is None:
+ raise Fail('"hdfs_binary" parameter is required for function router().')
+
+ if action == "configure":
+ import params
+ generate_logfeeder_input_config('hdfs',
Template("input.config-hdfs.json.j2", extra_imports=[default]))
+ # set up failover / secure zookeper ACLs, this feature is supported from
VDP 2.6 ownwards
+ set_up_zkfc_security(params)
+ elif action == "start":
+ import params
+ service(
+ action="start",
+ name="dfsrouter",
+ user=params.hdfs_user,
+ create_pid_dir=True,
+ create_log_dir=True
+ )
+
+ if params.security_enabled:
+ Execute(format("{kinit_path_local} -kt {hdfs_user_keytab}
{hdfs_principal_name}"),
+ user = params.hdfs_user)
+
+ name_service = get_name_service_by_hostname(params.hdfs_site,
params.hostname)
+ ensure_safemode_off = True
+
+ elif action == "stop":
+ import params
+ service(
+ action="stop", name="dfsrouter",
+ user=params.hdfs_user
+ )
+ elif action == "status":
+ import status_params
+ check_process_status(status_params.router_pid_file)
+
+
+
+
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/params_linux.py
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/params_linux.py
index cdd5365705..d6c99647e2 100644
---
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/params_linux.py
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/params_linux.py
@@ -195,6 +195,7 @@ hbase_master_hosts =
default("/clusterHostInfo/hbase_master_hosts", [])
hs_host = default("/clusterHostInfo/historyserver_hosts", [])
jtnode_host = default("/clusterHostInfo/jtnode_hosts", [])
namenode_host = default("/clusterHostInfo/namenode_hosts", [])
+router_host = default("/clusterHostInfo/router_hosts", [])
nm_host = default("/clusterHostInfo/nodemanager_hosts", [])
ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_hosts", [])
journalnode_hosts = default("/clusterHostInfo/journalnode_hosts", [])
@@ -295,6 +296,38 @@ dfs_data_dirs_perm = int(dfs_data_dirs_perm, base=8) #
convert int from octal re
data_dir_mount_file =
"/var/lib/ambari-agent/data/datanode/dfs_data_dir_mount.hist"
+router_address = None
+if 'dfs.federation.router.rpc-address' in
config['configurations']['hdfs-rbf-site']:
+ router_rpcaddress =
config['configurations']['hdfs-rbf-site']['dfs.federation.router.rpc-address']
+ router_address = format("hdfs://{router_rpcaddress}")
+else:
+ router_address = config['configurations']['core-site']['fs.defaultFS']
+if router_host:
+ router_hdfs_site = dict(config['configurations']['hdfs-site'])
+ router_core_site = dict(config['configurations']['core-site'])
+ nameservices = config['configurations']['hdfs-site'].get('dfs.nameservices')
+ if not isinstance(nameservices, str):
+ # handle the error, for example by raising an exception or setting a
default value
+ print("The dfs.nameservices property is not set or not a string")
+ nameservices = ''
+
+ router_hdfs_site['dfs.nameservices'] = nameservices + ",ns-fed"
+ router_hdfs_site['dfs.client.failover.proxy.provider.ns-fed'] =
'org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider'
+ router_hdfs_site['dfs.client.failover.random.order'] = 'true'
+ router_id_list = ["r" + str(i) for i in range(1, len(router_host) + 1)]
+ router_ids = ",".join(router_id_list)
+ router_hdfs_site['dfs.ha.namenodes.ns-fed'] = router_ids
+ for i, curr_router_host in enumerate(router_host):
+ id = router_id_list[i]
+ prop_name = "dfs.namenode.rpc-address.ns-fed." + id
+ prop_value = curr_router_host + ":" + "20010"
+ router_hdfs_site[prop_name] = prop_value
+
+ router_core_site['fs.defaultFS'] = "hdfs://ns-fed"
+ router_core_site['hadoop.zk.address'] =
config['configurations']['core-site'].get('ha.zookeeper.quorum')
+else:
+ print("No router hosts found")
+
# HDFS High Availability properties
dfs_ha_enabled = False
dfs_ha_nameservices =
default('/configurations/hdfs-site/dfs.internal.nameservices', None)
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/utils.py
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/utils.py
index 5ce8598bf5..9bebc21663 100644
---
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/utils.py
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HDFS/package/scripts/utils.py
@@ -403,6 +403,22 @@ def get_dfsadmin_base_command(hdfs_binary,
use_specific_namenode = False):
dfsadmin_base_command = format("{hdfs_binary} dfsadmin -fs
{params.namenode_address}")
return dfsadmin_base_command
+def get_dfsrouteradmin_base_command(hdfs_binary, use_specific_router = False):
+ """
+ Get the dfsrouteradmin base command constructed using hdfs_binary path and
passing router address as explicit -fs argument
+ :param hdfs_binary: path to hdfs binary to use
+ :param use_specific_router: flag if set and Router HA is enabled, then the
dfsrouteradmin command will use
+ current router's address
+ :return: the constructed dfsrouteradmin base command
+ """
+ import params
+ dfsrouteradmin_base_command = ""
+ if params.dfs_ha_enabled and use_specific_router:
+ dfsrouteradmin_base_command = format("{hdfs_binary} dfsrouteradmin -fs
hdfs://{params.router_rpc}")
+ else:
+ dfsadmin_base_command = format("{hdfs_binary} dfsrouteradmin -fs
{params.router_address}")
+ return dfsrouteradmin_base_command
+
def set_up_zkfc_security(params):
""" Sets up security for accessing zookeper on secure clusters """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
