AMBARI-19758 : Post Ambari upgrade AMS config properties changes are marking HDFS/YARN/Hive/HBase with restart required. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/05ce603a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/05ce603a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/05ce603a Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 05ce603a9fa5a6a72af4de70cbc21208f6749384 Parents: 7abf4e6 Author: Aravindan Vijayan <[email protected]> Authored: Thu Feb 9 13:45:57 2017 -0800 Committer: Aravindan Vijayan <[email protected]> Committed: Thu Feb 9 13:45:57 2017 -0800 ---------------------------------------------------------------------- .../hadoop-metrics2.properties.xml | 125 ------------------- .../common-services/HDFS/2.1.0.2.0/metainfo.xml | 1 - .../2.0.6/hooks/before-START/scripts/params.py | 4 +- .../scripts/shared_initialization.py | 17 ++- .../hadoop-metrics2.properties.xml | 125 +++++++++++++++++++ .../stacks/HDP/2.6/services/HDFS/metainfo.xml | 3 + .../PERF/1.0/services/FAKEHDFS/metainfo.xml | 1 - 7 files changed, 143 insertions(+), 133 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/05ce603a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-metrics2.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-metrics2.properties.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-metrics2.properties.xml deleted file mode 100644 index 6b45e84..0000000 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-metrics2.properties.xml +++ /dev/null @@ -1,125 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> - -<configuration> - <!-- hadoop-metrics2.properties --> - <property> - <name>content</name> - <display-name>hadoop-metrics2.properties template</display-name> - <description>This is the jinja template for hadoop-metrics2.properties file</description> - <value> -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# syntax: [prefix].[source|sink|jmx].[instance].[options] -# See package.html for org.apache.hadoop.metrics2 for details - -{% if has_ganglia_server %} -*.period=60 - -*.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31 -*.sink.ganglia.period=10 - -# default for supportsparse is false -*.sink.ganglia.supportsparse=true - -.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both -.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40 - -# Hook up to the server -namenode.sink.ganglia.servers={{ganglia_server_host}}:8661 -datanode.sink.ganglia.servers={{ganglia_server_host}}:8659 -jobtracker.sink.ganglia.servers={{ganglia_server_host}}:8662 -tasktracker.sink.ganglia.servers={{ganglia_server_host}}:8658 -maptask.sink.ganglia.servers={{ganglia_server_host}}:8660 -reducetask.sink.ganglia.servers={{ganglia_server_host}}:8660 -resourcemanager.sink.ganglia.servers={{ganglia_server_host}}:8664 -nodemanager.sink.ganglia.servers={{ganglia_server_host}}:8657 -historyserver.sink.ganglia.servers={{ganglia_server_host}}:8666 -journalnode.sink.ganglia.servers={{ganglia_server_host}}:8654 -nimbus.sink.ganglia.servers={{ganglia_server_host}}:8649 -supervisor.sink.ganglia.servers={{ganglia_server_host}}:8650 - -resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue - -{% endif %} - -{% if has_metric_collector %} - -*.period={{metrics_collection_period}} -*.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar -*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink -*.sink.timeline.period={{metrics_collection_period}} -*.sink.timeline.sendInterval={{metrics_report_interval}}000 -*.sink.timeline.slave.host.name={{hostname}} -*.sink.timeline.zookeeper.quorum={{zookeeper_quorum}} -*.sink.timeline.protocol={{metric_collector_protocol}} -*.sink.timeline.port={{metric_collector_port}} - -# HTTPS properties -*.sink.timeline.truststore.path = {{metric_truststore_path}} -*.sink.timeline.truststore.type = {{metric_truststore_type}} -*.sink.timeline.truststore.password = {{metric_truststore_password}} - -datanode.sink.timeline.collector.hosts={{ams_collector_hosts}} -namenode.sink.timeline.collector.hosts={{ams_collector_hosts}} -resourcemanager.sink.timeline.collector.hosts={{ams_collector_hosts}} -nodemanager.sink.timeline.collector.hosts={{ams_collector_hosts}} -jobhistoryserver.sink.timeline.collector.hosts={{ams_collector_hosts}} -journalnode.sink.timeline.collector.hosts={{ams_collector_hosts}} -maptask.sink.timeline.collector.hosts={{ams_collector_hosts}} -reducetask.sink.timeline.collector.hosts={{ams_collector_hosts}} -applicationhistoryserver.sink.timeline.collector.hosts={{ams_collector_hosts}} - -resourcemanager.sink.timeline.tagsForPrefix.yarn=Queue - -{% if is_nn_client_port_configured %} -# Namenode rpc ports customization -namenode.sink.timeline.metric.rpc.client.port={{nn_rpc_client_port}} -{% endif %} -{% if is_nn_dn_port_configured %} -namenode.sink.timeline.metric.rpc.datanode.port={{nn_rpc_dn_port}} -{% endif %} -{% if is_nn_healthcheck_port_configured %} -namenode.sink.timeline.metric.rpc.healthcheck.port={{nn_rpc_healthcheck_port}} -{% endif %} - -{% endif %} - </value> - <value-attributes> - <type>content</type> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/05ce603a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml index 30de1be..da7daad 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml @@ -365,7 +365,6 @@ <config-type>ranger-hdfs-policymgr-ssl</config-type> <config-type>ranger-hdfs-security</config-type> <config-type>ams-ssl-client</config-type> - <config-type>hadoop-metrics2.properties</config-type> </configuration-dependencies> <restartRequiredAfterRackChange>true</restartRequiredAfterRackChange> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/05ce603a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py index c77a906..29c88b4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py @@ -46,7 +46,9 @@ component_list = default("/localComponents", []) hdfs_tmp_dir = default("/configurations/hadoop-env/hdfs_tmp_dir", "/tmp") -hadoop_metrics2_properties_content = config['configurations']['hadoop-metrics2.properties']['content'] +hadoop_metrics2_properties_content = None +if 'hadoop-metrics2.properties' in config['configurations']: + hadoop_metrics2_properties_content = config['configurations']['hadoop-metrics2.properties']['content'] # hadoop default params mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*" http://git-wip-us.apache.org/repos/asf/ambari/blob/05ce603a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py index cf958f0..9783aa4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py @@ -98,11 +98,18 @@ def setup_hadoop(): owner=params.hdfs_user, ) - File(os.path.join(params.hadoop_conf_dir, "hadoop-metrics2.properties"), - owner=params.hdfs_user, - group=params.user_group, - content=InlineTemplate(params.hadoop_metrics2_properties_content) - ) + if params.hadoop_metrics2_properties_content: + File(os.path.join(params.hadoop_conf_dir, "hadoop-metrics2.properties"), + owner=params.hdfs_user, + group=params.user_group, + content=InlineTemplate(params.hadoop_metrics2_properties_content) + ) + else: + File(os.path.join(params.hadoop_conf_dir, "hadoop-metrics2.properties"), + owner=params.hdfs_user, + group=params.user_group, + content=Template("hadoop-metrics2.properties.j2") + ) if params.dfs_type == 'HCFS' and params.has_core_site and 'ECS_CLIENT' in params.component_list: create_dirs() http://git-wip-us.apache.org/repos/asf/ambari/blob/05ce603a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml new file mode 100644 index 0000000..4aadb83 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml @@ -0,0 +1,125 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <!-- hadoop-metrics2.properties --> + <property> + <name>content</name> + <display-name>hadoop-metrics2.properties template</display-name> + <description>This is the jinja template for hadoop-metrics2.properties file</description> + <value> +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# syntax: [prefix].[source|sink|jmx].[instance].[options] +# See package.html for org.apache.hadoop.metrics2 for details + +{% if has_ganglia_server %} +*.period=60 + +*.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31 +*.sink.ganglia.period=10 + +# default for supportsparse is false +*.sink.ganglia.supportsparse=true + +.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both +.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40 + +# Hook up to the server +namenode.sink.ganglia.servers={{ganglia_server_host}}:8661 +datanode.sink.ganglia.servers={{ganglia_server_host}}:8659 +jobtracker.sink.ganglia.servers={{ganglia_server_host}}:8662 +tasktracker.sink.ganglia.servers={{ganglia_server_host}}:8658 +maptask.sink.ganglia.servers={{ganglia_server_host}}:8660 +reducetask.sink.ganglia.servers={{ganglia_server_host}}:8660 +resourcemanager.sink.ganglia.servers={{ganglia_server_host}}:8664 +nodemanager.sink.ganglia.servers={{ganglia_server_host}}:8657 +historyserver.sink.ganglia.servers={{ganglia_server_host}}:8666 +journalnode.sink.ganglia.servers={{ganglia_server_host}}:8654 +nimbus.sink.ganglia.servers={{ganglia_server_host}}:8649 +supervisor.sink.ganglia.servers={{ganglia_server_host}}:8650 + +resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue + +{% endif %} + +{% if has_metric_collector %} + +*.period={{metrics_collection_period}} +*.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar +*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +*.sink.timeline.period={{metrics_collection_period}} +*.sink.timeline.sendInterval={{metrics_report_interval}}000 +*.sink.timeline.slave.host.name={{hostname}} +*.sink.timeline.zookeeper.quorum={{zookeeper_quorum}} +*.sink.timeline.protocol={{metric_collector_protocol}} +*.sink.timeline.port={{metric_collector_port}} + +# HTTPS properties +*.sink.timeline.truststore.path = {{metric_truststore_path}} +*.sink.timeline.truststore.type = {{metric_truststore_type}} +*.sink.timeline.truststore.password = {{metric_truststore_password}} + +datanode.sink.timeline.collector.hosts={{ams_collector_hosts}} +namenode.sink.timeline.collector.hosts={{ams_collector_hosts}} +resourcemanager.sink.timeline.collector.hosts={{ams_collector_hosts}} +nodemanager.sink.timeline.collector.hosts={{ams_collector_hosts}} +jobhistoryserver.sink.timeline.collector.hosts={{ams_collector_hosts}} +journalnode.sink.timeline.collector.hosts={{ams_collector_hosts}} +maptask.sink.timeline.collector.hosts={{ams_collector_hosts}} +reducetask.sink.timeline.collector.hosts={{ams_collector_hosts}} +applicationhistoryserver.sink.timeline.collector.hosts={{ams_collector_hosts}} + +resourcemanager.sink.timeline.tagsForPrefix.yarn=Queue + +{% if is_nn_client_port_configured %} +# Namenode rpc ports customization +namenode.sink.timeline.metric.rpc.client.port={{nn_rpc_client_port}} +{% endif %} +{% if is_nn_dn_port_configured %} +namenode.sink.timeline.metric.rpc.datanode.port={{nn_rpc_dn_port}} +{% endif %} +{% if is_nn_healthcheck_port_configured %} +namenode.sink.timeline.metric.rpc.healthcheck.port={{nn_rpc_healthcheck_port}} +{% endif %} + +{% endif %} + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/05ce603a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/metainfo.xml index 1fc7f51..c80b2b4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/metainfo.xml @@ -21,6 +21,9 @@ <service> <name>HDFS</name> <version>2.7.3.2.6</version> + <configuration-dependencies> + <config-type>hadoop-metrics2.properties</config-type> + </configuration-dependencies> </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/05ce603a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHDFS/metainfo.xml index 14f15fc..99cedd9 100644 --- a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHDFS/metainfo.xml @@ -240,7 +240,6 @@ <config-type>ranger-hdfs-policymgr-ssl</config-type> <config-type>ranger-hdfs-security</config-type> <config-type>ams-ssl-client</config-type> - <config-type>hadoop-metrics2.properties</config-type> </configuration-dependencies> <restartRequiredAfterRackChange>true</restartRequiredAfterRackChange>
