AMBARI-8522. Enable Flume metrics sink to AMS. Missing stack changes. (Szilard Nemethy via mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/86b0ce46 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/86b0ce46 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/86b0ce46 Branch: refs/heads/trunk Commit: 86b0ce46c69c22d1bb23e976fcea06cd93dae8ea Parents: 9d80663 Author: Myroslav Papirkovskyy <mpapyrkovs...@hortonworks.com> Authored: Wed Dec 10 16:44:42 2014 +0200 Committer: Myroslav Papirkovskyy <mpapyrkovs...@hortonworks.com> Committed: Wed Dec 10 23:48:44 2014 +0200 ---------------------------------------------------------------------- .../services/FLUME/configuration/flume-env.xml | 3 ++- .../services/FLUME/package/scripts/flume.py | 8 +++++++ .../services/FLUME/package/scripts/params.py | 5 +++++ .../templates/flume-metrics2.properties.j2 | 22 ++++++++++++++++++++ .../2.2/services/AMS/configuration/ams-site.xml | 2 +- 5 files changed, 38 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/86b0ce46/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-env.xml index 547eb8c..bb56f9c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-env.xml @@ -69,7 +69,8 @@ export JAVA_HOME={{java_home}} # export JAVA_OPTS="-Xms100m -Xmx2000m -Dcom.sun.management.jmxremote" # Note that the Flume conf directory is always included in the classpath. -#FLUME_CLASSPATH="" +#TODO temporary addition +export FLUME_CLASSPATH=$FLUME_CLASSPATH:/usr/lib/flume/lib/* export HIVE_HOME={{flume_hive_home}} export HCAT_HOME={{flume_hcat_home}} http://git-wip-us.apache.org/repos/asf/ambari/blob/86b0ce46/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/scripts/flume.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/scripts/flume.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/scripts/flume.py index 9b1dfe1..f4870ad 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/scripts/flume.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/scripts/flume.py @@ -65,6 +65,12 @@ def flume(action = None): content=InlineTemplate(params.flume_env_sh_template) ) + if params.has_metric_collector: + File(os.path.join(flume_agent_conf_dir, "flume-metrics2.properties"), + owner=params.flume_user, + content=Template("flume-metrics2.properties.j2") + ) + elif action == 'start': # desired state for service should be STARTED if len(params.flume_command_targets) == 0: @@ -89,6 +95,8 @@ def flume(action = None): if params.ganglia_server_host is not None: extra_args = '-Dflume.monitoring.type=ganglia -Dflume.monitoring.hosts={0}:{1}' extra_args = extra_args.format(params.ganglia_server_host, '8655') + if params.has_metric_collector: + extra_args = '-Dflume.monitoring.type=org.apache.hadoop.metrics2.sink.flume.FlumeTimelineMetricsSink' flume_cmd = flume_base.format(agent, flume_agent_conf_dir, flume_agent_conf_file, extra_args) http://git-wip-us.apache.org/repos/asf/ambari/blob/86b0ce46/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/scripts/params.py index 1b16d26..2997867 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/scripts/params.py @@ -73,3 +73,8 @@ if 0 != len(ganglia_server_hosts): hostname = None if config.has_key('hostname'): hostname = config['hostname'] + +ams_collector_hosts = default("/clusterHostInfo/metric_collector_hosts", []) +has_metric_collector = not len(ams_collector_hosts) == 0 +if has_metric_collector: + metric_collector_host = ams_collector_hosts[0] http://git-wip-us.apache.org/repos/asf/ambari/blob/86b0ce46/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/templates/flume-metrics2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/templates/flume-metrics2.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/templates/flume-metrics2.properties.j2 new file mode 100644 index 0000000..7458bf8 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/templates/flume-metrics2.properties.j2 @@ -0,0 +1,22 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +collector={{metric_collector_host}}:8188 +collectionFrequency=60000 +maxRowCacheSize=10000 +sendInterval=59000 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/86b0ce46/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-site.xml index 55c79ef..f84a418 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-site.xml @@ -205,7 +205,7 @@ </property> <property> <name>timeline.metrics.service.checkpointDelay</name> - <value>120</value> + <value>60</value> <description> Time in seconds to sleep on the first run or when the checkpoint is too old.
