Repository: incubator-slider Updated Branches: refs/heads/develop f63d272f3 -> 49ead0dee
SLIDER-769 Modify Slider app packages for seamless integration with AMS provider (Ambari Metrics Service) Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/49ead0de Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/49ead0de Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/49ead0de Branch: refs/heads/develop Commit: 49ead0deefa011ed55dcef4a35bc62137f169b25 Parents: f63d272 Author: Gour Saha <[email protected]> Authored: Thu Feb 5 16:51:12 2015 -0800 Committer: Gour Saha <[email protected]> Committed: Thu Feb 5 16:57:09 2015 -0800 ---------------------------------------------------------------------- app-packages/accumulo/appConfig-default.json | 10 +--- .../accumulo/appConfig-secured-default.json | 11 ++-- .../accumulo/configuration/metrics2.xml | 28 --------- app-packages/accumulo/metainfo.xml | 13 +---- .../accumulo/package/files/accumulo-metrics.xml | 60 -------------------- .../package/scripts/accumulo_configuration.py | 7 +-- app-packages/accumulo/package/scripts/params.py | 9 +++ .../hadoop-metrics2-accumulo.properties.j2 | 46 +++++++++++++++ app-packages/accumulo/timeline_metrics.json | 38 +++++++++++++ app-packages/hbase/package/scripts/params.py | 6 +- ...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 2 +- ...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 2 +- 12 files changed, 108 insertions(+), 124 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ead0de/app-packages/accumulo/appConfig-default.json ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/appConfig-default.json b/app-packages/accumulo/appConfig-default.json index 6e3f1fc..d3f963e 100644 --- a/app-packages/accumulo/appConfig-default.json +++ b/app-packages/accumulo/appConfig-default.json @@ -9,9 +9,9 @@ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}", "site.global.app_user": "${app.user}", "site.global.user_group": "${app.user.group}", - "site.global.ganglia_server_host": "${NN_HOST}", - "site.global.ganglia_server_port": "8669", - "site.global.ganglia_server_id": "Application3", + "site.global.metric_collector_host": "${NN_HOST}", + "site.global.metric_collector_port": "6188", + "site.global.metric_collector_lib": "", "site.accumulo-env.java_home": "${JAVA_HOME}", "site.accumulo-env.tserver_heapsize": "256m", @@ -27,10 +27,6 @@ "site.proxy.port": "${ACCUMULO_PROXY.ALLOCATED_PORT}{PER_CONTAINER}", - "site.metrics2.accumulo.sink.ganglia.class": "org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31", - "site.metrics2.accumulo.sink.ganglia.period": "10", - "site.metrics2.accumulo.sink.ganglia.servers": "${@//site/global/ganglia_server_host}:${@//site/global/ganglia_server_port}", - "site.global.accumulo_root_password": "NOT_USED", "site.global.ssl_cert_dir": "ssl", "site.global.monitor_protocol": "http", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ead0de/app-packages/accumulo/appConfig-secured-default.json ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/appConfig-secured-default.json b/app-packages/accumulo/appConfig-secured-default.json index 358df65..3332662 100644 --- a/app-packages/accumulo/appConfig-secured-default.json +++ b/app-packages/accumulo/appConfig-secured-default.json @@ -9,9 +9,10 @@ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}", "site.global.app_user": "${USER}", "site.global.user_group": "${USER}", - "site.global.ganglia_server_host": "${NN_HOST}", - "site.global.ganglia_server_port": "8669", - "site.global.ganglia_server_id": "Application3", + + "site.global.metric_collector_host": "${NN_HOST}", + "site.global.metric_collector_port": "6188", + "site.global.metric_collector_lib": "", "site.accumulo-env.java_home": "${JAVA_HOME}", "site.accumulo-env.tserver_heapsize": "256m", @@ -27,10 +28,6 @@ "site.proxy.port": "${ACCUMULO_PROXY.ALLOCATED_PORT}{PER_CONTAINER}", - "site.metrics2.accumulo.sink.ganglia.class": "org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31", - "site.metrics2.accumulo.sink.ganglia.period": "10", - "site.metrics2.accumulo.sink.ganglia.servers": "${@//site/global/ganglia_server_host}:${@//site/global/ganglia_server_port}", - "site.global.accumulo_root_password": "NOT_USED", "site.global.ssl_cert_dir": "ssl", "site.global.monitor_protocol": "http", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ead0de/app-packages/accumulo/configuration/metrics2.xml ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/configuration/metrics2.xml b/app-packages/accumulo/configuration/metrics2.xml deleted file mode 100644 index ad3ab73..0000000 --- a/app-packages/accumulo/configuration/metrics2.xml +++ /dev/null @@ -1,28 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> - -<configuration> - <property> - <name>*.period</name> - <value>60</value> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ead0de/app-packages/accumulo/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/metainfo.xml b/app-packages/accumulo/metainfo.xml index 0bf374e..f0ae34d 100644 --- a/app-packages/accumulo/metainfo.xml +++ b/app-packages/accumulo/metainfo.xml @@ -47,11 +47,7 @@ </export> <export> <name>org.apache.slider.metrics</name> - <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value> - </export> - <export> - <name>org.apache.slider.metrics.ui</name> - <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value> + <value>http://${site.global.metric_collector_host}:${site.global.metric_collector_port}/ws/v1/timeline/metrics</value> </export> </exports> </exportGroup> @@ -105,7 +101,7 @@ <name>ACCUMULO_MONITOR</name> <category>MASTER</category> <publishConfig>true</publishConfig> - <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.metrics,QuickLinks-org.apache.slider.metrics.ui</appExports> + <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.metrics</appExports> <commandScript> <script>scripts/accumulo_monitor.py</script> <scriptType>PYTHON</scriptType> @@ -200,11 +196,6 @@ <fileName>proxy.properties</fileName> <dictionaryName>proxy</dictionaryName> </configFile> - <configFile> - <type>properties</type> - <fileName>hadoop-metrics2-accumulo.properties</fileName> - <dictionaryName>metrics2</dictionaryName> - </configFile> </configFiles> </application> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ead0de/app-packages/accumulo/package/files/accumulo-metrics.xml ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/package/files/accumulo-metrics.xml b/app-packages/accumulo/package/files/accumulo-metrics.xml deleted file mode 100644 index 3b97809..0000000 --- a/app-packages/accumulo/package/files/accumulo-metrics.xml +++ /dev/null @@ -1,60 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<!-- - This file follows the conventions for XMLConfiguration files specified in the Apache Commons Configuration 1.5 Library. Changes to this file will be noticed - at runtime (see the FileChangedReloadingStrategy class in Commons Configuration). ---> -<config> -<!-- - Metrics log directory ---> - <logging> - <dir>${ACCUMULO_HOME}/metrics</dir> - </logging> -<!-- - Enable/Disable metrics accumulation on the different servers and their components - NOTE: Turning on logging can be expensive because it will use several more file handles and will create a lot of short lived objects. ---> - <master> - <enabled type="boolean">false</enabled> - <logging type="boolean">false</logging> - </master> - <tserver> - <enabled type="boolean">false</enabled> - <logging type="boolean">false</logging> - <update> - <enabled type="boolean">false</enabled> - <logging type="boolean">false</logging> - </update> - <scan> - <enabled type="boolean">false</enabled> - <logging type="boolean">false</logging> - </scan> - <minc> - <enabled type="boolean">false</enabled> - <logging type="boolean">false</logging> - </minc> - </tserver> - <thrift> - <enabled type="boolean">false</enabled> - <logging type="boolean">false</logging> - </thrift> - <replication> - <enabled type="boolean">false</enabled> - <logging type="boolean">false</logging> - </replication> -</config> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ead0de/app-packages/accumulo/package/scripts/accumulo_configuration.py ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/package/scripts/accumulo_configuration.py b/app-packages/accumulo/package/scripts/accumulo_configuration.py index f5cb7fe..c55994c 100644 --- a/app-packages/accumulo/package/scripts/accumulo_configuration.py +++ b/app-packages/accumulo/package/scripts/accumulo_configuration.py @@ -134,11 +134,7 @@ def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or ' ) # create metrics2 properties file - PropertiesFile(format("{params.conf_dir}/hadoop-metrics2-accumulo.properties"), - properties = params.config['configurations']['metrics2'], - owner = params.accumulo_user, - group = params.user_group - ) + accumulo_TemplateConfig('hadoop-metrics2-accumulo.properties') if name == "proxy": # create proxy.properties file @@ -170,7 +166,6 @@ def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or ' accumulo_StaticFile("auditLog.xml") accumulo_StaticFile("generic_logger.xml") accumulo_StaticFile("monitor_logger.xml") - accumulo_StaticFile("accumulo-metrics.xml") # create the policy file if 'accumulo-policy' in params.config['configurations']: http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ead0de/app-packages/accumulo/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/package/scripts/params.py b/app-packages/accumulo/package/scripts/params.py index 981d3e0..3a53cd6 100644 --- a/app-packages/accumulo/package/scripts/params.py +++ b/app-packages/accumulo/package/scripts/params.py @@ -45,6 +45,7 @@ env_sh_template = config['configurations']['accumulo-env']['content'] # accumulo local directory structure accumulo_root = config['configurations']['global']['app_root'] +app_name = config['clusterName'] conf_dir = format("{accumulo_root}/conf") log_dir = config['configurations']['global']['app_log_dir'] daemon_script = format("{accumulo_root}/bin/accumulo") @@ -84,3 +85,11 @@ if (('accumulo-log4j' in config['configurations']) and ('content' in config['con log4j_props = config['configurations']['accumulo-log4j']['content'] else: log4j_props = None + +metric_collector_host = default('/configurations/global/metric_collector_host', '') +metric_collector_port = default('/configurations/global/metric_collector_port', '') +metric_collector_lib = default('/configurations/global/metric_collector_lib', '') +has_metric_collector = 1 +if not metric_collector_lib: + has_metric_collector = 0 + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ead0de/app-packages/accumulo/package/templates/hadoop-metrics2-accumulo.properties.j2 ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/package/templates/hadoop-metrics2-accumulo.properties.j2 b/app-packages/accumulo/package/templates/hadoop-metrics2-accumulo.properties.j2 new file mode 100644 index 0000000..900a507 --- /dev/null +++ b/app-packages/accumulo/package/templates/hadoop-metrics2-accumulo.properties.j2 @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Poll collectors every ten seconds +*.period=60 + +{% if has_metric_collector %} + +*.timeline.plugin.urls={{metric_collector_lib}} +accumulo.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +accumulo.period=10 +accumulo.collector={{metric_collector_host}}:{{metric_collector_port}} + +jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +jvm.period=10 +jvm.collector={{metric_collector_host}}:{{metric_collector_port}} + +rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +rpc.period=10 +rpc.collector={{metric_collector_host}}:{{metric_collector_port}} + +accumulo.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +accumulo.sink.timeline.period=10 +accumulo.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}} +accumulo.sink.timeline.serviceName-prefix={{app_name}} + +{% else %} + +accumulo.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31 +accumulo.sink.ganglia.period=10 +accumulo.sink.ganglia.servers={{metric_collector_host}}:{{metric_collector_port}} + +{% endif %} + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ead0de/app-packages/accumulo/timeline_metrics.json ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/timeline_metrics.json b/app-packages/accumulo/timeline_metrics.json new file mode 100644 index 0000000..a284f79 --- /dev/null +++ b/app-packages/accumulo/timeline_metrics.json @@ -0,0 +1,38 @@ +{ + "Component": { + "ACCUMULO_MASTER": { + "master_files_pending_repl_avg_time": { + "metric": "master.MasterReplication.FilesPendingReplicationAvgTime", + "pointInTime": false, + "temporal": true + }, + "master_max_repl_threads_avg_time": { + "metric": "master.MasterReplication.MaxReplicationThreadsAvgTime", + "pointInTime": false, + "temporal": true + }, + "master_num_peers_avg_time": { + "metric": "master.MasterReplication.NumPeersAvgTime", + "pointInTime": false, + "temporal": true + } + }, + "ACCUMULO_TSERVER": { + "tserver_entries": { + "metric": "tserver.general.entries", + "pointInTime": false, + "temporal": true + }, + "tserver_minc_avg_count": { + "metric": "tserver.MinorCompactions.MincAvgCount", + "pointInTime": false, + "temporal": true + }, + "tserver_scans_result_avg_count": { + "metric": "tserver.Scans.ResultAvgCount", + "pointInTime": false, + "temporal": true + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ead0de/app-packages/hbase/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase/package/scripts/params.py b/app-packages/hbase/package/scripts/params.py index 2da4389..578ca64 100644 --- a/app-packages/hbase/package/scripts/params.py +++ b/app-packages/hbase/package/scripts/params.py @@ -60,9 +60,9 @@ client_jaas_config_file = default('hbase_client_jaas_config_file', format("{conf master_jaas_config_file = default('hbase_master_jaas_config_file', format("{conf_dir}/hbase_master_jaas.conf")) regionserver_jaas_config_file = default('hbase_regionserver_jaas_config_file', format("{conf_dir}/hbase_regionserver_jaas.conf")) -metric_collector_host = config['configurations']['global']['metric_collector_host'] -metric_collector_port = config['configurations']['global']['metric_collector_port'] -metric_collector_lib = config['configurations']['global']['metric_collector_lib'] +metric_collector_host = default('/configurations/global/metric_collector_host', '') +metric_collector_port = default('/configurations/global/metric_collector_port', '') +metric_collector_lib = default('/configurations/global/metric_collector_lib', '') has_metric_collector = 1 if not metric_collector_lib: has_metric_collector = 0 http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ead0de/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 b/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 index 2b37512..43aea01 100644 --- a/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 +++ b/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 @@ -30,7 +30,7 @@ hbase.extendedperiod = 3600 {% if has_metric_collector %} -*.timeline.plugin.urls=file://{{metric_collector_lib}} +*.timeline.plugin.urls={{metric_collector_lib}} hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink hbase.period=10 hbase.collector={{metric_collector_host}}:{{metric_collector_port}} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ead0de/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 b/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 index 1954373..2ec4b24 100644 --- a/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 +++ b/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 @@ -30,7 +30,7 @@ hbase.extendedperiod = 3600 {% if has_metric_collector %} -*.timeline.plugin.urls=file://{{metric_collector_lib}} +*.timeline.plugin.urls={{metric_collector_lib}} hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink hbase.period=10 hbase.collector={{metric_collector_host}}:{{metric_collector_port}}
