Repository: incubator-slider Updated Branches: refs/heads/develop 96f890b1b -> 618cdc4c4
SLIDER-769 Modify Slider app packages for seamless integration with AMS provider (Ambari Metrics Service) Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/618cdc4c Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/618cdc4c Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/618cdc4c Branch: refs/heads/develop Commit: 618cdc4c45b77f4f2e82cb5b86aeb1300cc005c7 Parents: 96f890b Author: Gour Saha <[email protected]> Authored: Tue Jan 27 15:31:38 2015 -0800 Committer: Gour Saha <[email protected]> Committed: Tue Jan 27 15:33:14 2015 -0800 ---------------------------------------------------------------------- app-packages/hbase/appConfig-default.json | 7 ++-- .../hbase/appConfig-phoenix-default.json | 7 ++-- .../hbase/appConfig-secured-default.json | 7 ++-- app-packages/hbase/ganglia_metrics.json | 38 -------------------- app-packages/hbase/metainfo.xml | 8 ++--- app-packages/hbase/package/scripts/params.py | 11 ++++-- ...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 32 ++++++++++++++--- ...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 32 ++++++++++++++--- app-packages/hbase/timeline_metrics.json | 38 ++++++++++++++++++++ 9 files changed, 113 insertions(+), 67 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/appConfig-default.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase/appConfig-default.json b/app-packages/hbase/appConfig-default.json index f065185..4a91884 100644 --- a/app-packages/hbase/appConfig-default.json +++ b/app-packages/hbase/appConfig-default.json @@ -11,10 +11,9 @@ "site.global.app_user": "yarn", "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}", - "site.global.ganglia_server_host": "${NN_HOST}", - "site.global.ganglia_server_port": "8667", - "site.global.ganglia_server_id": "Application1", - "site.global.ganglia_enabled":"true", + "site.global.metric_collector_host": "${NN_HOST}", + "site.global.metric_collector_port": "6188", + "site.global.metric_collector_lib": "", "site.global.hbase_instance_name": "instancename", "site.global.hbase_root_password": "secret", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/appConfig-phoenix-default.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase/appConfig-phoenix-default.json b/app-packages/hbase/appConfig-phoenix-default.json index d74562f..68df0cc 100644 --- a/app-packages/hbase/appConfig-phoenix-default.json +++ b/app-packages/hbase/appConfig-phoenix-default.json @@ -11,10 +11,9 @@ "site.global.app_user": "yarn", "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}", - "site.global.ganglia_server_host": "${NN_HOST}", - "site.global.ganglia_server_port": "8667", - "site.global.ganglia_server_id": "Application1", - "site.global.ganglia_enabled":"true", + "site.global.metric_collector_host": "${NN_HOST}", + "site.global.metric_collector_port": "6188", + "site.global.metric_collector_lib": "", "site.global.hbase_instance_name": "instancename", "site.global.hbase_root_password": "secret", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/appConfig-secured-default.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase/appConfig-secured-default.json b/app-packages/hbase/appConfig-secured-default.json index 2a2b08f..553117e 100644 --- a/app-packages/hbase/appConfig-secured-default.json +++ b/app-packages/hbase/appConfig-secured-default.json @@ -11,10 +11,9 @@ "site.global.app_user": "${USER_NAME}", "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}", - "site.global.ganglia_server_host": "${NN_HOST}", - "site.global.ganglia_server_port": "8667", - "site.global.ganglia_server_id": "Application1", - "site.global.ganglia_enabled":"true", + "site.global.metric_collector_host": "${NN_HOST}", + "site.global.metric_collector_port": "6188", + "site.global.metric_collector_lib": "", "site.global.hbase_instance_name": "instancename", "site.global.hbase_root_password": "secret", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/ganglia_metrics.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase/ganglia_metrics.json b/app-packages/hbase/ganglia_metrics.json deleted file mode 100644 index da73d48..0000000 --- a/app-packages/hbase/ganglia_metrics.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "Component": { - "HBASE_REGIONSERVER": { - "readRequestsCount": { - "metric": "regionserver.Server.readRequestCount", - "pointInTime": false, - "temporal": true - }, - "regions": { - "metric": "regionserver.Server.regionCount", - "pointInTime": false, - "temporal": true - }, - "flushQueueSize": { - "metric": "regionserver.Server.flushQueueLength", - "pointInTime": false, - "temporal": true - } - }, - "HBASE_MASTER": { - "cluster_requests": { - "metric": "master.Server.clusterRequests", - "pointInTime": false, - "temporal": true - }, - "splitTime_avg_time": { - "metric": "master.FileSystem.HlogSplitTime_mean", - "pointInTime": false, - "temporal": true - }, - "splitSize_avg_time": { - "metric": "master.FileSystem.HlogSplitSize_mean", - "pointInTime": false, - "temporal": true - } - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml index d5e07a7..1d01ae7 100644 --- a/app-packages/hbase/metainfo.xml +++ b/app-packages/hbase/metainfo.xml @@ -55,11 +55,7 @@ </export> <export> <name>org.apache.slider.metrics</name> - <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value> - </export> - <export> - <name>org.apache.slider.metrics.ui</name> - <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value> + <value>http://${site.global.metric_collector_host}:${site.global.metric_collector_port}/ws/v1/timeline/metrics</value> </export> </exports> </exportGroup> @@ -81,7 +77,7 @@ <name>HBASE_MASTER</name> <category>MASTER</category> <minInstanceCount>1</minInstanceCount> - <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.metrics,QuickLinks-org.apache.slider.metrics.ui</appExports> + <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.metrics</appExports> <componentExports> <componentExport> <name>org.apache.slider.jmx</name> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase/package/scripts/params.py b/app-packages/hbase/package/scripts/params.py index 027630b..2da4389 100644 --- a/app-packages/hbase/package/scripts/params.py +++ b/app-packages/hbase/package/scripts/params.py @@ -26,6 +26,7 @@ import status_params config = Script.get_config() hbase_root = config['configurations']['global']['app_root'] +app_name = config['clusterName'] conf_dir = format("{hbase_root}/conf") daemon_script = format("{hbase_root}/bin/hbase-daemon.sh") @@ -59,8 +60,12 @@ client_jaas_config_file = default('hbase_client_jaas_config_file', format("{conf master_jaas_config_file = default('hbase_master_jaas_config_file', format("{conf_dir}/hbase_master_jaas.conf")) regionserver_jaas_config_file = default('hbase_regionserver_jaas_config_file', format("{conf_dir}/hbase_regionserver_jaas.conf")) -ganglia_server_host = default('/configurations/global/ganglia_server_host', '') -ganglia_server_port = default('/configurations/global/ganglia_server_port', '8663') +metric_collector_host = config['configurations']['global']['metric_collector_host'] +metric_collector_port = config['configurations']['global']['metric_collector_port'] +metric_collector_lib = config['configurations']['global']['metric_collector_lib'] +has_metric_collector = 1 +if not metric_collector_lib: + has_metric_collector = 0 rest_port = config['configurations']['global']['hbase_rest_port'] rest_infoport = config['configurations']['global']['hbase_rest_infoport'] @@ -88,7 +93,7 @@ if security_enabled: master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase) regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase) - + master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file'] regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file'] kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 b/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 index a3eb1fc..2b37512 100644 --- a/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 +++ b/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 @@ -28,26 +28,48 @@ # If this variable is left out, then the default is no expiration. hbase.extendedperiod = 3600 +{% if has_metric_collector %} + +*.timeline.plugin.urls=file://{{metric_collector_lib}} +hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +hbase.period=10 +hbase.collector={{metric_collector_host}}:{{metric_collector_port}} + +jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +jvm.period=10 +jvm.collector={{metric_collector_host}}:{{metric_collector_port}} + +rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +rpc.period=10 +rpc.collector={{metric_collector_host}}:{{metric_collector_port}} + +hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +hbase.sink.timeline.period=10 +hbase.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}} +hbase.sink.timeline.serviceName-prefix={{app_name}}-master + +{% else %} + # Configuration of the "hbase" context for ganglia # Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) # hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 hbase.period=10 -hbase.servers={{ganglia_server_host}}:{{ganglia_server_port}} +hbase.servers={{metric_collector_host}}:{{metric_collector_port}} # Configuration of the "jvm" context for ganglia # Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) # jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 jvm.period=10 -jvm.servers={{ganglia_server_host}}:{{ganglia_server_port}} +jvm.servers={{metric_collector_host}}:{{metric_collector_port}} # Configuration of the "rpc" context for ganglia # Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) # rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 rpc.period=10 -rpc.servers={{ganglia_server_host}}:{{ganglia_server_port}} +rpc.servers={{metric_collector_host}}:{{metric_collector_port}} #Ganglia following hadoop example hbase.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31 @@ -59,4 +81,6 @@ hbase.sink.ganglia.period=10 .sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both .sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40 -hbase.sink.ganglia.servers={{ganglia_server_host}}:{{ganglia_server_port}} +hbase.sink.ganglia.servers={{metric_collector_host}}:{{metric_collector_port}} + +{% endif %} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 b/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 index a3eb1fc..1954373 100644 --- a/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 +++ b/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 @@ -28,26 +28,48 @@ # If this variable is left out, then the default is no expiration. hbase.extendedperiod = 3600 +{% if has_metric_collector %} + +*.timeline.plugin.urls=file://{{metric_collector_lib}} +hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +hbase.period=10 +hbase.collector={{metric_collector_host}}:{{metric_collector_port}} + +jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +jvm.period=10 +jvm.collector={{metric_collector_host}}:{{metric_collector_port}} + +rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +rpc.period=10 +rpc.collector={{metric_collector_host}}:{{metric_collector_port}} + +hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +hbase.sink.timeline.period=10 +hbase.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}} +hbase.sink.timeline.serviceName-prefix={{app_name}}-rs + +{% else %} + # Configuration of the "hbase" context for ganglia # Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) # hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 hbase.period=10 -hbase.servers={{ganglia_server_host}}:{{ganglia_server_port}} +hbase.servers={{metric_collector_host}}:{{metric_collector_port}} # Configuration of the "jvm" context for ganglia # Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) # jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 jvm.period=10 -jvm.servers={{ganglia_server_host}}:{{ganglia_server_port}} +jvm.servers={{metric_collector_host}}:{{metric_collector_port}} # Configuration of the "rpc" context for ganglia # Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) # rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 rpc.period=10 -rpc.servers={{ganglia_server_host}}:{{ganglia_server_port}} +rpc.servers={{metric_collector_host}}:{{metric_collector_port}} #Ganglia following hadoop example hbase.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31 @@ -59,4 +81,6 @@ hbase.sink.ganglia.period=10 .sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both .sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40 -hbase.sink.ganglia.servers={{ganglia_server_host}}:{{ganglia_server_port}} +hbase.sink.ganglia.servers={{metric_collector_host}}:{{metric_collector_port}} + +{% endif %} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/timeline_metrics.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase/timeline_metrics.json b/app-packages/hbase/timeline_metrics.json new file mode 100644 index 0000000..da73d48 --- /dev/null +++ b/app-packages/hbase/timeline_metrics.json @@ -0,0 +1,38 @@ +{ + "Component": { + "HBASE_REGIONSERVER": { + "readRequestsCount": { + "metric": "regionserver.Server.readRequestCount", + "pointInTime": false, + "temporal": true + }, + "regions": { + "metric": "regionserver.Server.regionCount", + "pointInTime": false, + "temporal": true + }, + "flushQueueSize": { + "metric": "regionserver.Server.flushQueueLength", + "pointInTime": false, + "temporal": true + } + }, + "HBASE_MASTER": { + "cluster_requests": { + "metric": "master.Server.clusterRequests", + "pointInTime": false, + "temporal": true + }, + "splitTime_avg_time": { + "metric": "master.FileSystem.HlogSplitTime_mean", + "pointInTime": false, + "temporal": true + }, + "splitSize_avg_time": { + "metric": "master.FileSystem.HlogSplitSize_mean", + "pointInTime": false, + "temporal": true + } + } + } +}
