AMBARI-8782: Common Services: Refactor HDP-2.2 AMS, KAFKA, KNOX, SLIDER services (Jayush Luniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ef7fe6b5 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ef7fe6b5 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ef7fe6b5 Branch: refs/heads/trunk Commit: ef7fe6b5cefa87042083deaa484bf7bd773b9ec7 Parents: 9e22e17 Author: Jayush Luniya <[email protected]> Authored: Thu Dec 18 21:26:54 2014 -0800 Committer: Jayush Luniya <[email protected]> Committed: Thu Dec 18 21:26:54 2014 -0800 ---------------------------------------------------------------------- .../common-services/AMS/0.1.0/alerts.json | 155 +++++++++ .../AMS/0.1.0/configuration/ams-env.xml | 41 +++ .../AMS/0.1.0/configuration/ams-hbase-env.xml | 137 ++++++++ .../AMS/0.1.0/configuration/ams-hbase-log4j.xml | 143 +++++++++ .../0.1.0/configuration/ams-hbase-policy.xml | 53 +++ .../AMS/0.1.0/configuration/ams-hbase-site.xml | 268 ++++++++++++++++ .../AMS/0.1.0/configuration/ams-log4j.xml | 59 ++++ .../AMS/0.1.0/configuration/ams-site.xml | 221 +++++++++++++ .../common-services/AMS/0.1.0/metainfo.xml | 104 ++++++ .../files/alert_ambari_metrics_monitor.py | 80 +++++ .../AMS/0.1.0/package/files/hbaseSmokeVerify.sh | 34 ++ .../AMS/0.1.0/package/scripts/__init__.py | 19 ++ .../AMS/0.1.0/package/scripts/ams.py | 89 +++++ .../AMS/0.1.0/package/scripts/ams_service.py | 72 +++++ .../AMS/0.1.0/package/scripts/functions.py | 40 +++ .../AMS/0.1.0/package/scripts/hbase.py | 123 +++++++ .../AMS/0.1.0/package/scripts/hbase_master.py | 70 ++++ .../0.1.0/package/scripts/hbase_regionserver.py | 66 ++++ .../AMS/0.1.0/package/scripts/hbase_service.py | 51 +++ .../0.1.0/package/scripts/metric_collector.py | 63 ++++ .../AMS/0.1.0/package/scripts/metric_monitor.py | 61 ++++ .../AMS/0.1.0/package/scripts/params.py | 174 ++++++++++ .../AMS/0.1.0/package/scripts/service_check.py | 19 ++ .../AMS/0.1.0/package/scripts/status_params.py | 30 ++ .../hadoop-metrics2-hbase.properties.j2 | 56 ++++ .../package/templates/hbase_client_jaas.conf.j2 | 23 ++ .../templates/hbase_grant_permissions.j2 | 39 +++ .../package/templates/hbase_master_jaas.conf.j2 | 26 ++ .../templates/hbase_regionserver_jaas.conf.j2 | 26 ++ .../package/templates/metric_groups.conf.j2 | 37 +++ .../package/templates/metric_monitor.ini.j2 | 30 ++ .../0.1.0/package/templates/regionservers.j2 | 20 ++ .../common-services/KAFKA/0.8.1.2.2/alerts.json | 32 ++ .../0.8.1.2.2/configuration/kafka-broker.xml | 321 +++++++++++++++++++ .../KAFKA/0.8.1.2.2/configuration/kafka-env.xml | 55 ++++ .../0.8.1.2.2/configuration/kafka-log4j.xml | 116 +++++++ .../KAFKA/0.8.1.2.2/metainfo.xml | 82 +++++ .../KAFKA/0.8.1.2.2/metrics.json | 262 +++++++++++++++ .../KAFKA/0.8.1.2.2/package/scripts/kafka.py | 70 ++++ .../0.8.1.2.2/package/scripts/kafka_broker.py | 63 ++++ .../KAFKA/0.8.1.2.2/package/scripts/params.py | 57 ++++ .../package/scripts/properties_config.py | 32 ++ .../0.8.1.2.2/package/scripts/service_check.py | 65 ++++ .../0.8.1.2.2/package/scripts/status_params.py | 26 ++ .../common-services/KNOX/0.5.0.2.2/alerts.json | 32 ++ .../0.5.0.2.2/configuration/gateway-log4j.xml | 83 +++++ .../0.5.0.2.2/configuration/gateway-site.xml | 72 +++++ .../KNOX/0.5.0.2.2/configuration/knox-env.xml | 53 +++ .../KNOX/0.5.0.2.2/configuration/ldap-log4j.xml | 66 ++++ .../KNOX/0.5.0.2.2/configuration/topology.xml | 116 +++++++ .../KNOX/0.5.0.2.2/configuration/users-ldif.xml | 135 ++++++++ .../common-services/KNOX/0.5.0.2.2/metainfo.xml | 87 +++++ .../package/files/validateKnoxStatus.py | 43 +++ .../KNOX/0.5.0.2.2/package/scripts/knox.py | 85 +++++ .../0.5.0.2.2/package/scripts/knox_gateway.py | 103 ++++++ .../KNOX/0.5.0.2.2/package/scripts/ldap.py | 39 +++ .../KNOX/0.5.0.2.2/package/scripts/params.py | 139 ++++++++ .../0.5.0.2.2/package/scripts/service_check.py | 58 ++++ .../0.5.0.2.2/package/scripts/status_params.py | 27 ++ .../package/templates/krb5JAASLogin.conf.j2 | 30 ++ .../0.60.0.2.2/configuration/slider-client.xml | 60 ++++ .../0.60.0.2.2/configuration/slider-env.xml | 43 +++ .../0.60.0.2.2/configuration/slider-log4j.xml | 89 +++++ .../SLIDER/0.60.0.2.2/metainfo.xml | 132 ++++++++ .../package/files/hbaseSmokeVerify.sh | 34 ++ .../0.60.0.2.2/package/scripts/__init__.py | 19 ++ .../SLIDER/0.60.0.2.2/package/scripts/params.py | 48 +++ .../0.60.0.2.2/package/scripts/service_check.py | 45 +++ .../SLIDER/0.60.0.2.2/package/scripts/slider.py | 62 ++++ .../0.60.0.2.2/package/scripts/slider_client.py | 43 +++ .../package/templates/storm-slider-env.sh.j2 | 38 +++ .../stacks/HDP/2.2/services/AMS/alerts.json | 155 --------- .../2.2/services/AMS/configuration/ams-env.xml | 41 --- .../AMS/configuration/ams-hbase-env.xml | 137 -------- .../AMS/configuration/ams-hbase-log4j.xml | 143 --------- .../AMS/configuration/ams-hbase-policy.xml | 53 --- .../AMS/configuration/ams-hbase-site.xml | 268 ---------------- .../services/AMS/configuration/ams-log4j.xml | 59 ---- .../2.2/services/AMS/configuration/ams-site.xml | 221 ------------- .../stacks/HDP/2.2/services/AMS/metainfo.xml | 80 +---- .../files/alert_ambari_metrics_monitor.py | 80 ----- .../AMS/package/files/hbaseSmokeVerify.sh | 34 -- .../services/AMS/package/scripts/__init__.py | 19 -- .../HDP/2.2/services/AMS/package/scripts/ams.py | 89 ----- .../services/AMS/package/scripts/ams_service.py | 72 ----- .../services/AMS/package/scripts/functions.py | 40 --- .../2.2/services/AMS/package/scripts/hbase.py | 123 ------- .../AMS/package/scripts/hbase_master.py | 70 ---- .../AMS/package/scripts/hbase_regionserver.py | 66 ---- .../AMS/package/scripts/hbase_service.py | 51 --- .../AMS/package/scripts/metric_collector.py | 63 ---- .../AMS/package/scripts/metric_monitor.py | 61 ---- .../2.2/services/AMS/package/scripts/params.py | 174 ---------- .../AMS/package/scripts/service_check.py | 19 -- .../AMS/package/scripts/status_params.py | 30 -- .../hadoop-metrics2-hbase.properties.j2 | 56 ---- .../package/templates/hbase_client_jaas.conf.j2 | 23 -- .../templates/hbase_grant_permissions.j2 | 39 --- .../package/templates/hbase_master_jaas.conf.j2 | 26 -- .../templates/hbase_regionserver_jaas.conf.j2 | 26 -- .../AMS/package/templates/metric_groups.conf.j2 | 37 --- .../AMS/package/templates/metric_monitor.ini.j2 | 30 -- .../AMS/package/templates/regionservers.j2 | 20 -- .../stacks/HDP/2.2/services/KAFKA/alerts.json | 32 -- .../KAFKA/configuration/kafka-broker.xml | 321 ------------------- .../services/KAFKA/configuration/kafka-env.xml | 55 ---- .../KAFKA/configuration/kafka-log4j.xml | 116 ------- .../stacks/HDP/2.2/services/KAFKA/metainfo.xml | 58 +--- .../stacks/HDP/2.2/services/KAFKA/metrics.json | 262 --------------- .../2.2/services/KAFKA/package/scripts/kafka.py | 70 ---- .../KAFKA/package/scripts/kafka_broker.py | 63 ---- .../services/KAFKA/package/scripts/params.py | 57 ---- .../KAFKA/package/scripts/properties_config.py | 32 -- .../KAFKA/package/scripts/service_check.py | 65 ---- .../KAFKA/package/scripts/status_params.py | 26 -- .../stacks/HDP/2.2/services/KNOX/alerts.json | 32 -- .../KNOX/configuration/gateway-log4j.xml | 83 ----- .../KNOX/configuration/gateway-site.xml | 72 ----- .../services/KNOX/configuration/knox-env.xml | 53 --- .../services/KNOX/configuration/ldap-log4j.xml | 66 ---- .../services/KNOX/configuration/topology.xml | 116 ------- .../services/KNOX/configuration/users-ldif.xml | 135 -------- .../stacks/HDP/2.2/services/KNOX/metainfo.xml | 63 +--- .../KNOX/package/files/validateKnoxStatus.py | 43 --- .../2.2/services/KNOX/package/scripts/knox.py | 85 ----- .../KNOX/package/scripts/knox_gateway.py | 103 ------ .../2.2/services/KNOX/package/scripts/ldap.py | 39 --- .../2.2/services/KNOX/package/scripts/params.py | 139 -------- .../KNOX/package/scripts/service_check.py | 58 ---- .../KNOX/package/scripts/status_params.py | 27 -- .../package/templates/krb5JAASLogin.conf.j2 | 30 -- .../SLIDER/configuration/slider-client.xml | 60 ---- .../SLIDER/configuration/slider-env.xml | 43 --- .../SLIDER/configuration/slider-log4j.xml | 89 ----- .../stacks/HDP/2.2/services/SLIDER/metainfo.xml | 108 +------ .../SLIDER/package/files/hbaseSmokeVerify.sh | 34 -- .../services/SLIDER/package/scripts/__init__.py | 19 -- .../services/SLIDER/package/scripts/params.py | 48 --- .../SLIDER/package/scripts/service_check.py | 45 --- .../services/SLIDER/package/scripts/slider.py | 62 ---- .../SLIDER/package/scripts/slider_client.py | 43 --- .../package/templates/storm-slider-env.sh.j2 | 38 --- .../stacks/2.2/KAFKA/test_kafka_broker.py | 8 +- .../python/stacks/2.2/KNOX/test_knox_gateway.py | 8 +- .../stacks/2.2/SLIDER/test_slider_client.py | 21 +- 145 files changed, 5422 insertions(+), 5301 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/alerts.json b/ambari-server/src/main/resources/common-services/AMS/0.1.0/alerts.json new file mode 100644 index 0000000..714b940 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/alerts.json @@ -0,0 +1,155 @@ +{ + "AMS": { + "service": [ + { + "name": "ams_metric_monitor_process_percent", + "label": "Percent AMS Metric Monitors Available", + "interval": 1, + "scope": "SERVICE", + "enabled": true, + "source": { + "type": "AGGREGATE", + "alert_name": "ams_metric_monitor_process", + "reporting": { + "ok": { + "text": "affected: [{1}], total: [{0}]" + }, + "warning": { + "text": "affected: [{1}], total: [{0}]", + "value": 0.1 + }, + "critical": { + "text": "affected: [{1}], total: [{0}]", + "value": 0.3 + } + } + } + } + ], + "METRIC_COLLECTOR": [ + { + "name": "ams_metric_collector_process", + "label": "AMS Metric Collector Process", + "interval": 1, + "scope": "ANY", + "enabled": true, + "source": { + "type": "PORT", + "uri": "8188", + "default_port": 8188, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + }, + { + "name": "ams_metric_collector_hbase_master_process", + "label": "AMS Metric Collector HBase Master Process", + "interval": 1, + "scope": "ANY", + "source": { + "type": "PORT", + "uri": "{{ams-hbase-site/hbase.master.info.port}}", + "default_port": 61310, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + }, + { + "name": "ams_metric_collector_hbase_master_cpu", + "label": "AMS Metric Collector HBase Maser CPU Utilization", + "interval": 5, + "scope": "ANY", + "enabled": true, + "source": { + "type": "METRIC", + "uri": { + "http": "{{ams-hbase-site/hbase.master.info.port}}", + "https": "{{ams-hbase-site/hbase.master.info.port}}", + "https_property": "{{cluster-env/security_enabled}}", + "https_property_value": "true", + "default_port": 61310 + }, + "reporting": { + "ok": { + "text": "{1} CPU, load {0:.1%}" + }, + "warning": { + "text": "{1} CPU, load {0:.1%}", + "value": 200 + }, + "critical": { + "text": "{1} CPU, load {0:.1%}", + "value": 250 + }, + "units" : "%" + }, + "jmx": { + "property_list": [ + "java.lang:type=OperatingSystem/SystemCpuLoad", + "java.lang:type=OperatingSystem/AvailableProcessors" + ], + "value": "{0} * 100" + } + } + }, + { + "name": "ams_metric_collector_zookeeper_server_process", + "label": "AMS Metric Collector ZooKeeper Server Process", + "interval": 1, + "scope": "ANY", + "source": { + "type": "PORT", + "uri": "{{ams-hbase-site/hbase.zookeeper.property.clientPort}}", + "default_port": 61181, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + } + ], + "METRIC_MONITOR": [ + { + "name": "ams_metric_monitor_process", + "label": "AMS Metric Monitor Status", + "interval": 1, + "scope": "ANY", + "source": { + "type": "SCRIPT", + "path": "AMS/0.1.0/package/files/alert_ambari_metrics_monitor.py" + } + } + ] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-env.xml b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-env.xml new file mode 100644 index 0000000..fda1df0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-env.xml @@ -0,0 +1,41 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<configuration> + <property> + <name>ams_user</name> + <value>root</value> + <property-type>USER</property-type> + <description>AMS User Name.</description> + </property> + + <property> + <name>content</name> + <value> + # Set environment variables here. + + # The java implementation to use. Java 1.6 required. + export JAVA_HOME={{java64_home}} + + #TODO + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-env.xml b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-env.xml new file mode 100644 index 0000000..7a61c60 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-env.xml @@ -0,0 +1,137 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>hbase_log_dir</name> + <value>/var/log/ams-hbase/</value> + <description>Log Directories for HBase.</description> + </property> + <property> + <name>hbase_pid_dir</name> + <value>/var/run/ams-hbase/</value> + <description>Pid Directory for HBase.</description> + </property> + <property> + <name>hbase_regionserver_heapsize</name> + <value>1024m</value> + <description>HBase RegionServer Heap Size.</description> + </property> + <property> + <name>hbase_regionserver_xmn_max</name> + <value>512m</value> + <description>HBase RegionServer maximum value for minimum heap size.</description> + </property> + <property> + <name>hbase_regionserver_xmn_ratio</name> + <value>0.2</value> + <description>HBase RegionServer minimum heap size is calculated as a percentage of max heap size.</description> + </property> + <property> + <name>hbase_master_heapsize</name> + <value>1024m</value> + <description>HBase Master Heap Size</description> + </property> + <property> + <name>hbase_user</name> + <value>hbase</value> + <property-type>USER</property-type> + <description>HBase User Name.</description> + </property> + + <!-- hbase-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for hbase-env.sh file</description> + <value> + # Set environment variables here. + + # The java implementation to use. Java 1.6 required. + export JAVA_HOME={{java64_home}} + + # HBase Configuration directory + export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}} + + # Extra Java CLASSPATH elements. Optional. + export HBASE_CLASSPATH=${HBASE_CLASSPATH} + + if [ -f "/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar" ]; then + export HBASE_CLASSPATH=${HBASE_CLASSPATH}:/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar + fi + + # The maximum amount of heap to use, in MB. Default is 1000. + # export HBASE_HEAPSIZE=1000 + + # Extra Java runtime options. + # Below are what we set by default. May only work with SUN JVM. + # For more on why as well as other possible settings, + # see http://wiki.apache.org/hadoop/PerformanceTuning + export HBASE_OPTS="-XX:+UseConcMarkSweepGC -XX:ErrorFile={{hbase_log_dir}}/hs_err_pid%p.log" + export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`" + # Uncomment below to enable java garbage collection logging. + # export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log" + + # Uncomment and adjust to enable JMX exporting + # See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. + # More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html + # + # export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" + export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}" + export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}" + # export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" + # export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" + + # File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. + export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers + + # Extra ssh options. Empty by default. + # export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + + # Where log files are stored. $HBASE_HOME/logs by default. + export HBASE_LOG_DIR={{hbase_log_dir}} + + # A string representing this instance of hbase. $USER by default. + # export HBASE_IDENT_STRING=$USER + + # The scheduling priority for daemon processes. See 'man nice'. + # export HBASE_NICENESS=10 + + # The directory where pid files are stored. /tmp by default. + export HBASE_PID_DIR={{hbase_pid_dir}} + + # Seconds to sleep between slave commands. Unset by default. This + # can be useful in large clusters, where, e.g., slave rsyncs can + # otherwise arrive faster than the master can service them. + # export HBASE_SLAVE_SLEEP=0.1 + + # Tell HBase whether it should manage it's own instance of Zookeeper or not. + export HBASE_MANAGES_ZK=false + + {% if security_enabled %} + export HBASE_OPTS="$HBASE_OPTS -Djava.security.auth.login.config={{client_jaas_config_file}}" + export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Djava.security.auth.login.config={{master_jaas_config_file}}" + export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config={{regionserver_jaas_config_file}}" + {% endif %} + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-log4j.xml b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-log4j.xml new file mode 100644 index 0000000..24ba5b7 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-log4j.xml @@ -0,0 +1,143 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_final="false"> + + <property> + <name>content</name> + <description>Custom log4j.properties</description> + <value> + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + + # Define some default values that can be overridden by system properties + hbase.root.logger=INFO,console + hbase.security.logger=INFO,console + hbase.log.dir=. + hbase.log.file=hbase.log + + # Define the root logger to the system property "hbase.root.logger". + log4j.rootLogger=${hbase.root.logger} + + # Logging Threshold + log4j.threshold=ALL + + # + # Daily Rolling File Appender + # + log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender + log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} + + # Rollver at midnight + log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + + # 30-day backup + #log4j.appender.DRFA.MaxBackupIndex=30 + log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + + # Pattern format: Date LogLevel LoggerName LogMessage + log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + + # Rolling File Appender properties + hbase.log.maxfilesize=256MB + hbase.log.maxbackupindex=20 + + # Rolling File Appender + log4j.appender.RFA=org.apache.log4j.RollingFileAppender + log4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file} + + log4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize} + log4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex} + + log4j.appender.RFA.layout=org.apache.log4j.PatternLayout + log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + + # + # Security audit appender + # + hbase.security.log.file=SecurityAuth.audit + hbase.security.log.maxfilesize=256MB + hbase.security.log.maxbackupindex=20 + log4j.appender.RFAS=org.apache.log4j.RollingFileAppender + log4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file} + log4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize} + log4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex} + log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout + log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n + log4j.category.SecurityLogger=${hbase.security.logger} + log4j.additivity.SecurityLogger=false + #log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE + + # + # Null Appender + # + log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + + # + # console + # Add "console" to rootlogger above if you want to use this + # + log4j.appender.console=org.apache.log4j.ConsoleAppender + log4j.appender.console.target=System.err + log4j.appender.console.layout=org.apache.log4j.PatternLayout + log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + + # Custom Logging levels + + log4j.logger.org.apache.zookeeper=INFO + #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG + log4j.logger.org.apache.hadoop.hbase=DEBUG + # Make these two classes INFO-level. Make them DEBUG to see more zk debug. + log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO + log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO + #log4j.logger.org.apache.hadoop.dfs=DEBUG + # Set this class to log INFO only otherwise its OTT + # Enable this to get detailed connection error/retry logging. + # log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE + + + # Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output) + #log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG + + # Uncomment the below if you want to remove logging of client region caching' + # and scan of .META. messages + # log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO + # log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO + + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-policy.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-policy.xml b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-policy.xml new file mode 100644 index 0000000..febbd44 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-policy.xml @@ -0,0 +1,53 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_final="true"> + <property> + <name>security.client.protocol.acl</name> + <value>*</value> + <description>ACL for HRegionInterface protocol implementations (ie. + clients talking to HRegionServers) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed.</description> + </property> + + <property> + <name>security.admin.protocol.acl</name> + <value>*</value> + <description>ACL for HMasterInterface protocol implementation (ie. + clients talking to HMaster for admin operations). + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed.</description> + </property> + + <property> + <name>security.masterregion.protocol.acl</name> + <value>*</value> + <description>ACL for HMasterRegionInterface protocol implementations + (for HRegionServers communicating with HMaster) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed.</description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-site.xml b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-site.xml new file mode 100644 index 0000000..9d35ba2 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-site.xml @@ -0,0 +1,268 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>hbase.rootdir</name> + <value>file:///var/lib/ambari-metrics-collector/hbase</value> + <description> + AMS service uses HBase as default storage backend. Set the rootdir for + HBase to either local filesystem path if using AMS in embedded mode or + to a HDFS dir, example: hdfs://namenode.example.org:9000/hbase. By + default HBase writes into /tmp. Change this configuration else all data + will be lost on machine restart. + </description> + </property> + <property> + <name>hbase.tmp.dir</name> + <value>/var/lib/ambari-metrics-collector/hbase-tmp</value> + <description> + Temporary directory on the local filesystem. + Change this setting to point to a location more permanent + than '/tmp' (The '/tmp' directory is often cleared on + machine restart). + </description> + </property> + <property> + <name>hbase.local.dir</name> + <value>${hbase.tmp.dir}/local</value> + <description>Directory on the local filesystem to be used as a local storage + </description> + </property> + <property> + <name>hbase.cluster.distributed</name> + <value>false</value> + <description> + The mode the cluster will be in. Possible values are false for + standalone mode and true for distributed mode. If false, startup will run + all HBase and ZooKeeper daemons together in the one JVM. + </description> + </property> + <property> + <name>hbase.master.wait.on.regionservers.mintostart</name> + <value>1</value> + <description> + Ensure that HBase Master waits for # many region server to start. + </description> + </property> + <property> + <name>hbase.zookeeper.quorum</name> + <value>localhost</value> + <description>Comma separated list of servers in the ZooKeeper Quorum. + For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com". + By default this is set to localhost for local and pseudo-distributed modes + of operation. For a fully-distributed setup, this should be set to a full + list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh + this is the list of servers which we will start/stop ZooKeeper on. + </description> + </property> + <property> + <name>hbase.master.info.bindAddress</name> + <value>0.0.0.0</value> + <description>The bind address for the HBase Master web UI</description> + </property> + <property> + <name>hbase.master.info.port</name> + <value>61310</value> + <description>The port for the HBase Master web UI.</description> + </property> + <property> + <name>hbase.regionserver.info.port</name> + <value>61330</value> + <description>The port for the HBase RegionServer web UI.</description> + </property> + <property> + <name>hbase.master.port</name> + <value>61300</value> + <description>The port for the HBase Master web UI.</description> + </property> + <property> + <name>hbase.regionserver.port</name> + <value>61320</value> + <description>The port for the HBase RegionServer web UI.</description> + </property> + <property> + <name>hbase.hregion.majorcompaction</name> + <value>0</value> + <description> + The time (in milliseconds) between 'major' compactions of all + HStoreFiles in a region. + 0 to disable automated major compactions. + </description> + </property> + <property> + <name>phoenix.query.spoolThresholdBytes</name> + <value>12582912</value> + <description> + Threshold size in bytes after which results from parallelly executed + query results are spooled to disk. Default is 20 mb. + </description> + </property> + <property> + <name>hbase.zookeeper.property.dataDir</name> + <value>${hbase.tmp.dir}/zookeeper</value> + <description> + Property from ZooKeeper's config zoo.cfg. + The directory where the snapshot is stored. + </description> + </property> + <property> + <name>hbase.client.scanner.caching</name> + <value>10000</value> + <description> + Number of rows that will be fetched when calling next on a scanner + if it is not served from (local, client) memory. + </description> + </property> + <property> + <name>hfile.block.cache.size</name> + <value>0.3</value> + <description> + Percentage of maximum heap (-Xmx setting) to allocate to block cache + used by a StoreFile. Default of 0.4 means allocate 40%. + </description> + </property> + <property> + <name>hbase.regionserver.global.memstore.upperLimit</name> + <value>0.5</value> + <description> + Maximum size of all memstores in a region server before new + updates are blocked and flushes are forced. Defaults to 40% of heap + </description> + </property> + <property> + <name>hbase.regionserver.global.memstore.lowerLimit</name> + <value>0.4</value> + <description> + When memstores are being forced to flush to make room in + memory, keep flushing until we hit this mark. Defaults to 35% of heap. + This value equal to hbase.regionserver.global.memstore.upperLimit causes + the minimum possible flushing to occur when updates are blocked due to + memstore limiting. + </description> + </property> + <property> + <name>phoenix.groupby.maxCacheSize</name> + <value>307200000</value> + <description> + Size in bytes of pages cached during GROUP BY spilling. Default is 100Mb. + </description> + </property> + <property> + <name>hbase.hregion.memstore.block.multiplier</name> + <value>4</value> + <description> + Block updates if memstore has hbase.hregion.memstore.block.multiplier + times hbase.hregion.memstore.flush.size bytes. Useful preventing runaway + memstore during spikes in update traffic. + </description> + </property> + <property> + <name>hbase.hstore.flusher.count</name> + <value>2</value> + <description> + The number of flush threads. With fewer threads, the MemStore flushes + will be queued. With more threads, the flushes will be executed in parallel, + increasing the load on HDFS, and potentially causing more compactions. + </description> + </property> + <property> + <name>phoenix.query.timeoutMs</name> + <value>1200000</value> + <description> + Number of milliseconds after which a query will timeout on the client. + Default is 10 min. + </description> + </property> + <property> + <name>hbase.client.scanner.timeout.period</name> + <value>900000</value> + <description> + Client scanner lease period in milliseconds. + </description> + </property> + <property> + <name>hbase.regionserver.thread.compaction.large</name> + <value>2</value> + <description> + Configuration key for the large compaction threads. + </description> + </property> + <property> + <name>hbase.regionserver.thread.compaction.small</name> + <value>3</value> + <description> + Configuration key for the small compaction threads. + </description> + </property> + <property> + <name>hbase.zookeeper.property.clientPort</name> + <value>61181</value> + </property> + <property> + <name>hbase.zookeeper.peerport</name> + <value>61288</value> + </property> + <property> + <name>hbase.zookeeper.leaderport</name> + <value>61388</value> + </property> + <property> + <name>hbase.hstore.blockingStoreFiles</name> + <value>200</value> + <description> + If more than this number of StoreFiles exist in any one Store + (one StoreFile is written per flush of MemStore), updates are blocked for + this region until a compaction is completed, or until + hbase.hstore.blockingWaitTime has been exceeded. + </description> + </property> + <property> + <name>hbase.hregion.memstore.flush.size</name> + <value>134217728</value> + <description> + Memstore will be flushed to disk if size of the memstore exceeds this + number of bytes. Value is checked by a thread that runs every + hbase.server.thread.wakefrequency. + </description> + </property> + <property> + <name>phoenix.query.spoolThresholdBytes</name> + <value>12582912</value> + </property> + <property> + <name>hbase.snapshot.enabled</name> + <value>false</value> + <description>Enable/Disable HBase snapshots.</description> + </property> + <property> + <name>hbase.replication</name> + <value>false</value> + <description>Enable/Disable HBase replication.</description> + </property> + <property> + <name>zookeeper.session.timeout</name> + <value>120000</value> + <description>ZooKeeper session timeout in milliseconds.</description> + </property> + +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-log4j.xml b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-log4j.xml new file mode 100644 index 0000000..3f2e148 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-log4j.xml @@ -0,0 +1,59 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<configuration supports_final="false"> + + <property> + <name>content</name> + <description>Custom log4j.properties</description> + <value> + # + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + # + + # Define some default values that can be overridden by system properties + # Root logger option + log4j.rootLogger=INFO,file + + # Direct log messages to a log file + log4j.appender.file=org.apache.log4j.RollingFileAppender + log4j.appender.file.File=/var/log/ambari-metrics-collector/ambari-metrics-collector.log + log4j.appender.file.MaxFileSize=80MB + log4j.appender.file.MaxBackupIndex=60 + log4j.appender.file.layout=org.apache.log4j.PatternLayout + log4j.appender.file.layout.ConversionPattern=%d{ABSOLUTE} %5p [%t] %c{1}:%L - %m%n + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-site.xml b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-site.xml new file mode 100644 index 0000000..f84a418 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-site.xml @@ -0,0 +1,221 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>timeline.metrics.service.operation.mode</name> + <value>embedded</value> + <description> + Service Operation modes: + 1) embedded: Metrics stored on local FS, HBase in Standalone mode + 2) distributed: HBase daemons writing to HDFS + 3) external: External HBase storage backend + </description> + </property> + <property> + <name>timeline.metrics.aggregator.checkpoint.dir</name> + <value>/tmp</value> + <description> + Directory to store aggregator checkpoints. Change to a permanent + location so that checkpoint ar not lost. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.hourly.interval</name> + <value>3600</value> + <description> + Time in seconds to sleep for the hourly resolution host based + aggregator. Default resolution is 1 hour. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.minute.interval</name> + <value>300</value> + <description> + Time in seconds to sleep for the minute resolution host based + aggregator. Default resolution is 5 minutes. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.hourly.interval</name> + <value>3600</value> + <description> + Time in seconds to sleep for the hourly resolution cluster wide + aggregator. Default is 1 hour. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.minute.interval</name> + <value>120</value> + <description> + Time in seconds to sleep for the minute resolution cluster wide + aggregator. Default resolution is 2 minutes. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.hourly.checkpointCutOffMultiplier</name> + <value>2</value> + <description> + Multiplier value * interval = Max allowed checkpoint lag. Effectively + if aggregator checkpoint is greater than max allowed checkpoint delay, + the checkpoint will be discarded by the aggregator. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.minute.checkpointCutOffMultiplier</name> + <value>2</value> + <description> + Multiplier value * interval = Max allowed checkpoint lag. Effectively + if aggregator checkpoint is greater than max allowed checkpoint delay, + the checkpoint will be discarded by the aggregator. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.hourly.checkpointCutOffMultiplier</name> + <value>2</value> + <description> + Multiplier value * interval = Max allowed checkpoint lag. Effectively + if aggregator checkpoint is greater than max allowed checkpoint delay, + the checkpoint will be discarded by the aggregator. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.minute.checkpointCutOffMultiplier</name> + <value>2</value> + <description> + Multiplier value * interval = Max allowed checkpoint lag. Effectively + if aggregator checkpoint is greater than max allowed checkpoint delay, + the checkpoint will be discarded by the aggregator. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.hourly.disabled</name> + <value>false</value> + <description> + Disable host based hourly aggregations. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.minute.disabled</name> + <value>false</value> + <description> + Disable host based minute aggregations. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.hourly.disabled</name> + <value>false</value> + <description> + Disable cluster based hourly aggregations. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.minute.disabled</name> + <value>false</value> + <description> + Disable cluster based minute aggregations. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.minute.timeslice.interval</name> + <value>15</value> + <description> + Lowest resolution of desired data for cluster level minute aggregates. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.hourly.ttl</name> + <value>2592000</value> + <description> + Host based hourly resolution data purge interval. Default is 30 days. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.minute.ttl</name> + <value>604800</value> + <description> + Host based minute resolution data purge interval. Default is 7 days. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.minute.ttl</name> + <value>2592000</value> + <description> + Cluster wide minute resolution data purge interval. Default is 30 days. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.hourly.ttl</name> + <value>31536000</value> + <description> + Cluster wide hourly resolution data purge interval. Default is 1 year. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.ttl</name> + <value>86400</value> + <description> + 1 minute resolution data purge interval. Default is 1 day. + </description> + </property> + <property> + <name>timeline.metrics.hbase.data.block.encoding</name> + <value>FAST_DIFF</value> + <description> + Codecs are enabled on a table by setting the DATA_BLOCK_ENCODING property. + Default encoding is FAST_DIFF. This can be changed only before creating + tables. + </description> + </property> + <property> + <name>timeline.metrics.hbase.compression.scheme</name> + <value>SNAPPY</value> + <description> + Compression codes need to be installed and available before setting the + scheme. Default compression is SNAPPY. Disable by setting to None. + This can be changed only before creating tables. + </description> + </property> + <property> + <name>timeline.metrics.service.default.result.limit</name> + <value>5760</value> + <description> + Max result limit on number of rows returned. Calculated as follows: + 4 aggregate metrics/min * 60 * 24: Retrieve aggregate data for 1 day. + </description> + </property> + <property> + <name>timeline.metrics.service.checkpointDelay</name> + <value>60</value> + <description> + Time in seconds to sleep on the first run or when the checkpoint is + too old. + </description> + </property> + <property> + <name>timeline.metrics.service.resultset.fetchSize</name> + <value>2000</value> + <description> + JDBC resultset prefect size for aggregator queries. + </description> + </property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMS/0.1.0/metainfo.xml new file mode 100644 index 0000000..51d8177 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/metainfo.xml @@ -0,0 +1,104 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>AMS</name> + <displayName>AMS</displayName> + <version>0.1.0</version> + <comment>Ambari Metrics Service - Monitoring service that provides + storage and retrieval capability for metrics collected from services + & hosts. + </comment> + <components> + <component> + <name>METRIC_COLLECTOR</name> + <displayName>Metric Collector</displayName> + <category>MASTER</category> + <cardinality>1+</cardinality> + <dependencies> + <dependency> + <name>ZOOKEEPER/ZOOKEEPER_SERVER</name> + <scope>cluster</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/metric_collector.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + <component> + <name>METRIC_MONITOR</name> + <displayName>Metric Monitor</displayName> + <category>SLAVE</category> + <cardinality>ALL</cardinality> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + <commandScript> + <script>scripts/metric_monitor.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + </components> + + <osSpecifics> + <osSpecific> + <osFamily>any</osFamily> + <packages> + <package> + <name>ambari-metrics-collector</name> + </package> + <package> + <name>ambari-metrics-monitor</name> + </package> + <package> + <name>hadoop_2_2_*</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <requiredServices> + <service>ZOOKEEPER</service> + </requiredServices> + + <configuration-dependencies> + <config-type>ams-site</config-type> + <config-type>ams-log4j</config-type> + <config-type>ams-hbase-policy</config-type> + <config-type>ams-hbase-site</config-type> + <config-type>ams-hbase-env</config-type> + <config-type>ams-hbase-log4j</config-type> + </configuration-dependencies> + + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/files/alert_ambari_metrics_monitor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/files/alert_ambari_metrics_monitor.py b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/files/alert_ambari_metrics_monitor.py new file mode 100644 index 0000000..5841267 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/files/alert_ambari_metrics_monitor.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import socket + +from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.core.exceptions import ComponentIsNotRunning + +RESULT_CODE_OK = 'OK' +RESULT_CODE_CRITICAL = 'CRITICAL' +RESULT_CODE_UNKNOWN = 'UNKNOWN' + +AMS_MONITOR_PID_PATH = '/var/run/ambari-metrics-monitor/ambari-metrics-monitor.pid' + + +def get_tokens(): + """ + Returns a tuple of tokens in the format {{site/property}} that will be used + to build the dictionary passed into execute + """ + return () + + +def is_monitor_process_live(pid_file): + """ + Gets whether the AMS monitor represented by the specified file is running. + :param pid_file: the PID file of the monitor to check + :return: True if the monitor is running, False otherwise + """ + live = False + + try: + check_process_status(pid_file) + live = True + except ComponentIsNotRunning: + pass + + return live + + +def execute(parameters=None, host_name=None): + """ + Returns a tuple containing the result code and a pre-formatted result label + + Keyword arguments: + parameters (dictionary): a mapping of parameter key to value + host_name (string): the name of this host where the alert is running + """ + + if parameters is None: + return (RESULT_CODE_UNKNOWN, ['There were no parameters supplied to the script.']) + + if host_name is None: + host_name = socket.getfqdn() + + ams_monitor_process_running = is_monitor_process_live(AMS_MONITOR_PID_PATH) + + alert_state = RESULT_CODE_OK if ams_monitor_process_running else RESULT_CODE_CRITICAL + + alert_label = 'Ambari Monitor is running on {0}' if ams_monitor_process_running else 'Ambari Monitor is NOT running on {0}' + alert_label = alert_label.format(host_name) + + return (alert_state, [alert_label]) http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/files/hbaseSmokeVerify.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/files/hbaseSmokeVerify.sh b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/files/hbaseSmokeVerify.sh new file mode 100644 index 0000000..5c320c0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/files/hbaseSmokeVerify.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +conf_dir=$1 +data=$2 +hbase_cmd=$3 +echo "scan 'ambarismoketest'" | $hbase_cmd --config $conf_dir shell > /tmp/hbase_chk_verify +cat /tmp/hbase_chk_verify +echo "Looking for $data" +grep -q $data /tmp/hbase_chk_verify +if [ "$?" -ne 0 ] +then + exit 1 +fi + +grep -q '1 row(s)' /tmp/hbase_chk_verify http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/__init__.py b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/__init__.py new file mode 100644 index 0000000..5561e10 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/__init__.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams.py b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams.py new file mode 100644 index 0000000..1f09d7d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + + +def ams(name=None): + import params + + if name == 'collector': + Directory(params.ams_collector_conf_dir, + owner=params.ams_user, + group=params.user_group, + recursive=True + ) + + XmlConfig("ams-site.xml", + conf_dir=params.ams_collector_conf_dir, + configurations=params.config['configurations']['ams-site'], + configuration_attributes=params.config['configuration_attributes']['ams-site'], + owner=params.ams_user, + group=params.user_group + ) + + XmlConfig( "hbase-site.xml", + conf_dir = params.ams_collector_conf_dir, + configurations = params.config['configurations']['ams-hbase-site'], + configuration_attributes=params.config['configuration_attributes']['ams-hbase-site'], + owner = params.ams_user, + group = params.user_group + ) + + if (params.log4j_props != None): + File(format("{params.ams_collector_conf_dir}/log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.ams_user, + content=params.log4j_props + ) + + File(format("{ams_collector_conf_dir}/ams-env.sh"), + owner=params.ams_user, + content=InlineTemplate(params.ams_env_sh_template) + ) + + pass + + elif name == 'monitor': + Directory(params.ams_monitor_conf_dir, + owner=params.ams_user, + group=params.user_group, + recursive=True + ) + + TemplateConfig( + format("{ams_monitor_conf_dir}/metric_monitor.ini"), + owner=params.ams_user, + group=params.user_group, + template_tag=None + ) + + TemplateConfig( + format("{ams_monitor_conf_dir}/metric_groups.conf"), + owner=params.ams_user, + group=params.user_group, + template_tag=None + ) + + # TODO + pass + + pass http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams_service.py b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams_service.py new file mode 100644 index 0000000..5fdd0f7 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams_service.py @@ -0,0 +1,72 @@ +# !/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + + +def ams_service(name='collector', action='start'): + import params + + if name == 'collector': + cmd = format("{ams_collector_script} --config {ams_collector_conf_dir}") + pid_file = format("{ams_collector_pid_dir}/ambari-metrics-collector.pid") + no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1") + + if action == 'start': + daemon_cmd = format("{cmd} start") + Execute(daemon_cmd, + not_if=no_op_test, + user=params.ams_user + ) + + pass + elif action == 'stop': + daemon_cmd = format("{cmd} stop") + Execute(daemon_cmd, + user=params.ams_user + ) + + pass + pass + elif name == 'monitor': + cmd = format("{ams_monitor_script} --config {ams_monitor_conf_dir}") + pid_file = format("{ams_monitor_pid_dir}/ambari-metrics-monitor.pid") + no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1") + + if action == 'start': + daemon_cmd = format("{cmd} start") + Execute(daemon_cmd, + user=params.ams_user + ) + + pass + elif action == 'stop': + + daemon_cmd = format("{cmd} stop") + Execute(daemon_cmd, + user=params.ams_user + ) + + pass + pass + + #TODO + pass + pass http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/functions.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/functions.py b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/functions.py new file mode 100644 index 0000000..a5107d6 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/functions.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os +import re +import math +import datetime + +from resource_management.core.shell import checked_call + +def calc_xmn_from_xms(heapsize_str, xmn_percent, xmn_max): + """ + @param heapsize_str: str (e.g '1000m') + @param xmn_percent: float (e.g 0.2) + @param xmn_max: integer (e.g 512) + """ + heapsize = int(re.search('\d+', str(heapsize_str)).group(0)) + heapsize_unit = re.search('\D+', str(heapsize_str)).group(0) + xmn_val = int(math.floor(heapsize*xmn_percent)) + xmn_val -= xmn_val % 8 + + result_xmn_val = xmn_max if xmn_val > xmn_max else xmn_val + return str(result_xmn_val) + heapsize_unit http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/hbase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/hbase.py b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/hbase.py new file mode 100644 index 0000000..d861338 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/hbase.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import os + +from resource_management import * +import sys + +def hbase(name=None # 'master' or 'regionserver' or 'client' + ): + import params + + Directory(params.hbase_conf_dir, + owner = params.hbase_user, + group = params.user_group, + recursive = True + ) + + Directory (params.hbase_tmp_dir, + owner = params.hbase_user, + recursive = True + ) + + Directory (os.path.join(params.local_dir, "jars"), + owner = params.hbase_user, + group = params.user_group, + mode=0775, + recursive = True + ) + + XmlConfig("hbase-site.xml", + conf_dir = params.hbase_conf_dir, + configurations = params.config['configurations']['ams-hbase-site'], + configuration_attributes=params.config['configuration_attributes']['ams-hbase-site'], + owner = params.hbase_user, + group = params.user_group + ) + + if 'ams-hbase-policy' in params.config['configurations']: + XmlConfig("hbase-policy.xml", + conf_dir = params.hbase_conf_dir, + configurations = params.config['configurations']['ams-hbase-policy'], + configuration_attributes=params.config['configuration_attributes']['ams-hbase-policy'], + owner = params.hbase_user, + group = params.user_group + ) + # Manually overriding ownership of file installed by hadoop package + else: + File( format("{params.hbase_conf_dir}/hbase-policy.xml"), + owner = params.hbase_user, + group = params.user_group + ) + + File(format("{hbase_conf_dir}/hbase-env.sh"), + owner = params.hbase_user, + content=InlineTemplate(params.hbase_env_sh_template) + ) + + # Metrics properties + File(os.path.join(params.hbase_conf_dir, "hadoop-metrics2-hbase.properties"), + owner = params.hbase_user, + group = params.user_group, + content=Template("hadoop-metrics2-hbase.properties.j2") + ) + + # hbase_TemplateConfig( params.metric_prop_file_name, + # tag = 'GANGLIA-MASTER' if name == 'master' else 'GANGLIA-RS' + # ) + + hbase_TemplateConfig('regionservers') + + if params.security_enabled: + hbase_TemplateConfig( format("hbase_{name}_jaas.conf")) + + if name != "client": + Directory( params.hbase_pid_dir, + owner = params.hbase_user, + recursive = True + ) + + Directory (params.hbase_log_dir, + owner = params.hbase_user, + recursive = True + ) + + if (params.hbase_log4j_props != None): + File(format("{params.hbase_conf_dir}/log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.hbase_user, + content=params.hbase_log4j_props + ) + elif (os.path.exists(format("{params.hbase_conf_dir}/log4j.properties"))): + File(format("{params.hbase_conf_dir}/log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.hbase_user + ) + + +def hbase_TemplateConfig(name, tag=None): + import params + + TemplateConfig( format("{hbase_conf_dir}/{name}"), + owner = params.hbase_user, + template_tag = tag + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/hbase_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/hbase_master.py b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/hbase_master.py new file mode 100644 index 0000000..a26254d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/hbase_master.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * + +from hbase import hbase +from hbase_service import hbase_service +from hbase_decommission import hbase_decommission + + +class HbaseMaster(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + + hbase(name='master') + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # for security + + hbase_service( 'master', + action = 'start' + ) + + def stop(self, env): + import params + env.set_params(params) + + hbase_service( 'master', + action = 'stop' + ) + + def status(self, env): + import status_params + env.set_params(status_params) + pid_file = format("{pid_dir}/hbase-{hbase_user}-master.pid") + check_process_status(pid_file) + + def decommission(self, env): + import params + env.set_params(params) + + hbase_decommission(env) + + +if __name__ == "__main__": + HbaseMaster().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/hbase_regionserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/hbase_regionserver.py b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/hbase_regionserver.py new file mode 100644 index 0000000..8d66dcc --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/hbase_regionserver.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * + +from hbase import hbase +from hbase_service import hbase_service + + +class HbaseRegionServer(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + + hbase(name='regionserver') + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # for security + + hbase_service( 'regionserver', + action = 'start' + ) + + def stop(self, env): + import params + env.set_params(params) + + hbase_service( 'regionserver', + action = 'stop' + ) + + def status(self, env): + import status_params + env.set_params(status_params) + pid_file = format("{pid_dir}/hbase-{hbase_user}-regionserver.pid") + check_process_status(pid_file) + + def decommission(self, env): + print "Decommission not yet implemented!" + + +if __name__ == "__main__": + HbaseRegionServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/hbase_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/hbase_service.py b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/hbase_service.py new file mode 100644 index 0000000..f2d20d6 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/hbase_service.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +def hbase_service( + name, + action = 'start'): # 'start' or 'stop' or 'status' + + import params + + role = name + cmd = format("{daemon_script} --config {hbase_conf_dir}") + pid_file = format("{pid_dir}/hbase-{hbase_user}-{role}.pid") + no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1") + + if action == 'start': + daemon_cmd = format("{cmd} start {role}") + + Execute ( daemon_cmd, + not_if = no_op_test, + user = params.hbase_user + ) + elif action == 'stop': + daemon_cmd = format("{cmd} stop {role}") + + Execute ( daemon_cmd, + user = params.hbase_user, + # BUGFIX: hbase regionserver sometimes hangs when nn is in safemode + timeout = 30, + on_timeout = format("{no_op_test} && kill -9 `cat {pid_file}`") + ) + + Execute (format("rm -f {pid_file}")) http://git-wip-us.apache.org/repos/asf/ambari/blob/ef7fe6b5/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/metric_collector.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/metric_collector.py b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/metric_collector.py new file mode 100644 index 0000000..ae71aa1 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/metric_collector.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from ams import ams +from ams_service import ams_service +from hbase import hbase + +class AmsCollector(Script): + def install(self, env): + self.install_packages(env) + + + def configure(self, env): + import params + env.set_params(params) + hbase('master') + hbase('regionserver') + ams(name='collector') + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # for security + + ams_service( 'collector', + action = 'start' + ) + + def stop(self, env): + import params + env.set_params(params) + + ams_service( 'collector', + action = 'stop' + ) + + def status(self, env): + import status_params + env.set_params(status_params) + pid_file = format("{ams_collector_pid_dir}/ambari-metrics-collector.pid") + check_process_status(pid_file) + + +if __name__ == "__main__": + AmsCollector().execute()
