Repository: ambari Updated Branches: refs/heads/branch-metrics-dev a3ed7a3fd -> dd066b945
AMBARI-7681. Add Metrics Service to common services stack. Additional fixes. (mpapirkovskyy) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dd066b94 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dd066b94 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dd066b94 Branch: refs/heads/branch-metrics-dev Commit: dd066b945490cb56fcdbf01782970056af9885c3 Parents: a3ed7a3 Author: Myroslav Papirkovskyy <mpapyrkovs...@hortonworks.com> Authored: Thu Nov 20 10:07:23 2014 +0200 Committer: Myroslav Papirkovskyy <mpapyrkovs...@hortonworks.com> Committed: Thu Nov 20 11:53:31 2014 +0200 ---------------------------------------------------------------------- .../metrics2/sink/timeline/TimelineMetric.java | 8 ++++--- .../conf/unix/ambari-metrics-collector | 2 +- .../pom.xml | 21 +++++++++++++++++-- .../metrics/timeline/PhoenixTransactSQL.java | 14 ++++++++++++- .../main/python/core/application_metric_map.py | 8 +++++-- .../src/main/python/core/controller.py | 2 +- .../src/main/python/core/emitter.py | 4 ++-- .../src/main/python/core/host_info.py | 9 +++++--- ambari-metrics/pom.xml | 6 +++--- .../templates/hadoop-metrics2.properties.j2 | 5 +++++ .../2.2/services/AMS/configuration/ams-env.xml | 2 -- .../AMS/package/templates/metric_groups.conf.j2 | 22 ++------------------ 12 files changed, 63 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java index 1b35d92..68b4be8 100644 --- a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java @@ -127,7 +127,8 @@ public class TimelineMetric implements Comparable<TimelineMetric> { if (!metricName.equals(metric.metricName)) return false; if (hostName != null ? !hostName.equals(metric.hostName) : metric.hostName != null) return false; - if (!appId.equals(metric.appId)) return false; + if (appId != null ? !appId.equals(metric.appId) : metric.appId != null) + return false; if (instanceId != null ? !instanceId.equals(metric.instanceId) : metric.instanceId != null) return false; if (timestamp != metric.timestamp) return false; @@ -140,7 +141,8 @@ public class TimelineMetric implements Comparable<TimelineMetric> { if (!metricName.equals(metric.metricName)) return false; if (hostName != null ? !hostName.equals(metric.hostName) : metric.hostName != null) return false; - if (!appId.equals(metric.appId)) return false; + if (appId != null ? !appId.equals(metric.appId) : metric.appId != null) + return false; if (instanceId != null ? !instanceId.equals(metric.instanceId) : metric.instanceId != null) return false; @@ -150,7 +152,7 @@ public class TimelineMetric implements Comparable<TimelineMetric> { @Override public int hashCode() { int result = metricName.hashCode(); - result = 31 * result + appId.hashCode(); + result = 31 * result + (appId != null ? appId.hashCode() : 0); result = 31 * result + (instanceId != null ? instanceId.hashCode() : 0); result = 31 * result + (hostName != null ? hostName.hashCode() : 0); result = 31 * result + (int) (timestamp ^ (timestamp >>> 32)); http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/ambari-metrics-hadoop-timelineservice/conf/unix/ambari-metrics-collector ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/conf/unix/ambari-metrics-collector b/ambari-metrics/ambari-metrics-hadoop-timelineservice/conf/unix/ambari-metrics-collector index eed1c9c..b6e17cf 100644 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/conf/unix/ambari-metrics-collector +++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/conf/unix/ambari-metrics-collector @@ -200,7 +200,7 @@ case "$1" in rm -f "${PIDFILE}" >/dev/null 2>&1 fi - nohup "${JAVA}" "-cp" "/usr/lib/ambari-metrics-collector/*:${COLLECTOR_CONF_DIR}" "-Dproc_${DAEMON_NAME}" "${CLASS}" "$@" > $OUTFILE 2>&1 & + nohup "${JAVA}" "-cp" "/usr/lib/ambari-metrics-collector/*:${COLLECTOR_CONF_DIR}" "-Djava.net.preferIPv4Stack=true" "-Dproc_${DAEMON_NAME}" "${CLASS}" "$@" > $OUTFILE 2>&1 & PID=$! write_pidfile "${PIDFILE}" sleep 2 http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/ambari-metrics-hadoop-timelineservice/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/pom.xml b/ambari-metrics/ambari-metrics-hadoop-timelineservice/pom.xml index 06f84be..7efdb6b 100644 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/pom.xml +++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/pom.xml @@ -56,6 +56,8 @@ </goals> <configuration> <outputDirectory>${project.build.directory}/lib</outputDirectory> + <includeScope>compile</includeScope> + <excludeScope>test</excludeScope> </configuration> </execution> </executions> @@ -150,7 +152,7 @@ <source> <location>target/embedded/${hbase.folder}</location> <excludes> - <exclude>target/embedded/${hbase.folder}/bin/*</exclude> + <exclude>bin/*</exclude> </excludes> </source> </sources> @@ -220,7 +222,17 @@ <dependency> <groupId>org.apache.phoenix</groupId> <artifactId>phoenix-core</artifactId> - <version>4.2.0.2.2.0.0-2038</version> + <version>4.2.0.2.2.0.0-2041</version> + <exclusions> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-annotations</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> @@ -324,6 +336,11 @@ <artifactId>jersey-guice</artifactId> <version>1.11</version> </dependency> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-server</artifactId> + <version>1.11</version> + </dependency> <!-- 'mvn dependency:analyze' fails to detect use of this dependency --> <dependency> <groupId>org.apache.hadoop</groupId> http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixTransactSQL.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixTransactSQL.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixTransactSQL.java index ed8f978..77bd0f9 100644 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixTransactSQL.java +++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixTransactSQL.java @@ -242,23 +242,34 @@ public class PhoenixTransactSQL { int pos = 1; if (condition.getMetricNames() != null) { for (; pos <= condition.getMetricNames().size(); pos++) { + LOG.debug("Setting pos: " + pos + ", value = " + condition.getMetricNames().get(pos - 1)); stmt.setString(pos, condition.getMetricNames().get(pos - 1)); } } if (condition.getHostname() != null) { + LOG.debug("Setting pos: " + pos + ", value: " + condition.getHostname()); stmt.setString(pos++, condition.getHostname()); } // TODO: Upper case all strings on POST if (condition.getAppId() != null) { - stmt.setString(pos++, condition.getAppId().toLowerCase()); + // TODO: fix case of appId coming from host metrics + String appId = condition.getAppId(); + if (!condition.getAppId().equals("HOST")) { + appId = appId.toLowerCase(); + } + LOG.debug("Setting pos: " + pos + ", value: " + appId); + stmt.setString(pos++, appId); } if (condition.getInstanceId() != null) { + LOG.debug("Setting pos: " + pos + ", value: " + condition.getInstanceId()); stmt.setString(pos++, condition.getInstanceId()); } if (condition.getStartTime() != null) { + LOG.debug("Setting pos: " + pos + ", value: " + condition.getStartTime()); stmt.setLong(pos++, condition.getStartTime()); } if (condition.getEndTime() != null) { + LOG.debug("Setting pos: " + pos + ", value: " + condition.getEndTime()); stmt.setLong(pos, condition.getEndTime()); } if (condition.getFetchSize() != null) { @@ -268,6 +279,7 @@ public class PhoenixTransactSQL { return stmt; } + public static PreparedStatement prepareGetAggregateSqlStmt( Connection connection, Condition condition) throws SQLException { http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py index 8c32661..0b3f1dd 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py @@ -95,8 +95,8 @@ class ApplicationMetricMap: timeline_metric = { "hostname" : self.hostname, "metricname" : metricId, - "appid" : self.get_app_id(appId), - "instanceid" : self.get_instance_id(appId), + "appid" : "HOST", + "instanceid" : "", "starttime" : self.get_start_time(appId, metricId), "metrics" : metricData } @@ -125,3 +125,7 @@ class ApplicationMetricMap: parts = app_id.split("_") return parts[1] if len(parts) > 1 else '' pass + + def clear(self): + self.app_metric_map.clear() + pass http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py index 3d1f487..d5299d1 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py @@ -70,7 +70,7 @@ class Controller(threading.Thread): def enqueque_events(self): # Queue events for up to a minute for event in self.events_cache: - t = Timer(event.get_collect_interval(), self.metric_collector.process_event(event)) + t = Timer(event.get_collect_interval(), self.metric_collector.process_event, args=(event,)) t.start() pass http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py index a107374..7045c2e 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py @@ -55,7 +55,7 @@ class Emitter(threading.Thread): pass response = self.push_metrics(json_data) - if response and response.getcode() == '200': + if response and response.getcode() == 200: retry_count = MAX_RETRY_COUNT self.application_metric_map.clear() self.application_metric_map.release_lock() @@ -83,7 +83,7 @@ class Emitter(threading.Thread): response = urllib2.urlopen(req, timeout=int(self.send_interval - 10)) if response: logger.debug("POST response from server: retcode = {0}".format(response.getcode())) - logger.debug(response.read()) + logger.debug(str(response.read())) pass return response http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py index 4b39119..7ca11b6 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py @@ -74,7 +74,8 @@ class HostInfo(): 'mem_cached' : mem_stats.cached if hasattr(mem_stats, 'cached') else '', 'swap_free' : swap_stats.free if hasattr(mem_stats, 'free') else '', 'disk_free' : disk_usage.get("disk_free"), - 'part_max_used' : disk_usage.get("max_part_used")[0], + # todo: cannot send string + #'part_max_used' : disk_usage.get("max_part_used")[0], 'disk_total' : disk_usage.get("disk_total") } pass @@ -127,8 +128,10 @@ class HostInfo(): return { "disk_total" : bytes2human(combined_disk_total), "disk_used" : bytes2human(combined_disk_used), "disk_free" : bytes2human(combined_disk_free), - "disk_percent" : bytes2human(combined_disk_percent), - "max_part_used" : max_percent_usage } + "disk_percent" : bytes2human(combined_disk_percent) + # todo: cannot send string + #"max_part_used" : max_percent_usage } + } pass def get_host_static_info(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/pom.xml b/ambari-metrics/pom.xml index 67f5892..2ee4c1e 100644 --- a/ambari-metrics/pom.xml +++ b/ambari-metrics/pom.xml @@ -35,8 +35,8 @@ <python.ver>python >= 2.6</python.ver> <deb.python.ver>python (>= 2.6)</deb.python.ver> <!--TODO change to HDP URL--> - <hbase.tar>http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/updates/2.2.0.0/tars/hbase-0.98.4.2.2.0.0-1995-hadoop2.tar.gz</hbase.tar> - <hbase.folder>hbase-0.98.4.2.2.0.0-1995-hadoop2</hbase.folder> + <hbase.tar>http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/updates/2.2.0.0/tars/hbase-0.98.4.2.2.0.0-2041-hadoop2.tar.gz</hbase.tar> + <hbase.folder>hbase-0.98.4.2.2.0.0-2041-hadoop2</hbase.folder> </properties> <repositories> <repository> @@ -196,4 +196,4 @@ </profiles> -</project> \ No newline at end of file +</project> http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 index d1b82b6..4b35ff4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 @@ -66,6 +66,11 @@ resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue {% if has_metric_collector %} +*.period=60 + +*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +*.sink.timeline.period=10 + datanode.sink.timeline.collector={{metric_collector_host}}:8188 namenode.sink.timeline.collector={{metric_collector_host}}:8188 resourcemanager.sink.timeline.collector={{metric_collector_host}}:8188 http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-env.xml index 746db5c..fda1df0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-env.xml @@ -34,8 +34,6 @@ # The java implementation to use. Java 1.6 required. export JAVA_HOME={{java64_home}} - export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:{{hadoop_native_lib}} - #TODO </value> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2 b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2 index f86343f..aa03d19 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2 @@ -18,26 +18,8 @@ { "host_metric_groups": { - "cpu_info": { - "collect_every": "15", - "metrics": [ - { - "name": "cpu_user", - "value_threshold": "1.0" - } - ] - }, - "disk_info": { - "collect_every": "30", - "metrics": [ - { - "name": "disk_free", - "value_threshold": "5.0" - } - ] - }, - "network_info": { - "collect_every": "20", + "all": { + "collect_every": "10", "metrics": [ { "name": "bytes_out",
