CHUKWA-758. Updated default dashboard to include system metrics. (Eric Yang)
Project: http://git-wip-us.apache.org/repos/asf/chukwa/repo Commit: http://git-wip-us.apache.org/repos/asf/chukwa/commit/d8abe979 Tree: http://git-wip-us.apache.org/repos/asf/chukwa/tree/d8abe979 Diff: http://git-wip-us.apache.org/repos/asf/chukwa/diff/d8abe979 Branch: refs/heads/master Commit: d8abe979aaa2f107240ea5aeaeca9207eff65332 Parents: 0cad3aa Author: Eric Yang <[email protected]> Authored: Thu Jun 18 20:38:30 2015 -0700 Committer: Eric Yang <[email protected]> Committed: Thu Jun 18 20:38:30 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../writer/hbase/HBaseWriter.java | 2 +- .../chukwa/datastore/ChukwaHBaseStore.java | 138 ++++++++++++++++--- .../hbase/HadoopMetricsProcessor.java | 74 +++++----- src/main/web/hicc/home/css/component.css | 1 + src/main/web/hicc/home/index.html | 2 +- 6 files changed, 161 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/chukwa/blob/d8abe979/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index ef7af42..adcd624 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -42,6 +42,8 @@ Trunk (unreleased changes) BUGS + CHUKWA-758. Updated default dashboard to include system metrics. (Eric Yang) + CHUKWA-743. Revised pid locking using standard Hadoop approach. (Eric Yang) CHUKWA-742. Updated license header for StatusCheckerException.java and exclude http://git-wip-us.apache.org/repos/asf/chukwa/blob/d8abe979/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java index e5a8083..0718feb 100644 --- a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java +++ b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java @@ -98,7 +98,7 @@ public class HBaseWriter extends PipelineableWriter { this.statTimer = new Timer(); this.defaultProcessor = conf.get( "chukwa.demux.mapper.default.processor", - "org.apache.hadoop.chukwa.extraction.demux.processor.mapper.DefaultProcessor"); + "org.apache.hadoop.chukwa.extraction.hbase.DefaultProcessor"); log.info("hbase.zookeeper.quorum: " + hconf.get(HConstants.ZOOKEEPER_QUORUM) + ":" + hconf.get(HConstants.ZOOKEEPER_CLIENT_PORT)); if (reportStats) { statTimer.schedule(new StatReportingTask(), 1000, 10 * 1000); http://git-wip-us.apache.org/repos/asf/chukwa/blob/d8abe979/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java b/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java index 66553c4..d75a350 100644 --- a/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java +++ b/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java @@ -20,6 +20,7 @@ package org.apache.hadoop.chukwa.datastore; import java.io.IOException; import java.net.InetAddress; import java.net.URI; +import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Calendar; @@ -240,7 +241,7 @@ public class ChukwaHBaseStore { Result result = it.next(); for (Cell cell : result.rawCells()) { JSONObject json = (JSONObject) JSONValue.parse(new String(cell.getValue(), "UTF-8")); - if (json.get("type").equals("source")) { + if (json!=null && json.get("type")!=null && json.get("type").equals("source")) { pk.add(new String(cell.getQualifier(), "UTF-8")); } } @@ -418,6 +419,30 @@ public class ChukwaHBaseStore { } /** + * Create a chart in HBase by specifying parameters. + * @throws URISyntaxException + */ + public static synchronized String createChart(String id, String yunitType, + String title, String[] metrics, String source) throws URISyntaxException { + Chart chart = new Chart(id); + chart.setYUnitType(yunitType); + chart.setTitle(title); + ArrayList<SeriesMetaData> series = new ArrayList<SeriesMetaData>(); + for(String metric : metrics) { + SeriesMetaData s = new SeriesMetaData(); + s.setLabel(metric + "/" + source); + s.setUrl(new URI("/hicc/v1/metrics/series/" + metric + "/" + + source)); + LineOptions l = new LineOptions(); + s.setLineOptions(l); + series.add(s); + } + chart.SetSeries(series); + return createChart(chart); + + } + + /** * Create a chart in HBase. * * @param chart @@ -719,13 +744,27 @@ public class ChukwaHBaseStore { public static void populateDefaults() { try { String hostname = InetAddress.getLocalHost().getHostName(); + // Populate example chart widgets + String[] metrics = { "SystemMetrics.LoadAverage.1" }; + createChart("1", "", "System Load Average", metrics, hostname); + String[] cpuMetrics = { "SystemMetrics.cpu.combined", "SystemMetrics.cpu.sys", "SystemMetrics.cpu.user" }; + createChart("2", "%", "CPU Utilization", cpuMetrics, hostname); + String[] memMetrics = { "SystemMetrics.memory.FreePercent", "SystemMetrics.memory.UsedPercent"}; + createChart("3", "%", "Memory Utilization", memMetrics, hostname); + String[] diskMetrics = { "SystemMetrics.disk.ReadBytes", "SystemMetrics.disk.WriteBytes" }; + createChart("4", "bytes-decimal", "Disk Utilization", diskMetrics, hostname); + String[] netMetrics = { "SystemMetrics.network.TxBytes", "SystemMetrics.network.RxBytes" }; + createChart("5", "bytes", "Network Utilization", netMetrics, hostname); + String[] swapMetrics = { "SystemMetrics.swap.Total", "SystemMetrics.swap.Used", "SystemMetrics.swap.Free" }; + createChart("6", "", "Swap Utilization", swapMetrics, hostname); + // Populate default widgets Widget widget = new Widget(); widget.setTitle("System Load Average"); widget.setSrc(new URI("/hicc/v1/chart/draw/1")); widget.setCol(1); widget.setRow(1); - widget.setSize_x(7); + widget.setSize_x(2); widget.setSize_y(1); createWidget(widget); @@ -733,32 +772,89 @@ public class ChukwaHBaseStore { widget = new Widget(); widget.setTitle("CPU Heatmap"); widget.setSrc(new URI("/hicc/jsp/heatmap.jsp")); + widget.setCol(3); + widget.setRow(1); + widget.setSize_x(5); + widget.setSize_y(1); + createWidget(widget); + + Dashboard dashboard = new Dashboard(); + // Log Search widget + widget = new Widget(); + widget.setTitle("Log Search"); + widget.setSrc(new URI("/hicc/ajax-solr/chukwa")); + widget.setCol(1); + widget.setRow(4); + widget.setSize_x(7); + widget.setSize_y(4); + createWidget(widget); + dashboard.add(widget); + updateDashboard("default", "", dashboard); + + // Populate system dashboards + dashboard = new Dashboard(); + widget = new Widget(); + widget.setTitle("CPU Utilization"); + widget.setSrc(new URI("/hicc/v1/chart/draw/2")); widget.setCol(1); widget.setRow(1); - widget.setSize_x(4); - widget.setSize_y(2); + widget.setSize_x(3); + widget.setSize_y(1); createWidget(widget); + dashboard.add(widget); - // Populate example chart widgets - Chart chart = new Chart("1"); - chart.setYUnitType(""); - chart.setTitle("Load Average"); - ArrayList<SeriesMetaData> series = new ArrayList<SeriesMetaData>(); + widget = new Widget(); + widget.setTitle("Memory Utilization"); + widget.setSrc(new URI("/hicc/v1/chart/draw/3")); + widget.setCol(4); + widget.setRow(1); + widget.setSize_x(3); + widget.setSize_y(1); + createWidget(widget); + dashboard.add(widget); - SeriesMetaData s = new SeriesMetaData(); - s.setLabel("SystemMetrics.LoadAverage.1/" + hostname); - s.setUrl(new URI("/hicc/v1/metrics/series/SystemMetrics.LoadAverage.1/" - + hostname)); - LineOptions l = new LineOptions(); - s.setLineOptions(l); - series.add(s); + widget = new Widget(); + widget.setTitle("Disk Utilization"); + widget.setSrc(new URI("/hicc/v1/chart/draw/4")); + widget.setCol(1); + widget.setRow(2); + widget.setSize_x(3); + widget.setSize_y(1); + createWidget(widget); + dashboard.add(widget); - chart.SetSeries(series); - createChart(chart); - - Dashboard dashboard = new Dashboard(); + widget = new Widget(); + widget.setTitle("Network Utilization"); + widget.setSrc(new URI("/hicc/v1/chart/draw/5")); + widget.setCol(4); + widget.setRow(2); + widget.setSize_x(3); + widget.setSize_y(1); + createWidget(widget); dashboard.add(widget); - updateDashboard("default", "", dashboard); + + widget = new Widget(); + widget.setTitle("Swap Utilization"); + widget.setSrc(new URI("/hicc/v1/chart/draw/6")); + widget.setCol(1); + widget.setRow(3); + widget.setSize_x(3); + widget.setSize_y(1); + createWidget(widget); + dashboard.add(widget); + + widget = new Widget(); + widget.setTitle("System Load Average"); + widget.setSrc(new URI("/hicc/v1/chart/draw/1")); + widget.setCol(4); + widget.setRow(3); + widget.setSize_x(3); + widget.setSize_y(1); + createWidget(widget); + dashboard.add(widget); + + updateDashboard("system", "", dashboard); + } catch (Throwable ex) { LOG.error(ExceptionUtil.getStackTrace(ex)); } http://git-wip-us.apache.org/repos/asf/chukwa/blob/d8abe979/src/main/java/org/apache/hadoop/chukwa/extraction/hbase/HadoopMetricsProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/hadoop/chukwa/extraction/hbase/HadoopMetricsProcessor.java b/src/main/java/org/apache/hadoop/chukwa/extraction/hbase/HadoopMetricsProcessor.java index 3afd71a..19df607 100644 --- a/src/main/java/org/apache/hadoop/chukwa/extraction/hbase/HadoopMetricsProcessor.java +++ b/src/main/java/org/apache/hadoop/chukwa/extraction/hbase/HadoopMetricsProcessor.java @@ -35,6 +35,8 @@ public class HadoopMetricsProcessor extends AbstractProcessor { static final String timestampField = "timestamp"; static final String contextNameField = "contextName"; static final String recordNameField = "recordName"; + static final String hostName = "Hostname"; + static final String processName = "ProcessName"; static final byte[] cf = "t".getBytes(); public HadoopMetricsProcessor() throws NoSuchAlgorithmException { @@ -43,44 +45,46 @@ public class HadoopMetricsProcessor extends AbstractProcessor { @Override protected void parse(byte[] recordEntry) throws Throwable { try { - String body = new String(recordEntry); - int start = body.indexOf('{'); - JSONObject json = (JSONObject) JSONValue.parse(body.substring(start)); + String body = new String(recordEntry); + int start = body.indexOf('{'); + JSONObject json = (JSONObject) JSONValue.parse(body.substring(start)); - time = ((Long) json.get(timestampField)).longValue(); - String contextName = (String) json.get(contextNameField); - String recordName = (String) json.get(recordNameField); - byte[] timeInBytes = ByteBuffer.allocate(8).putLong(time).array(); - - @SuppressWarnings("unchecked") - Iterator<String> ki = json.keySet().iterator(); - while (ki.hasNext()) { - String keyName = ki.next(); - if (timestampField.intern() == keyName.intern()) { - continue; - } else if (contextNameField.intern() == keyName.intern()) { - continue; - } else if (recordNameField.intern() == keyName.intern()) { - continue; - } else { - if(json.get(keyName)!=null) { - byte[] v = json.get(keyName).toString().getBytes(); - String primaryKey = new StringBuilder(contextName).append("."). - append(recordName).append("."). - append(keyName).toString(); - byte[] rowKey = HBaseUtil.buildKey(time, primaryKey, chunk.getSource()); - Put r = new Put(rowKey); - r.add(cf, timeInBytes, time, v); - output.add(r); - } + time = ((Long) json.get(timestampField)).longValue(); + String contextName = (String) json.get(contextNameField); + String recordName = (String) json.get(recordNameField); + String src = (String) json.get(hostName); + if(json.get(processName)!=null) { + src = new StringBuilder(src).append(":").append(json.get(processName)).toString(); + } + @SuppressWarnings("unchecked") + Iterator<String> ki = json.keySet().iterator(); + while (ki.hasNext()) { + String keyName = ki.next(); + if (timestampField.intern() == keyName.intern()) { + continue; + } else if (contextNameField.intern() == keyName.intern()) { + continue; + } else if (recordNameField.intern() == keyName.intern()) { + continue; + } else if (hostName.intern() == keyName.intern()) { + continue; + } else if (processName.intern() == keyName.intern()) { + continue; + } else { + if (json.get(keyName) != null) { + String v = json.get(keyName).toString(); + String primaryKey = new StringBuilder(contextName).append(".") + .append(recordName).append(".").append(keyName).toString(); + addRecord(time, primaryKey, src, v.getBytes(), output); } } - - } catch (Exception e) { - LOG.warn("Wrong format in HadoopMetricsProcessor [" + recordEntry + "]", - e); - throw e; - } + } + + } catch (Exception e) { + LOG.warn("Wrong format in HadoopMetricsProcessor [" + recordEntry + "]", + e); + throw e; + } } } http://git-wip-us.apache.org/repos/asf/chukwa/blob/d8abe979/src/main/web/hicc/home/css/component.css ---------------------------------------------------------------------- diff --git a/src/main/web/hicc/home/css/component.css b/src/main/web/hicc/home/css/component.css index 98c38bc..9828873 100755 --- a/src/main/web/hicc/home/css/component.css +++ b/src/main/web/hicc/home/css/component.css @@ -602,6 +602,7 @@ input.gn-search:focus { left: 0; z-index: 99; background: transparent; + display: none; } .frame { http://git-wip-us.apache.org/repos/asf/chukwa/blob/d8abe979/src/main/web/hicc/home/index.html ---------------------------------------------------------------------- diff --git a/src/main/web/hicc/home/index.html b/src/main/web/hicc/home/index.html index 22a6e52..78c117c 100755 --- a/src/main/web/hicc/home/index.html +++ b/src/main/web/hicc/home/index.html @@ -33,7 +33,7 @@ <script src="js/jquery.gridster.js" type="text/javascript" charset="utf-8"></script> <script src="js/typeahead.bundle.js" type="text/javascript"></script> </head> - <body> + <body onunload=""> <div id="glass" class="glass" onclick="toggleGlass()"></div> <div class="frame"> <ul><li><iframe id="frame"></iframe></li></ul>
