CHUKWA-769. Updated Chukwa to current HBase API. (Eric Yang)
Project: http://git-wip-us.apache.org/repos/asf/chukwa/repo Commit: http://git-wip-us.apache.org/repos/asf/chukwa/commit/384d3449 Tree: http://git-wip-us.apache.org/repos/asf/chukwa/tree/384d3449 Diff: http://git-wip-us.apache.org/repos/asf/chukwa/diff/384d3449 Branch: refs/heads/master Commit: 384d3449ce18694ac58ab77754f18ec87492ae00 Parents: f3dfc94 Author: Eric Yang <[email protected]> Authored: Thu Jun 25 12:33:44 2015 -0700 Committer: Eric Yang <[email protected]> Committed: Thu Jun 25 12:33:44 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../writer/hbase/HBaseWriter.java | 3 -- .../datacollection/writer/hbase/Reporter.java | 53 ++++++++++++-------- .../chukwa/datastore/ChukwaHBaseStore.java | 51 +++++++++---------- .../hadoop/chukwa/hicc/HiccWebServer.java | 2 - .../chukwa/hicc/rest/MetricsController.java | 42 ++++++++-------- 6 files changed, 79 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/chukwa/blob/384d3449/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 633a7c9..26999a0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -18,6 +18,8 @@ Trunk (unreleased changes) IMPROVEMENTS + CHUKWA-769. Updated Chukwa to current HBase API. (Eric Yang) + CHUKWA-767. Implemented low pass filter for Charting REST API. (Eric Yang) CHUKWA-765. Minor stylesheets clean up. (Eric Yang) http://git-wip-us.apache.org/repos/asf/chukwa/blob/384d3449/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java index 0718feb..621f22e 100644 --- a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java +++ b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java @@ -40,9 +40,6 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.HConnection; -import org.apache.hadoop.hbase.client.HConnectionManager; -import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; import org.apache.log4j.Logger; http://git-wip-us.apache.org/repos/asf/chukwa/blob/384d3449/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/Reporter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/Reporter.java b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/Reporter.java index e6e1f1b..30442e2 100644 --- a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/Reporter.java +++ b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/Reporter.java @@ -19,15 +19,20 @@ package org.apache.hadoop.chukwa.datacollection.writer.hbase; import java.io.UnsupportedEncodingException; +import java.lang.reflect.Type; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.hadoop.hbase.client.Put; -import org.json.simple.JSONObject; import org.mortbay.log.Log; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + public class Reporter { private ArrayList<Put> meta = new ArrayList<Put>(); private MessageDigest md5 = null; @@ -38,33 +43,38 @@ public class Reporter { public void putSource(String type, String source) { byte[] value = getHash(source); - JSONObject json = new JSONObject(); + String buffer; try { - json.put("sig", new String(value, "UTF-8")); - json.put("type", "source"); - } catch (UnsupportedEncodingException e) { + Type metaType = new TypeToken<Map<String, String>>(){}.getType(); + Map<String, String> meta = new HashMap<String, String>(); + meta.put("sig", new String(value, "UTF-8")); + meta.put("type", "source"); + Gson gson = new Gson(); + buffer = gson.toJson(meta, metaType); + put(type.getBytes(), source.getBytes(), buffer.toString().getBytes()); + } catch (Exception e) { Log.warn("Error encoding metadata."); Log.warn(e); } - put(type.getBytes(), source.getBytes(), json.toString().getBytes()); - } public void putMetric(String type, String metric) { String buf = new StringBuilder(type).append(".").append(metric).toString(); byte[] pk = getHash(buf); - - JSONObject json = new JSONObject(); + String buffer; try { - json.put("sig", new String(pk, "UTF-8")); - json.put("type", "metric"); - } catch (UnsupportedEncodingException e) { + Type metaType = new TypeToken<Map<String, String>>(){}.getType(); + Map<String, String> meta = new HashMap<String, String>(); + meta.put("sig", new String(pk, "UTF-8")); + meta.put("type", "metric"); + Gson gson = new Gson(); + buffer = gson.toJson(meta, metaType); + put(type.getBytes(), metric.getBytes(), buffer.toString().getBytes()); + } catch (Exception e) { Log.warn("Error encoding metadata."); Log.warn(e); } - put(type.getBytes(), metric.getBytes(), json.toString().getBytes()); - } public void put(String key, String source, String info) { @@ -73,7 +83,7 @@ public class Reporter { public void put(byte[] key, byte[] source, byte[] info) { Put put = new Put(key); - put.add("k".getBytes(), source, info); + put.addColumn("k".getBytes(), source, info); meta.add(put); } @@ -93,16 +103,19 @@ public class Reporter { public void putClusterName(String type, String clusterName) { byte[] value = getHash(clusterName); - JSONObject json = new JSONObject(); - + String buffer; try { - json.put("sig", new String(value, "UTF-8")); - json.put("type", "cluster"); + Type metaType = new TypeToken<Map<String, String>>(){}.getType(); + Map<String, String> meta = new HashMap<String, String>(); + meta.put("sig", new String(value, "UTF-8")); + meta.put("type", "cluster"); + Gson gson = new Gson(); + buffer = gson.toJson(meta, metaType); + put(type.getBytes(), clusterName.getBytes(), buffer.toString().getBytes()); } catch (UnsupportedEncodingException e) { Log.warn("Error encoding metadata."); Log.warn(e); } - put(type.getBytes(), clusterName.getBytes(), json.toString().getBytes()); } } http://git-wip-us.apache.org/repos/asf/chukwa/blob/384d3449/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java b/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java index 2bef452..cd8a15f 100644 --- a/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java +++ b/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java @@ -46,7 +46,6 @@ import org.apache.hadoop.chukwa.util.ExceptionUtil; import org.apache.hadoop.chukwa.util.HBaseUtil; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; @@ -63,7 +62,6 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.log4j.Logger; import org.json.simple.JSONObject; import org.json.simple.JSONValue; -import org.mortbay.log.Log; import com.google.gson.Gson; @@ -72,7 +70,7 @@ public class ChukwaHBaseStore { static int MINUTES_IN_HOUR = 60; static double RESOLUTION = 360; static int MINUTE = 60000; //60 milliseconds - static int SECOND = 1000; + final static int SECOND = (int) TimeUnit.SECONDS.toMillis(1); static byte[] COLUMN_FAMILY = "t".getBytes(); static byte[] ANNOTATION_FAMILY = "a".getBytes(); @@ -169,11 +167,11 @@ public class ChukwaHBaseStore { while (it.hasNext()) { Result result = it.next(); - for (KeyValue kv : result.raw()) { - byte[] key = kv.getQualifier(); + for (Cell kv : result.rawCells()) { + byte[] key = CellUtil.cloneQualifier(kv); long timestamp = ByteBuffer.wrap(key).getLong(); double value = Double - .parseDouble(new String(kv.getValue(), "UTF-8")); + .parseDouble(new String(CellUtil.cloneValue(kv), "UTF-8")); series.add(timestamp, value); } } @@ -195,10 +193,10 @@ public class ChukwaHBaseStore { Table table = connection.getTable(TableName.valueOf(CHUKWA_META)); Get get = new Get(metricGroup.getBytes()); Result result = table.get(get); - for (KeyValue kv : result.raw()) { - JSONObject json = (JSONObject) JSONValue.parse(new String(kv.getValue(), "UTF-8")); + for (Cell kv : result.rawCells()) { + JSONObject json = (JSONObject) JSONValue.parse(new String(CellUtil.cloneValue(kv), "UTF-8")); if (json.get("type").equals("metric")) { - familyNames.add(new String(kv.getQualifier(), "UTF-8")); + familyNames.add(new String(CellUtil.cloneQualifier(kv), "UTF-8")); } } table.close(); @@ -243,9 +241,9 @@ public class ChukwaHBaseStore { while (it.hasNext()) { Result result = it.next(); for (Cell cell : result.rawCells()) { - JSONObject json = (JSONObject) JSONValue.parse(new String(cell.getValue(), "UTF-8")); + JSONObject json = (JSONObject) JSONValue.parse(new String(CellUtil.cloneValue(cell), "UTF-8")); if (json!=null && json.get("type")!=null && json.get("type").equals("source")) { - pk.add(new String(cell.getQualifier(), "UTF-8")); + pk.add(new String(CellUtil.cloneQualifier(cell), "UTF-8")); } } } @@ -260,7 +258,6 @@ public class ChukwaHBaseStore { public static Heatmap getHeatmap(String metricGroup, String metric, long startTime, long endTime, double max, double scale, int width, int height) { - final long SECONDS = TimeUnit.SECONDS.toMillis(1); Heatmap heatmap = new Heatmap(); Set<String> sources = getSourceNames(metricGroup); Set<String> metrics = getMetricNames(metricGroup); @@ -298,7 +295,7 @@ public class ChukwaHBaseStore { for (Result result : rs) { for(Cell cell : result.rawCells()) { byte[] dest = new byte[5]; - System.arraycopy(cell.getRow(), 3, dest, 0, 5); + System.arraycopy(CellUtil.cloneRow(cell), 3, dest, 0, 5); String source = new String(dest); long time = cell.getTimestamp(); // Time display in x axis @@ -358,9 +355,9 @@ public class ChukwaHBaseStore { while (it.hasNext()) { Result result = it.next(); for (Cell cell : result.rawCells()) { - JSONObject json = (JSONObject) JSONValue.parse(new String(cell.getValue(), "UTF-8")); + JSONObject json = (JSONObject) JSONValue.parse(new String(CellUtil.cloneValue(cell), "UTF-8")); if (json.get("type").equals("cluster")) { - clusters.add(new String(cell.getQualifier(), "UTF-8")); + clusters.add(new String(CellUtil.cloneQualifier(cell), "UTF-8")); } } } @@ -411,7 +408,7 @@ public class ChukwaHBaseStore { Put put = new Put(CHART_TYPE); Gson gson = new Gson(); String buffer = gson.toJson(chart); - put.add(CHART_FAMILY, id.getBytes(), buffer.getBytes()); + put.addColumn(CHART_FAMILY, id.getBytes(), buffer.getBytes()); table.put(put); table.close(); } catch (Exception e) { @@ -472,7 +469,7 @@ public class ChukwaHBaseStore { Put put = new Put(CHART_TYPE); Gson gson = new Gson(); String buffer = gson.toJson(chart); - put.add(CHART_FAMILY, id.getBytes(), buffer.getBytes()); + put.addColumn(CHART_FAMILY, id.getBytes(), buffer.getBytes()); table.put(put); table.close(); } catch (Exception e) { @@ -549,10 +546,10 @@ public class ChukwaHBaseStore { while (it.hasNext()) { Result result = it.next(); - for (KeyValue kv : result.raw()) { - byte[] key = kv.getQualifier(); + for (Cell kv : result.rawCells()) { + byte[] key = CellUtil.cloneQualifier(kv); long timestamp = ByteBuffer.wrap(key).getLong(); - double value = Double.parseDouble(new String(kv.getValue(), + double value = Double.parseDouble(new String(CellUtil.cloneValue(kv), "UTF-8")); if(initial==0) { filteredValue = value; @@ -617,7 +614,7 @@ public class ChukwaHBaseStore { int c = 0; while(it.hasNext()) { Result result = it.next(); - for(KeyValue kv : result.raw()) { + for(Cell kv : result.rawCells()) { if(c > limit) { break; } @@ -625,7 +622,7 @@ public class ChukwaHBaseStore { continue; } Gson gson = new Gson(); - Widget widget = gson.fromJson(new String(kv.getValue(), "UTF-8"), Widget.class); + Widget widget = gson.fromJson(new String(CellUtil.cloneValue(kv), "UTF-8"), Widget.class); list.add(widget); c++; } @@ -659,9 +656,9 @@ public class ChukwaHBaseStore { Iterator<Result> it = rs.iterator(); while(it.hasNext()) { Result result = it.next(); - for(KeyValue kv : result.raw()) { + for(Cell kv : result.rawCells()) { Gson gson = new Gson(); - Widget widget = gson.fromJson(new String(kv.getValue(), "UTF-8"), Widget.class); + Widget widget = gson.fromJson(new String(CellUtil.cloneValue(kv), "UTF-8"), Widget.class); list.add(widget); } } @@ -719,7 +716,7 @@ public class ChukwaHBaseStore { Put put = new Put(WIDGET_TYPE); Gson gson = new Gson(); String buffer = gson.toJson(widget); - put.add(COMMON_FAMILY, widget.getTitle().getBytes(), buffer.getBytes()); + put.addColumn(COMMON_FAMILY, widget.getTitle().getBytes(), buffer.getBytes()); table.put(put); created = true; } @@ -749,7 +746,7 @@ public class ChukwaHBaseStore { Put put = new Put(WIDGET_TYPE); Gson gson = new Gson(); String buffer = gson.toJson(widget); - put.add(COMMON_FAMILY, title.getBytes(), buffer.getBytes()); + put.addColumn(COMMON_FAMILY, title.getBytes(), buffer.getBytes()); table.put(put); table.close(); result = true; @@ -947,7 +944,7 @@ public class ChukwaHBaseStore { Put put = new Put(DASHBOARD_TYPE); Gson gson = new Gson(); String buffer = gson.toJson(dash); - put.add(COMMON_FAMILY, key.getBytes(), buffer.getBytes()); + put.addColumn(COMMON_FAMILY, key.getBytes(), buffer.getBytes()); table.put(put); table.close(); result = true; http://git-wip-us.apache.org/repos/asf/chukwa/blob/384d3449/src/main/java/org/apache/hadoop/chukwa/hicc/HiccWebServer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/hadoop/chukwa/hicc/HiccWebServer.java b/src/main/java/org/apache/hadoop/chukwa/hicc/HiccWebServer.java index 073fc35..2d84c09 100644 --- a/src/main/java/org/apache/hadoop/chukwa/hicc/HiccWebServer.java +++ b/src/main/java/org/apache/hadoop/chukwa/hicc/HiccWebServer.java @@ -22,7 +22,6 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -32,7 +31,6 @@ import java.util.jar.JarEntry; import java.util.jar.JarFile; import org.apache.hadoop.chukwa.conf.ChukwaConfiguration; -import org.apache.hadoop.chukwa.datastore.ChukwaHBaseStore; import org.apache.hadoop.chukwa.util.ExceptionUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; http://git-wip-us.apache.org/repos/asf/chukwa/blob/384d3449/src/main/java/org/apache/hadoop/chukwa/hicc/rest/MetricsController.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/hadoop/chukwa/hicc/rest/MetricsController.java b/src/main/java/org/apache/hadoop/chukwa/hicc/rest/MetricsController.java index 9730b43..5130523 100644 --- a/src/main/java/org/apache/hadoop/chukwa/hicc/rest/MetricsController.java +++ b/src/main/java/org/apache/hadoop/chukwa/hicc/rest/MetricsController.java @@ -17,8 +17,11 @@ */ package org.apache.hadoop.chukwa.hicc.rest; +import java.lang.reflect.Type; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; import java.util.Set; import javax.servlet.http.HttpServletRequest; @@ -35,9 +38,9 @@ import javax.ws.rs.core.Response; import org.apache.hadoop.chukwa.datastore.ChukwaHBaseStore; import org.apache.hadoop.chukwa.hicc.TimeHandler; import org.apache.hadoop.chukwa.hicc.bean.Series; -import org.json.simple.JSONArray; import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; @Path("/metrics") public class MetricsController { @@ -95,15 +98,16 @@ public class MetricsController { if(skey!=null) { HttpSession session = request.getSession(); String[] sourcekeys = (session.getAttribute(skey).toString()).split(","); - JSONArray seriesList = new JSONArray(); + Type seriesListType =new TypeToken<ArrayList<Series>>(){}.getType(); + ArrayList<Series> seriesList = new ArrayList<Series>(); for(String source : sourcekeys) { - if (source == null || source.equals("")) { - continue; - } + if (source == null || source.equals("")) { + continue; + } Series output = ChukwaHBaseStore.getSeries(metricGroup, metric, source, startTime, endTime); - seriesList.add(output.toJSONObject()); + seriesList.add(output); } - buffer = seriesList.toString(); + buffer = new Gson().toJson(seriesList, seriesListType); } else { throw new WebApplicationException(Response.status(Response.Status.BAD_REQUEST) .entity("No session attribute key defined.").build()); @@ -120,11 +124,9 @@ public class MetricsController { @Produces("application/json") public String getTables() { Set<String> metricGroups = ChukwaHBaseStore.getMetricGroups(); - JSONArray groups = new JSONArray(); - for(String metric : metricGroups) { - groups.add(metric); - } - return groups.toString(); + Type metricGroupsType = new TypeToken<List<String>>(){}.getType(); + String groups = new Gson().toJson(metricGroups, metricGroupsType); + return groups; } @GET @@ -132,11 +134,9 @@ public class MetricsController { @Produces("application/json") public String getMetrics(@PathParam("metricGroup") String metricGroup) { Set<String> metricNames = ChukwaHBaseStore.getMetricNames(metricGroup); - JSONArray metrics = new JSONArray(); - for(String metric : metricNames) { - metrics.add(metric); - } - return metrics.toString(); + Type metricsType = new TypeToken<List<String>>(){}.getType(); + String metrics = new Gson().toJson(metricNames, metricsType); + return metrics; } @GET @@ -144,11 +144,9 @@ public class MetricsController { @Produces("application/json") public String getSourceNames(@Context HttpServletRequest request, @PathParam("metricGroup") String metricGroup) { Set<String> sourceNames = ChukwaHBaseStore.getSourceNames(metricGroup); - JSONArray rows = new JSONArray(); - for(String row : sourceNames) { - rows.add(row); - } - return rows.toString(); + Type rowsType = new TypeToken<List<String>>(){}.getType(); + String rows = new Gson().toJson(sourceNames, rowsType); + return rows; } }
