More blur shell top updates.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/d54a7fc8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/d54a7fc8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/d54a7fc8 Branch: refs/heads/master Commit: d54a7fc81ef1a5c956b2a20f943c49d457589fb7 Parents: c382549 Author: Aaron McCurry <[email protected]> Authored: Mon Jun 24 10:49:37 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Mon Jun 24 10:49:37 2013 -0400 ---------------------------------------------------------------------- .../blur/server/ShardServerEventHandler.java | 26 +++ .../java/org/apache/blur/shell/TopCommand.java | 233 ++++++++++++------- .../org/apache/blur/shell/top.properties | 66 ++++++ 3 files changed, 244 insertions(+), 81 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d54a7fc8/blur-core/src/main/java/org/apache/blur/server/ShardServerEventHandler.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/ShardServerEventHandler.java b/blur-core/src/main/java/org/apache/blur/server/ShardServerEventHandler.java index d8eb7cb..0f0742a 100644 --- a/blur-core/src/main/java/org/apache/blur/server/ShardServerEventHandler.java +++ b/blur-core/src/main/java/org/apache/blur/server/ShardServerEventHandler.java @@ -16,6 +16,12 @@ package org.apache.blur.server; * See the License for the specific language governing permissions and * limitations under the License. */ +import static org.apache.blur.metrics.MetricsConstants.BLUR; +import static org.apache.blur.metrics.MetricsConstants.ORG_APACHE_BLUR; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocol; @@ -23,6 +29,11 @@ import org.apache.blur.thirdparty.thrift_0_9_0.server.ServerContext; import org.apache.blur.thirdparty.thrift_0_9_0.server.TServerEventHandler; import org.apache.blur.thirdparty.thrift_0_9_0.transport.TTransport; +import com.yammer.metrics.Metrics; +import com.yammer.metrics.core.Gauge; +import com.yammer.metrics.core.Meter; +import com.yammer.metrics.core.MetricName; + /** * {@link ShardServerContext} is the session manager for the shard servers. It * allows for reader reuse across method calls. @@ -30,6 +41,18 @@ import org.apache.blur.thirdparty.thrift_0_9_0.transport.TTransport; public class ShardServerEventHandler implements TServerEventHandler { private static final Log LOG = LogFactory.getLog(ShardServerEventHandler.class); + private final Meter _connectionMeter; + private final AtomicLong _connections = new AtomicLong(); + + public ShardServerEventHandler() { + Metrics.newGauge(new MetricName(ORG_APACHE_BLUR, BLUR, "Connections"), new Gauge<Long>() { + @Override + public Long value() { + return null; + } + }); + _connectionMeter = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, BLUR, "Connections/s"), "Connections/s", TimeUnit.SECONDS); + } @Override public void preServe() { @@ -39,6 +62,8 @@ public class ShardServerEventHandler implements TServerEventHandler { @Override public ServerContext createContext(TProtocol input, TProtocol output) { LOG.debug("Client connected"); + _connectionMeter.mark(); + _connections.incrementAndGet(); return new ShardServerContext(); } @@ -47,6 +72,7 @@ public class ShardServerEventHandler implements TServerEventHandler { LOG.debug("Client disconnected"); ShardServerContext context = (ShardServerContext) serverContext; context.close(); + _connections.decrementAndGet(); } @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d54a7fc8/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java index b712c07..ca6eee8 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java @@ -20,11 +20,15 @@ package org.apache.blur.shell; import java.io.IOException; import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Properties; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; @@ -35,54 +39,26 @@ import jline.Terminal; import jline.console.ConsoleReader; import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.blur.thirdparty.thrift_0_9_0.transport.TTransportException; import org.apache.blur.thrift.BlurClientManager; import org.apache.blur.thrift.Connection; import org.apache.blur.thrift.generated.Blur; import org.apache.blur.thrift.generated.Blur.Client; -import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.Metric; public class TopCommand extends Command { + private static final String TOP = "top."; + private static final String LONGNAME = ".longname"; + private static final String SHORTNAME = ".shortname"; + private static final String UNKNOWN = "Unknown"; + private static final String TOP_SHARD_SERVER_SHORTNAME = "top.SHARD_SERVER.shortname"; + public enum SCREEN { HELP, TOP } - private static final String SC = "seg cnt"; - private static final String IC = "idx cnt"; - private static final String TC = "tble cnt"; - private static final String RE = "rd rec"; - private static final String IM = "idx mem"; - private static final String RO = "rd row"; - private static final String CS = "bc size"; - private static final String CE = "bc evt"; - private static final String CM = "bc miss"; - private static final String CH = "bc hit"; - private static final String IQ = "in qry"; - private static final String EQ = "ex qry"; - private static final String HU = "heap usd"; - private static final String SL = "sys load"; - - private static final String SHARD_SERVER = "Shard Server"; - private static final String INTERNAL_QUERIES = "\"org.apache.blur\":type=\"Blur\",name=\"Internal Queries/s\""; - private static final String EXTERNAL_QUERIES = "\"org.apache.blur\":type=\"Blur\",name=\"External Queries/s\""; - - private static final String CACHE_HIT = "\"org.apache.blur\":type=\"Cache\",name=\"Hit\""; - private static final String CACHE_MISS = "\"org.apache.blur\":type=\"Cache\",name=\"Miss\""; - private static final String CACHE_EVICTION = "\"org.apache.blur\":type=\"Cache\",name=\"Eviction\""; - private static final String CACHE_SIZE = "\"org.apache.blur\":type=\"Cache\",name=\"Size\""; - - private static final String READ_RECORDS = "\"org.apache.blur\":type=\"Blur\",name=\"Read Records/s\""; - private static final String READ_ROWS = "\"org.apache.blur\":type=\"Blur\",name=\"Read Row/s\""; - - private static final String INDEX_MEMORY_USAGE = "\"org.apache.blur\":type=\"Blur\",scope=\"default\",name=\"Index Memory Usage\""; - private static final String TABLE_COUNT = "\"org.apache.blur\":type=\"Blur\",scope=\"default\",name=\"Table Count\""; - private static final String INDEX_COUNT = "\"org.apache.blur\":type=\"Blur\",scope=\"default\",name=\"Index Count\""; - private static final String SEGMENT_COUNT = "\"org.apache.blur\":type=\"Blur\",scope=\"default\",name=\"Segment Count\""; - private static final String LOAD_AVERAGE = "\"org.apache.blur\":type=\"System\",name=\"Load Average\""; - private static final String HEAP_USED = "\"org.apache.blur\":type=\"JVM\",name=\"Heap Used\""; - private static final double ONE_THOUSAND = 1000; private static final double ONE_MILLION = ONE_THOUSAND * ONE_THOUSAND; private static final double ONE_BILLION = ONE_THOUSAND * ONE_MILLION; @@ -111,28 +87,32 @@ public class TopCommand extends Command { AtomicBoolean quit = new AtomicBoolean(); AtomicBoolean help = new AtomicBoolean(); + Properties properties = new Properties(); + try { + properties.load(getClass().getResourceAsStream("top.properties")); + } catch (IOException e) { + if (Main.debug) { + e.printStackTrace(); + } + throw new CommandException(e.getMessage()); + } + Map<String, String> metricNames = new HashMap<String, String>(); - metricNames.put(IQ, INTERNAL_QUERIES); - metricNames.put(EQ, EXTERNAL_QUERIES); - metricNames.put(CH, CACHE_HIT); - metricNames.put(CM, CACHE_MISS); - metricNames.put(CE, CACHE_EVICTION); - metricNames.put(CS, CACHE_SIZE); - metricNames.put(RO, READ_RECORDS); - metricNames.put(RE, READ_ROWS); - metricNames.put(IM, INDEX_MEMORY_USAGE); - metricNames.put(TC, TABLE_COUNT); - metricNames.put(IC, INDEX_COUNT); - metricNames.put(SC, SEGMENT_COUNT); - metricNames.put(HU, HEAP_USED); - metricNames.put(SL, LOAD_AVERAGE); - - Object[] labels = new Object[] { SHARD_SERVER, SL, HU, IM, EQ, IQ, RO, RE, CH, CM, CE, CS, TC, IC, SC }; - - Set<String> sizes = new HashSet<String>(); - sizes.add(IM); - sizes.add(SL); + Set<Object> keySet = properties.keySet(); + for (Object k : keySet) { + String key = k.toString(); + if (isShortName(key)) { + String shortName = getShortName(key, properties); + String longName = getLongName(getLongNameKey(key), properties); + metricNames.put(shortName, longName); + } + } + + String labelsStr = properties.getProperty("top.columns"); + String[] labels = resolveShortNames(labelsStr.split(","), properties); + String sizesStr = properties.getProperty("top.sizes"); + Set<String> sizes = new HashSet<String>(Arrays.asList(resolveShortNames(sizesStr.split(","), properties))); Set<String> keys = new HashSet<String>(metricNames.values()); String cluster; @@ -142,8 +122,6 @@ public class TopCommand extends Command { cluster = args[1]; } - List<String> shardServerList = client.shardServerList(cluster); - ConsoleReader reader = this.getConsoleReader(); if (reader != null) { Terminal terminal = reader.getTerminal(); @@ -161,25 +139,12 @@ public class TopCommand extends Command { startCommandWatcher(reader, quit, help, this); } - Map<String, AtomicReference<Blur.Iface>> shardClients = new ConcurrentHashMap<String, AtomicReference<Blur.Iface>>(); - for (String sc : shardServerList) { - AtomicReference<Iface> ref = shardClients.get(sc); - if (ref == null) { - ref = new AtomicReference<Blur.Iface>(); - shardClients.put(sc, ref); - } - try { - Client c = BlurClientManager.newClient(new Connection(sc)); - ref.set(c); - } catch (IOException e) { - ref.set(null); - if (Main.debug) { - e.printStackTrace(); - } - } - } + List<String> shardServerList = new ArrayList<String>(client.shardServerList(cluster)); + Collections.sort(shardServerList); + Map<String, AtomicReference<Client>> shardClients = setupClients(shardServerList); - int longestServerName = Math.max(getSizeOfLongestKey(shardClients), SHARD_SERVER.length()); + String shardServerLabel = properties.getProperty(TOP_SHARD_SERVER_SHORTNAME); + int longestServerName = Math.max(getSizeOfLongestKey(shardClients), shardServerLabel.length()); StringBuilder header = new StringBuilder("%" + longestServerName + "s"); for (int i = 1; i < labels.length; i++) { @@ -194,12 +159,13 @@ public class TopCommand extends Command { } else if (help.get()) { showHelp(output, labels, metricNames); } else { - output.append(truncate(String.format(header.toString(), labels))); - for (Entry<String, AtomicReference<Blur.Iface>> e : new TreeMap<String, AtomicReference<Blur.Iface>>( - shardClients).entrySet()) { + output.append(truncate(String.format(header.toString(), (Object[]) labels))); + for (Entry<String, AtomicReference<Client>> e : new TreeMap<String, AtomicReference<Client>>(shardClients) + .entrySet()) { String shardServer = e.getKey(); - Iface shardClient = e.getValue().get(); - if (shardClient == null) { + AtomicReference<Client> ref = e.getValue(); + Map<String, Metric> metrics = getMetrics(shardServer, ref, keys); + if (metrics == null) { String line = String.format("%" + longestServerName + "s*%n", shardServer); output.append(line); } else { @@ -207,7 +173,7 @@ public class TopCommand extends Command { int c = 0; cols[c++] = shardServer; StringBuilder sb = new StringBuilder("%" + longestServerName + "s"); - Map<String, Metric> metrics = shardClient.metrics(keys); + for (int i = 1; i < labels.length; i++) { String mn = metricNames.get(labels[i]); Metric metric = metrics.get(mn); @@ -244,9 +210,114 @@ public class TopCommand extends Command { Terminal terminal = reader.getTerminal(); _height = terminal.getHeight() - 2; _width = terminal.getWidth() - 2; + + List<String> currentShardServerList = new ArrayList<String>(client.shardServerList(cluster)); + Collections.sort(currentShardServerList); + if (!shardServerList.equals(currentShardServerList)) { + close(shardClients); + shardClients = setupClients(shardServerList); + } } } while (reader != null); + } + + private void close(Map<String, AtomicReference<Client>> shardClients) { + for (AtomicReference<Client> client : shardClients.values()) { + tryToClose(client); + } + } + + private Map<String, AtomicReference<Client>> setupClients(List<String> shardServerList) { + Map<String, AtomicReference<Client>> shardClients = new ConcurrentHashMap<String, AtomicReference<Client>>(); + for (String sc : shardServerList) { + AtomicReference<Client> ref = shardClients.get(sc); + if (ref == null) { + ref = new AtomicReference<Client>(); + shardClients.put(sc, ref); + } + tryToConnect(sc, ref); + } + return shardClients; + } + + private boolean tryToConnect(String sc, AtomicReference<Client> ref) { + try { + Client c = BlurClientManager.newClient(new Connection(sc)); + ref.set(c); + return true; + } catch (TTransportException e) { + ref.set(null); + if (Main.debug) { + e.printStackTrace(); + } + } catch (IOException e) { + ref.set(null); + if (Main.debug) { + e.printStackTrace(); + } + } + return false; + } + + private Map<String, Metric> getMetrics(String sc, AtomicReference<Client> ref, Set<String> keys) { + if (ref.get() == null) { + if (!tryToConnect(sc, ref)) { + return null; + } + } + try { + return ref.get().metrics(keys); + } catch (BlurException e) { + if (Main.debug) { + e.printStackTrace(); + } + } catch (TException e) { + if (Main.debug) { + e.printStackTrace(); + } + tryToClose(ref); + } + return null; + } + + private void tryToClose(AtomicReference<Client> ref) { + Client client = ref.get(); + if (client != null) { + ref.set(null); + try { + client.getInputProtocol().getTransport().close(); + } catch (Exception e) { + if (Main.debug) { + e.printStackTrace(); + } + } + } + } + + private String[] resolveShortNames(String[] keys, Properties properties) { + String[] labels = new String[keys.length]; + int i = 0; + for (String key : keys) { + String shortName = properties.getProperty(TOP + key + SHORTNAME); + labels[i++] = shortName; + } + return labels; + } + + private String getLongNameKey(String key) { + return key.replace(SHORTNAME, LONGNAME); + } + + private String getShortName(String key, Properties properties) { + return properties.getProperty(key, UNKNOWN); + } + + private String getLongName(String key, Properties properties) { + return properties.getProperty(key, UNKNOWN); + } + private boolean isShortName(String key) { + return key.endsWith(SHORTNAME); } private void showHelp(StringBuilder output, Object[] labels, Map<String, String> metricNames) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d54a7fc8/blur-shell/src/main/resources/org/apache/blur/shell/top.properties ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/resources/org/apache/blur/shell/top.properties b/blur-shell/src/main/resources/org/apache/blur/shell/top.properties new file mode 100644 index 0000000..9f3cbe2 --- /dev/null +++ b/blur-shell/src/main/resources/org/apache/blur/shell/top.properties @@ -0,0 +1,66 @@ +top.columns=SHARD_SERVER,SL,HU,IM,EQ,IQ,RO,RE,CH,CM,CE,CS,TC,IC,SC +top.sizes=IM,SL + +top.SHARD_SERVER.shortname=Shard Server +top.SC.shortname=seg cnt +top.IC.shortname=idx cnt +top.TC.shortname=tble cnt +top.RE.shortname=rd rec +top.IM.shortname=idx mem +top.RO.shortname=rd row +top.CS.shortname=bc size +top.CE.shortname=bc evt +top.CM.shortname=bc miss +top.CH.shortname=bc hit +top.IQ.shortname=in qry +top.EQ.shortname=ex qry +top.HU.shortname=heap usd +top.SL.shortname=sys load + +top.SHARD_SERVER.longname=Shard Server +top.IQ.longname="org.apache.blur":type="Blur",name="Internal Queries/s" +top.EQ.longname="org.apache.blur":type="Blur",name="External Queries/s" +top.CH.longname="org.apache.blur":type="Cache",name="Hit" +top.CM.longname="org.apache.blur":type="Cache",name="Miss" +top.CE.longname="org.apache.blur":type="Cache",name="Eviction" +top.CS.longname="org.apache.blur":type="Cache",name="Size" +top.RE.longname="org.apache.blur":type="Blur",name="Read Records/s" +top.RO.longname="org.apache.blur":type="Blur",name="Read Row/s" +top.IM.longname="org.apache.blur":type="Blur",scope="default",name="Index Memory Usage" +top.TC.longname="org.apache.blur":type="Blur",scope="default",name="Table Count" +top.IC.longname="org.apache.blur":type="Blur",scope="default",name="Index Count" +top.SC.longname="org.apache.blur":type="Blur",scope="default",name="Segment Count" +top.SL.longname="org.apache.blur":type="System",name="Load Average" +top.HU.longname="org.apache.blur":type="JVM",name="Heap Used" + +top.SHARD_SERVER.longname=Shard Server +top.IQ.longname="org.apache.blur":type="Blur",name="Internal Queries/s" +top.EQ.longname="org.apache.blur":type="Blur",name="External Queries/s" +top.CH.longname="org.apache.blur":type="Cache",name="Hit" +top.CM.longname="org.apache.blur":type="Cache",name="Miss" +top.CE.longname="org.apache.blur":type="Cache",name="Eviction" +top.CS.longname="org.apache.blur":type="Cache",name="Size" +top.RE.longname="org.apache.blur":type="Blur",name="Read Records/s" +top.RO.longname="org.apache.blur":type="Blur",name="Read Row/s" +top.IM.longname="org.apache.blur":type="Blur",scope="default",name="Index Memory Usage" +top.TC.longname="org.apache.blur":type="Blur",scope="default",name="Table Count" +top.IC.longname="org.apache.blur":type="Blur",scope="default",name="Index Count" +top.SC.longname="org.apache.blur":type="Blur",scope="default",name="Segment Count" +top.SL.longname="org.apache.blur":type="System",name="Load Average" +top.HU.longname="org.apache.blur":type="JVM",name="Heap Used" + +top.SHARD_SERVER.help=Shard Server - * indicates that the server is offline +top.IQ.help=Internal Queries/s - This counts every shard query (One Minute Average) +top.EQ.help=External Queries/s - This counts every query request no matter the number of shards (One Minute Average) +top.CH.help=Block Cache Hits/s - The number of 8k block hits (One Minute Average) +top.CM.help=Block Cache Misses/s - The number of 8k block misses (One Minute Average) +top.CE.help=Block Cache Evictions/s - The number of 8k block evictions (One Minute Average) +top.CS.help=Block Cache Size - The total block size +top.RE.help=Read Records/s - The number of records read per second (One Minute Average) +top.RO.help=Read Row/s - The number of rows read per second (One Minute Average) +top.IM.help=Index Memory Usage - The index heap usage +top.TC.help=Table Count - The number of tables +top.IC.help=Index Count - The number of indexes +top.SC.help=Segment Count - The number of index segments +top.SL.help=Load Average - Current server load average +top.HU.help=Heap Used - Current heap size \ No newline at end of file
