Inital commit of the top command in the blur-shell.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/7137006b Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/7137006b Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/7137006b Branch: refs/heads/master Commit: 7137006b58f1b55d1d36f1386333a6122ed11beb Parents: 878381e Author: Aaron McCurry <[email protected]> Authored: Sat Jun 22 17:04:22 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Sat Jun 22 17:04:22 2013 -0400 ---------------------------------------------------------------------- .../org/apache/blur/manager/IndexManager.java | 12 + .../blur/thrift/BlurControllerServer.java | 28 + .../org/apache/blur/thrift/BlurShardServer.java | 13 +- .../java/org/apache/blur/thrift/TableAdmin.java | 26 + .../blur/thrift/ThriftBlurControllerServer.java | 2 + .../blur/thrift/ThriftBlurShardServer.java | 2 + .../org/apache/blur/utils/MemoryReporter.java | 159 + .../main/java/org/apache/blur/shell/Main.java | 14 +- .../org/apache/blur/shell/ParseCommand.java | 51 + .../java/org/apache/blur/shell/TopCommand.java | 277 + .../org/apache/blur/thrift/generated/Blur.java | 8248 +++++++++++------- .../apache/blur/thrift/generated/Metric.java | 894 ++ .../apache/blur/thrift/util/QueryMetrics.java | 43 + .../src/main/scripts/interface/Blur.thrift | 17 +- .../main/scripts/interface/gen-html/Blur.html | 21 +- .../main/scripts/interface/gen-html/index.html | 3 + .../org/apache/blur/thrift/generated/Blur.java | 8248 +++++++++++------- .../apache/blur/thrift/generated/Metric.java | 894 ++ .../src/main/scripts/interface/gen-js/Blur.js | 808 +- .../main/scripts/interface/gen-js/Blur_types.js | 194 + .../scripts/interface/gen-perl/Blur/Blur.pm | 780 +- .../scripts/interface/gen-perl/Blur/Types.pm | 184 + .../src/main/scripts/interface/gen-rb/blur.rb | 124 + .../main/scripts/interface/gen-rb/blur_types.rb | 22 + pom.xml | 2 +- 25 files changed, 14494 insertions(+), 6572 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java index 1063b7f..7280ca7 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java +++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java @@ -406,6 +406,18 @@ public class IndexManager { } } + public String parseQuery(String table, SimpleQuery simpleQuery) throws ParseException, BlurException { + TableContext context = getTableContext(table); + BlurAnalyzer analyzer = _indexServer.getAnalyzer(table); + Filter preFilter = QueryParserUtil.parseFilter(table, simpleQuery.preSuperFilter, false, analyzer, _filterCache, + context); + Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.postSuperFilter, true, analyzer, _filterCache, + context); + Query userQuery = QueryParserUtil.parseQuery(simpleQuery.queryStr, simpleQuery.superQueryOn, analyzer, postFilter, + preFilter, getScoreType(simpleQuery.type), context); + return userQuery.toString(); + } + private TableContext getTableContext(final String table) { return TableContext.create(_clusterStatus.getTableDescriptor(true, _clusterStatus.getCluster(true, table), table)); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java index ecdccc7..54524f4 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java @@ -69,6 +69,7 @@ import org.apache.blur.thrift.generated.RowMutation; import org.apache.blur.thrift.generated.Schema; import org.apache.blur.thrift.generated.Selector; import org.apache.blur.thrift.generated.ShardState; +import org.apache.blur.thrift.generated.SimpleQuery; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.thrift.generated.TableStats; import org.apache.blur.utils.BlurConstants; @@ -911,4 +912,31 @@ public class BlurControllerServer extends TableAdmin implements Iface { } } + @Override + public String parseQuery(final String table, final SimpleQuery simpleQuery) throws BlurException, TException { + checkTable(table); + String cluster = getCluster(table); + List<String> onlineShardServers = _clusterStatus.getOnlineShardServers(true, cluster); + try { + return BlurClientManager.execute(getConnections(onlineShardServers), new BlurCommand<String>() { + @Override + public String call(Client client) throws BlurException, TException { + return client.parseQuery(table, simpleQuery); + } + }); + } catch (Exception e) { + LOG.error("Unknown error while trying to parse query [table={0},simpleQuery={1}]", e, table, simpleQuery); + throw new BException("Unknown error while trying to parse query [table={0},simpleQuery={1}]", e, table, + simpleQuery); + } + } + + private List<Connection> getConnections(List<String> onlineShardServers) { + List<Connection> connections = new ArrayList<Connection>(); + for (String c : onlineShardServers) { + connections.add(new Connection(c)); + } + return connections; + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java index db2a817..2d6a022 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java @@ -52,6 +52,7 @@ import org.apache.blur.thrift.generated.RowMutation; import org.apache.blur.thrift.generated.Schema; import org.apache.blur.thrift.generated.Selector; import org.apache.blur.thrift.generated.ShardState; +import org.apache.blur.thrift.generated.SimpleQuery; import org.apache.blur.thrift.generated.TableStats; import org.apache.blur.utils.BlurConstants; import org.apache.blur.utils.BlurUtil; @@ -100,7 +101,7 @@ public class BlurShardServer extends TableAdmin implements Iface { highlightOptions.setSimpleQuery(blurQuery.getSimpleQuery()); } } - + if (blurQuery.useCacheIfPresent) { LOG.debug("Using cache for query [{0}] on table [{1}].", blurQuery, table); QueryCacheKey key = QueryCache.getNormalizedBlurQueryKey(table, blurQuery); @@ -134,6 +135,16 @@ public class BlurShardServer extends TableAdmin implements Iface { } @Override + public String parseQuery(String table, SimpleQuery simpleQuery) throws BlurException, TException { + try { + return _indexManager.parseQuery(table, simpleQuery); + } catch (Throwable e) { + LOG.error("Unknown error during parsing of [table={0},simpleQuery={1}]", e, table, simpleQuery); + throw new BException(e.getMessage(), e); + } + } + + @Override public FetchResult fetchRow(String table, Selector selector) throws BlurException, TException { checkTable(_cluster, table); try { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java index 8069764..5826e27 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java @@ -16,9 +16,11 @@ package org.apache.blur.thrift; * See the License for the specific language governing permissions and * limitations under the License. */ +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import org.apache.blur.BlurConfiguration; import org.apache.blur.log.Log; @@ -28,10 +30,12 @@ import org.apache.blur.server.TableContext; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; +import org.apache.blur.thrift.generated.Metric; import org.apache.blur.thrift.generated.ShardState; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.thrift.generated.TableStats; import org.apache.blur.utils.BlurUtil; +import org.apache.blur.utils.MemoryReporter; import org.apache.zookeeper.ZooKeeper; public abstract class TableAdmin implements Iface { @@ -42,6 +46,28 @@ public abstract class TableAdmin implements Iface { protected BlurConfiguration _configuration; @Override + public Map<String, Metric> metrics(Set<String> metrics) throws BlurException, TException { + try { + Map<String, Metric> metricsMap = MemoryReporter.getMetrics(); + if (metrics == null) { + return metricsMap; + } else { + Map<String, Metric> result = new HashMap<String, Metric>(); + for (String n : metrics) { + Metric metric = metricsMap.get(n); + if (metric != null) { + result.put(n, metric); + } + } + return result; + } + } catch (Exception e) { + LOG.error("Unknown error while trying to get metrics [{0}] ", e, metrics); + throw new BException(e.getMessage(), e); + } + } + + @Override public TableStats getTableStats(String table) throws BlurException, TException { return tableStats(table); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java index 5780d27..5a5c4a1 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java @@ -51,6 +51,7 @@ import org.apache.blur.metrics.ReporterSetup; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.utils.BlurConstants; import org.apache.blur.utils.BlurUtil; +import org.apache.blur.utils.MemoryReporter; import org.apache.blur.zookeeper.ZkUtils; import org.apache.zookeeper.ZooKeeper; @@ -65,6 +66,7 @@ public class ThriftBlurControllerServer extends ThriftServer { BlurConfiguration configuration = new BlurConfiguration(); printUlimits(); ReporterSetup.setupReporters(configuration); + MemoryReporter.enable(); ThriftServer server = createServer(serverIndex, configuration); server.start(); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java index 45d0dd9..ff2cec6 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java @@ -71,6 +71,7 @@ import org.apache.blur.thirdparty.thrift_0_9_0.server.TServlet; import org.apache.blur.thrift.generated.Blur; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.utils.BlurUtil; +import org.apache.blur.utils.MemoryReporter; import org.apache.blur.zookeeper.ZkUtils; import org.apache.hadoop.conf.Configuration; import org.apache.zookeeper.ZooKeeper; @@ -89,6 +90,7 @@ public class ThriftBlurShardServer extends ThriftServer { BlurConfiguration configuration = new BlurConfiguration(); printUlimits(); ReporterSetup.setupReporters(configuration); + MemoryReporter.enable(); ThriftServer server = createServer(serverIndex, configuration); server.start(); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-core/src/main/java/org/apache/blur/utils/MemoryReporter.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/utils/MemoryReporter.java b/blur-core/src/main/java/org/apache/blur/utils/MemoryReporter.java new file mode 100644 index 0000000..a64f51b --- /dev/null +++ b/blur-core/src/main/java/org/apache/blur/utils/MemoryReporter.java @@ -0,0 +1,159 @@ +package org.apache.blur.utils; + +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; + +import org.apache.blur.log.Log; +import org.apache.blur.log.LogFactory; + +import com.yammer.metrics.Metrics; +import com.yammer.metrics.core.Counter; +import com.yammer.metrics.core.Gauge; +import com.yammer.metrics.core.Histogram; +import com.yammer.metrics.core.Metered; +import com.yammer.metrics.core.Metric; +import com.yammer.metrics.core.MetricName; +import com.yammer.metrics.core.MetricProcessor; +import com.yammer.metrics.core.MetricsRegistry; +import com.yammer.metrics.core.Timer; +import com.yammer.metrics.reporting.AbstractPollingReporter; +import com.yammer.metrics.stats.Snapshot; + +public class MemoryReporter extends AbstractPollingReporter implements + MetricProcessor<ConcurrentMap<String, org.apache.blur.thrift.generated.Metric>> { + + private static final Log LOG = LogFactory.getLog(MemoryReporter.class); + + private static ConcurrentMap<String, org.apache.blur.thrift.generated.Metric> _metrics = new ConcurrentHashMap<String, org.apache.blur.thrift.generated.Metric>(); + + public static Map<String, org.apache.blur.thrift.generated.Metric> getMetrics() { + return new HashMap<String, org.apache.blur.thrift.generated.Metric>(_metrics); + } + + public static void enable() { + MemoryReporter memoryReporter = new MemoryReporter(Metrics.defaultRegistry(), "memory-reporter"); + memoryReporter.start(1, TimeUnit.SECONDS); + } + + protected MemoryReporter(MetricsRegistry registry, String name) { + super(registry, name); + } + + @Override + public void run() { + try { + MetricsRegistry registry = getMetricsRegistry(); + Map<MetricName, Metric> allMetrics = registry.allMetrics(); + for (Entry<MetricName, Metric> entry : allMetrics.entrySet()) { + entry.getValue().processWith(this, entry.getKey(), _metrics); + } + } catch (Exception e) { + LOG.error("Unknown error during metrics processing.", e); + } + } + + @Override + public void processMeter(MetricName name, Metered meter, + ConcurrentMap<String, org.apache.blur.thrift.generated.Metric> context) throws Exception { + org.apache.blur.thrift.generated.Metric metric = getMetric(name, context); + addMeter(metric, meter, context); + } + + private org.apache.blur.thrift.generated.Metric getMetric(MetricName name, + ConcurrentMap<String, org.apache.blur.thrift.generated.Metric> context) { + String nameStr = name.toString(); + org.apache.blur.thrift.generated.Metric metric = context.get(nameStr); + if (metric == null) { + metric = new org.apache.blur.thrift.generated.Metric(); + context.put(nameStr, metric); + metric.setName(nameStr); + } + return metric; + } + + private void addMeter(org.apache.blur.thrift.generated.Metric metric, Metered meter, + ConcurrentMap<String, org.apache.blur.thrift.generated.Metric> context) { + metric.putToStrMap("rateUnit", meter.rateUnit().toString()); + metric.putToStrMap("eventType", meter.eventType()); + metric.putToLongMap("count", meter.count()); + metric.putToDoubleMap("meanRate", meter.meanRate()); + metric.putToDoubleMap("oneMinuteRate", meter.oneMinuteRate()); + metric.putToDoubleMap("fiveMinuteRate", meter.fiveMinuteRate()); + metric.putToDoubleMap("fifteenMinuteRate", meter.fifteenMinuteRate()); + } + + @Override + public void processCounter(MetricName name, Counter counter, + ConcurrentMap<String, org.apache.blur.thrift.generated.Metric> context) throws Exception { + org.apache.blur.thrift.generated.Metric metric = getMetric(name, context); + metric.putToLongMap("value", counter.count()); + } + + @Override + public void processHistogram(MetricName name, Histogram histogram, + ConcurrentMap<String, org.apache.blur.thrift.generated.Metric> context) throws Exception { + org.apache.blur.thrift.generated.Metric metric = getMetric(name, context); + metric.putToDoubleMap("min", histogram.min()); + metric.putToDoubleMap("max", histogram.max()); + metric.putToDoubleMap("mean", histogram.mean()); + metric.putToDoubleMap("stdDev", histogram.stdDev()); + + Snapshot snapshot = histogram.getSnapshot(); + metric.putToDoubleMap("median", snapshot.getMedian()); + metric.putToDoubleMap("75%", snapshot.get75thPercentile()); + metric.putToDoubleMap("95%", snapshot.get95thPercentile()); + metric.putToDoubleMap("98%", snapshot.get98thPercentile()); + metric.putToDoubleMap("99%", snapshot.get99thPercentile()); + metric.putToDoubleMap("99.9%", snapshot.get999thPercentile()); + } + + @Override + public void processTimer(MetricName name, Timer timer, + ConcurrentMap<String, org.apache.blur.thrift.generated.Metric> context) throws Exception { + + org.apache.blur.thrift.generated.Metric metric = getMetric(name, context); + addMeter(metric, timer, context); + metric.putToStrMap("unit", timer.durationUnit().toString()); + metric.putToDoubleMap("min", timer.min()); + metric.putToDoubleMap("max", timer.max()); + metric.putToDoubleMap("mean", timer.mean()); + metric.putToDoubleMap("stdDev", timer.stdDev()); + + Snapshot snapshot = timer.getSnapshot(); + metric.putToDoubleMap("median", snapshot.getMedian()); + metric.putToDoubleMap("75%", snapshot.get75thPercentile()); + metric.putToDoubleMap("95%", snapshot.get95thPercentile()); + metric.putToDoubleMap("98%", snapshot.get98thPercentile()); + metric.putToDoubleMap("99%", snapshot.get99thPercentile()); + metric.putToDoubleMap("99.9%", snapshot.get999thPercentile()); + } + + @Override + public void processGauge(MetricName name, Gauge<?> gauge, + ConcurrentMap<String, org.apache.blur.thrift.generated.Metric> context) throws Exception { + org.apache.blur.thrift.generated.Metric metric = getMetric(name, context); + metric.putToDoubleMap("value", getDouble(gauge.value())); + } + + private double getDouble(Object value) { + if (value instanceof Integer) { + Integer v = (Integer) value; + return (int) v; + } else if (value instanceof Long) { + Long v = (Long) value; + return (long) v; + } else if (value instanceof Double) { + Double v = (Double) value; + return v; + } else if (value instanceof Float) { + Float v = (Float) value; + return (float) v; + } + return 0; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-shell/src/main/java/org/apache/blur/shell/Main.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/Main.java b/blur-shell/src/main/java/org/apache/blur/shell/Main.java index bd4ed23..0a6958d 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/Main.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/Main.java @@ -49,6 +49,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap.Builder; public class Main { + static final String PROMPT = "blur> "; /** is debugging enabled - off by default */ static boolean debug = false; /** is timing enabled - off by default */ @@ -213,7 +214,7 @@ public class Main { int bufferLength = getMaxCommandLength(cmds.keySet()) + 2; out.println(" - Table commands - "); String[] tableCommands = { "create", "enable", "disable", "remove", "truncate", "describe", "list", "schema", - "stats", "layout" }; + "stats", "layout", "parse" }; printCommandAndHelp(out, cmds, tableCommands, bufferLength); out.println(); @@ -223,7 +224,7 @@ public class Main { out.println(); out.println(" - Cluster commands - "); - String[] clusterCommands = { "controllers", "shards", "clusterlist", "cluster", "safemodewait" }; + String[] clusterCommands = { "controllers", "shards", "clusterlist", "cluster", "safemodewait", "top" }; printCommandAndHelp(out, cmds, clusterCommands, bufferLength); out.println(); @@ -239,10 +240,11 @@ public class Main { out.println(" " + buffer(e.getKey(), bufferLength) + " - " + e.getValue().help()); } } - + out.println(); out.println(" " + buffer("shell", bufferLength) + " - enters into the Blur interactive shell"); - out.println(" " + buffer("execute", bufferLength) + " - executes a custom class passing all the command line args to the main method"); + out.println(" " + buffer("execute", bufferLength) + + " - executes a custom class passing all the command line args to the main method"); } private int getMaxCommandLength(Set<String> keySet) { @@ -326,6 +328,8 @@ public class Main { builder.put("truncate", new TruncateTableCommand()); builder.put("cluster", new ClusterCommand()); builder.put("safemodewait", new WaitInSafemodeCommand()); + builder.put("top", new TopCommand()); + builder.put("parse", new ParseCommand()); commands = builder.build(); CliShellOptions cliShellOptions = getCliShellOptions(args); @@ -338,7 +342,7 @@ public class Main { if (cliShellOptions.isShell()) { ConsoleReader reader = new ConsoleReader(); setConsoleReader(commands, reader); - reader.setPrompt("blur> "); + reader.setPrompt(PROMPT); List<Completer> completors = new LinkedList<Completer>(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-shell/src/main/java/org/apache/blur/shell/ParseCommand.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/ParseCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/ParseCommand.java new file mode 100644 index 0000000..e651a81 --- /dev/null +++ b/blur-shell/src/main/java/org/apache/blur/shell/ParseCommand.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.blur.shell; + +import java.io.PrintWriter; + +import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.blur.thrift.generated.Blur; +import org.apache.blur.thrift.generated.BlurException; +import org.apache.blur.thrift.generated.SimpleQuery; + +public class ParseCommand extends Command { + @Override + public void doit(PrintWriter out, Blur.Iface client, String[] args) throws CommandException, TException, + BlurException { + if (args.length < 3) { + throw new CommandException("Invalid args: " + help()); + } + String tablename = args[1]; + String queryStr = ""; + for (int i = 2; i < args.length; i++) { + queryStr += args[i] + " "; + } + + SimpleQuery simpleQuery = new SimpleQuery(); + simpleQuery.setQueryStr(queryStr); + String query = client.parseQuery(tablename, simpleQuery); + out.println(query); + } + + @Override + public String help() { + return "parse a query and return string representation, args; tablename query"; + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java new file mode 100644 index 0000000..a5108f8 --- /dev/null +++ b/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java @@ -0,0 +1,277 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.blur.shell; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +import jline.Terminal; +import jline.console.ConsoleReader; + +import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.blur.thrift.BlurClient; +import org.apache.blur.thrift.generated.Blur; +import org.apache.blur.thrift.generated.Blur.Iface; +import org.apache.blur.thrift.generated.BlurException; +import org.apache.blur.thrift.generated.Metric; + +public class TopCommand extends Command { + + public enum SCREEN { + HELP, TOP + } + + private static final String SC = "seg cnt"; + private static final String IC = "idx cnt"; + private static final String TC = "tble cnt"; + private static final String RE = "rd rec"; + private static final String IM = "idx mem"; + private static final String RO = "rd row"; + private static final String CS = "bc size"; + private static final String CE = "bc evt"; + private static final String CM = "bc miss"; + private static final String CH = "bc hit"; + private static final String IQ = "in qry"; + private static final String EQ = "ex qry"; + private static final String SHARD_SERVER = "Shard Server"; + private static final String INTERNAL_QUERIES = "\"org.apache.blur\":type=\"Blur\",name=\"Internal Queries/s\""; + private static final String EXTERNAL_QUERIES = "\"org.apache.blur\":type=\"Blur\",name=\"External Queries/s\""; + + private static final String CACHE_HIT = "\"org.apache.blur\":type=\"Cache\",name=\"Hit\""; + private static final String CACHE_MISS = "\"org.apache.blur\":type=\"Cache\",name=\"Miss\""; + private static final String CACHE_EVICTION = "\"org.apache.blur\":type=\"Cache\",name=\"Eviction\""; + private static final String CACHE_SIZE = "\"org.apache.blur\":type=\"Cache\",name=\"Size\""; + + private static final String READ_RECORDS = "\"org.apache.blur\":type=\"Blur\",name=\"Read Records/s\""; + private static final String READ_ROWS = "\"org.apache.blur\":type=\"Blur\",name=\"Read Row/s\""; + + private static final String INDEX_MEMORY_USAGE = "\"org.apache.blur\":type=\"Blur\",scope=\"default\",name=\"Index Memory Usage\""; + private static final String TABLE_COUNT = "\"org.apache.blur\":type=\"Blur\",scope=\"default\",name=\"Table Count\""; + private static final String INDEX_COUNT = "\"org.apache.blur\":type=\"Blur\",scope=\"default\",name=\"Index Count\""; + private static final String SEGMENT_COUNT = "\"org.apache.blur\":type=\"Blur\",scope=\"default\",name=\"Segment Count\""; + private static final double ONE_MILLION = 1000000; + private static final double ONE_BILLION = 1000 * ONE_MILLION; + private static final double ONE_TRILLION = 1000 * ONE_BILLION; + private static final double ONE_QUADRILLION = 1000 * ONE_TRILLION; + + private int _width = Integer.MAX_VALUE; + private int _height; + + @Override + public void doit(PrintWriter out, Blur.Iface client, String[] args) throws CommandException, TException, + BlurException { + try { + doitInternal(out, client, args); + } finally { + ConsoleReader reader = this.getConsoleReader(); + if (reader != null) { + reader.setPrompt(Main.PROMPT); + } + } + } + + public void doitInternal(PrintWriter out, Blur.Iface client, String[] args) throws CommandException, TException, + BlurException { + + AtomicBoolean quit = new AtomicBoolean(); + + Map<String, String> metricNames = new HashMap<String, String>(); + metricNames.put(IQ, INTERNAL_QUERIES); + metricNames.put(EQ, EXTERNAL_QUERIES); + metricNames.put(CH, CACHE_HIT); + metricNames.put(CM, CACHE_MISS); + metricNames.put(CE, CACHE_EVICTION); + metricNames.put(CS, CACHE_SIZE); + metricNames.put(RO, READ_RECORDS); + metricNames.put(RE, READ_ROWS); + metricNames.put(IM, INDEX_MEMORY_USAGE); + metricNames.put(TC, TABLE_COUNT); + metricNames.put(IC, INDEX_COUNT); + metricNames.put(SC, SEGMENT_COUNT); + + Object[] labels = new Object[] { SHARD_SERVER, EQ, IQ, CH, CM, CE, CS, RO, RE, IM, TC, IC, SC }; + + Set<String> sizes = new HashSet<String>(); + sizes.add(IM); + + Set<String> keys = new HashSet<String>(metricNames.values()); + + String cluster; + if (args.length != 2) { + cluster = Main.getCluster(client, "Invalid args: " + help()); + } else { + cluster = args[1]; + } + + List<String> shardServerList = client.shardServerList(cluster); + + ConsoleReader reader = this.getConsoleReader(); + if (reader != null) { + Terminal terminal = reader.getTerminal(); + _height = terminal.getHeight() - 2; + _width = terminal.getWidth() - 2; + try { + reader.setPrompt(""); + reader.clearScreen(); + } catch (IOException e) { + if (Main.debug) { + e.printStackTrace(); + } + } + startCommandWatcher(reader, quit, this); + } + + Map<String, Blur.Iface> shardClients = new HashMap<String, Blur.Iface>(); + for (String sc : shardServerList) { + shardClients.put(sc, BlurClient.getClient(sc)); + } + + int longestServerName = Math.max(getSizeOfLongestKey(shardClients), SHARD_SERVER.length()); + + StringBuilder header = new StringBuilder("%" + longestServerName + "s"); + for (int i = 1; i < labels.length; i++) { + header.append(" %10s"); + } + header.append("%n"); + + do { + if (quit.get()) { + return; + } + out.printf(truncate(header.toString()), labels); + for (Entry<String, Blur.Iface> e : shardClients.entrySet()) { + String shardServer = e.getKey(); + Iface shardClient = e.getValue(); + Object[] cols = new Object[labels.length]; + int c = 0; + cols[c++] = shardServer; + StringBuilder sb = new StringBuilder("%" + longestServerName + "s"); + + Map<String, Metric> metrics = shardClient.metrics(keys); + for (int i = 1; i < labels.length; i++) { + String mn = metricNames.get(labels[i]); + Metric metric = metrics.get(mn); + Map<String, Double> doubleMap = metric.getDoubleMap(); + Double value = doubleMap.get("oneMinuteRate"); + if (value == null) { + value = doubleMap.get("value"); + } + cols[c++] = humanize(value, sizes.contains(mn)); + sb.append(" %10s"); + } + sb.append("%n"); + out.printf(truncate(sb.toString()), cols); + } + out.flush(); + if (reader != null) { + try { + synchronized (this) { + wait(3000); + } + } catch (InterruptedException e) { + return; + } + try { + reader.clearScreen(); + } catch (IOException e) { + if (Main.debug) { + e.printStackTrace(); + } + } + Terminal terminal = reader.getTerminal(); + _height = terminal.getHeight() - 2; + _width = terminal.getWidth() - 2; + } + } while (reader != null); + + } + + private void startCommandWatcher(final ConsoleReader reader, final AtomicBoolean quit, final Object lock) { + Thread thread = new Thread(new Runnable() { + @Override + public void run() { + try { + while (true) { + int readCharacter = reader.readCharacter(); + if (readCharacter == 'q') { + quit.set(true); + synchronized (lock) { + lock.notify(); + } + return; + } + } + } catch (IOException e) { + if (Main.debug) { + e.printStackTrace(); + } + } + } + }); + thread.setDaemon(true); + thread.start(); + } + + private String truncate(String s) { + return s.substring(0, Math.min(_width, s.length())); + } + + private String humanize(double value, boolean size) { + long v = (long) (value / ONE_QUADRILLION); + if (v > 0) { + return String.format("%7.2f%s", value / ONE_QUADRILLION, size ? "Q" : "P"); + } + v = (long) (value / ONE_TRILLION); + if (v > 0) { + return String.format("%7.2f%s", value / ONE_TRILLION, "T"); + } + v = (long) (value / ONE_BILLION); + if (v > 0) { + return String.format("%7.2f%s", value / ONE_BILLION, size ? "B" : "G"); + } + v = (long) (value / ONE_MILLION); + if (v > 0) { + return String.format("%7.2f%s", value / ONE_MILLION, "M"); + } + return String.format("%7.2f", value); + } + + private int getSizeOfLongestKey(Map<String, ?> map) { + int i = 0; + for (String s : map.keySet()) { + int length = s.length(); + if (i < length) { + i = length; + } + } + return i; + } + + @Override + public String help() { + return "top arg; cluster"; + } +}
