ACCUMULO-3715 adjust, document and standardize span receiver configuration properties
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7e72d563 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7e72d563 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7e72d563 Branch: refs/heads/master Commit: 7e72d56331d84500d3887f8f304d1a0668f3aa89 Parents: 281c2f3 Author: Billie Rinaldi <[email protected]> Authored: Wed Apr 8 11:54:22 2015 -0700 Committer: Billie Rinaldi <[email protected]> Committed: Fri Apr 10 17:01:00 2015 -0700 ---------------------------------------------------------------------- .../org/apache/accumulo/core/conf/Property.java | 2 +- .../accumulo/core/trace/CountSampler.java | 3 + .../accumulo/core/trace/DistributedTrace.java | 1 - .../accumulo/core/trace/ProbabilitySampler.java | 3 + .../main/asciidoc/chapters/administration.txt | 58 ++++--- .../accumulo/tracer/AsyncSpanReceiver.java | 15 +- .../apache/accumulo/tracer/TraceTableStats.java | 157 +++++++++++++++++++ 7 files changed, 217 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/7e72d563/core/src/main/java/org/apache/accumulo/core/conf/Property.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index f9d8662..08ba3a2 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -382,7 +382,7 @@ public enum Property { TRACE_SPAN_RECEIVERS("trace.span.receivers", "org.apache.accumulo.tracer.ZooTraceClient", PropertyType.CLASSNAMELIST, "A list of span receiver classes to send trace spans"), TRACE_SPAN_RECEIVER_PREFIX("trace.span.receiver.", null, PropertyType.PREFIX, "Prefix for span receiver configuration properties"), - TRACE_ZK_PATH("trace.span.receiver.zookeeper.path", Constants.ZTRACERS, PropertyType.STRING, "The zookeeper node where tracers are registered"), + TRACE_ZK_PATH("trace.zookeeper.path", Constants.ZTRACERS, PropertyType.STRING, "The zookeeper node where tracers are registered"), TRACE_PORT("trace.port.client", "12234", PropertyType.PORT, "The listening port for the trace server"), TRACE_TABLE("trace.table", "trace", PropertyType.STRING, "The name of the table to store distributed traces"), TRACE_USER("trace.user", "root", PropertyType.STRING, "The name of the user to store distributed traces"), http://git-wip-us.apache.org/repos/asf/accumulo/blob/7e72d563/core/src/main/java/org/apache/accumulo/core/trace/CountSampler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/trace/CountSampler.java b/core/src/main/java/org/apache/accumulo/core/trace/CountSampler.java index 852fa9d..a4f206a 100644 --- a/core/src/main/java/org/apache/accumulo/core/trace/CountSampler.java +++ b/core/src/main/java/org/apache/accumulo/core/trace/CountSampler.java @@ -20,6 +20,9 @@ import org.apache.htrace.HTraceConfiguration; import java.util.Collections; +/** + * This wrapper intended for internal Accumulo tracing makes creating a CountSampler easier. + */ public class CountSampler extends org.apache.htrace.impl.CountSampler { public CountSampler(long frequency) { super(HTraceConfiguration.fromMap(Collections.singletonMap(CountSampler.SAMPLER_FREQUENCY_CONF_KEY, Long.toString(frequency)))); http://git-wip-us.apache.org/repos/asf/accumulo/blob/7e72d563/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java b/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java index c5b6eac..14886f1 100644 --- a/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java +++ b/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java @@ -47,7 +47,6 @@ public class DistributedTrace { public static final String TRACE_HOST_PROPERTY = "trace.host"; public static final String TRACE_SERVICE_PROPERTY = "trace.service"; - public static final String TRACE_QUEUE_SIZE_PROPERTY = "trace.queue.size"; public static final String TRACER_ZK_HOST = "tracer.zookeeper.host"; public static final String TRACER_ZK_TIMEOUT = "tracer.zookeeper.timeout"; http://git-wip-us.apache.org/repos/asf/accumulo/blob/7e72d563/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java b/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java index ff02fd7..7204e38 100644 --- a/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java +++ b/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java @@ -20,6 +20,9 @@ import org.apache.htrace.HTraceConfiguration; import java.util.Collections; +/** + * This wrapper intended for internal Accumulo tracing makes creating a ProbabilitySampler easier. + */ public class ProbabilitySampler extends org.apache.htrace.impl.ProbabilitySampler { public ProbabilitySampler(double d) { super(HTraceConfiguration.fromMap(Collections.singletonMap(ProbabilitySampler.SAMPLER_FRACTION_CONF_KEY, Double.toString(d)))); http://git-wip-us.apache.org/repos/asf/accumulo/blob/7e72d563/docs/src/main/asciidoc/chapters/administration.txt ---------------------------------------------------------------------- diff --git a/docs/src/main/asciidoc/chapters/administration.txt b/docs/src/main/asciidoc/chapters/administration.txt index 0382934..520a07c 100644 --- a/docs/src/main/asciidoc/chapters/administration.txt +++ b/docs/src/main/asciidoc/chapters/administration.txt @@ -495,8 +495,14 @@ the following properties Other tracer configuration properties include - trace.port.client - trace.table + trace.port.client - port tracer listens on + trace.table - table tracer writes to + trace.zookeeper.path - zookeeper path where tracers register + +The zookeeper path is configured to /tracers by default. If +multiple Accumulo instances are sharing the same ZooKeeper +quorum, take care to configure Accumulo with unique values for +this property. ==== Configuring Tracing Traces are collected via SpanReceivers. The default SpanReceiver @@ -510,36 +516,52 @@ comma-separated list, by modifying the property Individual span receivers may require their own configuration parameters, which are grouped under the trace.span.receiver.* -prefix. The ZooTraceClient requires the following property that -indicates where the tracer servers will register themselves in -ZooKeeper. - - trace.span.receiver.zookeeper.path - -This is configured to /tracers by default. If multiple Accumulo -instances are sharing the same ZooKeeper quorum, take care to -configure Accumulo with unique values for this property. +prefix. ZooTraceClient uses the following properties. The first +three properties are populated from other Accumulo properties, +while the remaining ones should be prefixed with +trace.span.receiver. when set in the Accumulo configuration. + + tracer.zookeeper.host - populated from instance.zookeepers + tracer.zookeeper.timeout - populated from instance.zookeeper.timeout + tracer.zookeeper.path - populated from trace.zookeeper.path + tracer.send.timer.millis - timer for flushing send queue (in ms, default 1000) + tracer.queue.size - max queue size (default 5000) + tracer.span.min.ms - minimum span length to store (in ms, default 1) + +Note that to configure an Accumulo client for tracing, including +the Accumulo shell, the client configuration must be given the same +trace.span.receivers, trace.span.receiver.*, and trace.zookeeper.path +properties as the servers have. Hadoop can also be configured to send traces to Accumulo, as of -Hadoop 2.6.0, by setting the following properties in Hadoop's -core-site.xml file (the path property is optional if left as the -default). +Hadoop 2.6.0, by setting properties in Hadoop's core-site.xml +file. Instead of using the trace.span.receiver.* prefix, Hadoop +uses hadoop.htrace.*. The Hadoop configuration does not have +access to Accumulo's properties, so the +hadoop.htrace.tracer.zookeeper.host property must be specified. +The zookeeper timeout defaults to 30000 (30 seconds), and the +zookeeper path defaults to /tracers. An example of configuring +Hadoop to send traces to ZooTraceClient is <property> <name>hadoop.htrace.spanreceiver.classes</name> <value>org.apache.accumulo.core.trace.ZooTraceClient</value> </property> <property> - <name>hadoop.tracer.zookeeper.host</name> + <name>hadoop.htrace.tracer.zookeeper.host</name> <value>zookeeperHost:2181</value> </property> <property> - <name>hadoop.tracer.zookeeper.path</name> + <name>hadoop.htrace.tracer.zookeeper.path</name> <value>/tracers</value> </property> + <property> + <name>hadoop.htrace.tracer.span.min.ms</name> + <value>1</value> + </property> -The accumulo-core, accumulo-trace, and libthrift jars must also -be placed on Hadoop's classpath. +The accumulo-core, accumulo-tracer, accumulo-fate and libthrift +jars must also be placed on Hadoop's classpath. ==== Instrumenting a Client Tracing can be used to measure a client operation, such as a scan, as http://git-wip-us.apache.org/repos/asf/accumulo/blob/7e72d563/server/tracer/src/main/java/org/apache/accumulo/tracer/AsyncSpanReceiver.java ---------------------------------------------------------------------- diff --git a/server/tracer/src/main/java/org/apache/accumulo/tracer/AsyncSpanReceiver.java b/server/tracer/src/main/java/org/apache/accumulo/tracer/AsyncSpanReceiver.java index dbcb335..34a75d5 100644 --- a/server/tracer/src/main/java/org/apache/accumulo/tracer/AsyncSpanReceiver.java +++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/AsyncSpanReceiver.java @@ -51,7 +51,9 @@ public abstract class AsyncSpanReceiver<SpanKey,Destination> implements SpanRece private static final Logger log = LoggerFactory.getLogger(AsyncSpanReceiver.class); - public static final String SEND_TIMER_MILLIS = "send.timer.millis"; + public static final String SEND_TIMER_MILLIS = "tracer.send.timer.millis"; + public static final String QUEUE_SIZE = "tracer.queue.size"; + private static final String SPAN_MIN_MS = "tracer.span.min.ms"; private final Map<SpanKey,Destination> clients = new HashMap<SpanKey,Destination>(); @@ -68,6 +70,7 @@ public abstract class AsyncSpanReceiver<SpanKey,Destination> implements SpanRece protected final AbstractQueue<RemoteSpan> sendQueue = new ConcurrentLinkedQueue<RemoteSpan>(); int maxQueueSize = 5000; long lastNotificationOfDroppedSpans = 0; + int minSpanSize = 1; // Visible for testing AsyncSpanReceiver() {} @@ -82,7 +85,8 @@ public abstract class AsyncSpanReceiver<SpanKey,Destination> implements SpanRece } } service = conf.get(DistributedTrace.TRACE_SERVICE_PROPERTY, service); - maxQueueSize = conf.getInt(DistributedTrace.TRACE_QUEUE_SIZE_PROPERTY, maxQueueSize); + maxQueueSize = conf.getInt(QUEUE_SIZE, maxQueueSize); + minSpanSize = conf.getInt(SPAN_MIN_MS, minSpanSize); int millis = conf.getInt(SEND_TIMER_MILLIS, 1000); timer.schedule(new TimerTask() { @@ -102,6 +106,13 @@ public abstract class AsyncSpanReceiver<SpanKey,Destination> implements SpanRece while (!sendQueue.isEmpty()) { boolean sent = false; RemoteSpan s = sendQueue.peek(); + if (s.stop - s.start < minSpanSize) { + synchronized (sendQueue) { + sendQueue.remove(); + sendQueue.notifyAll(); + } + continue; + } SpanKey dest = getSpanKey(s.data); Destination client = clients.get(dest); if (client == null) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/7e72d563/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceTableStats.java ---------------------------------------------------------------------- diff --git a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceTableStats.java b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceTableStats.java new file mode 100644 index 0000000..63c806f --- /dev/null +++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceTableStats.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.tracer; + +import org.apache.accumulo.core.cli.ClientOnDefaultTable; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.tracer.thrift.RemoteSpan; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; + +/** + * Reads the trace table and prints out some stats about the spans found. + */ +public class TraceTableStats { + static class Opts extends ClientOnDefaultTable { + public Opts() { + super("trace"); + } + } + + static class SpanTypeCount { + String type; + long nonzeroCount = 0l; + long zeroCount = 0l; + ArrayList<Long> log10SpanLength = new ArrayList<>(); + Set<Long> traceIds = new HashSet<>(); + + public SpanTypeCount() { + for (int i = 0; i < 7; i++) + log10SpanLength.add(0l); + } + + @Override + public String toString() { + return "{" + + "type='" + type + '\'' + + ", nonzeroCount=" + nonzeroCount + + ", zeroCount=" + zeroCount + + ", numTraces=" + traceIds.size() + + ", log10SpanLength=" + log10SpanLength + + '}'; + } + } + + public static void main(String[] args) throws Exception { + TraceTableStats stats = new TraceTableStats(); + Opts opts = new Opts(); + opts.parseArgs(TraceTableStats.class.getName(), args); + stats.count(opts); + } + + public void count(Opts opts) throws AccumuloSecurityException, AccumuloException, TableNotFoundException { + Connector conn = opts.getConnector(); + Scanner scanner = conn.createScanner(opts.getTableName(), Authorizations.EMPTY); + scanner.setRange(new Range(null, true, "idx:", false)); + Map<String, SpanTypeCount> counts = new TreeMap<>(); + final SpanTypeCount hdfs = new SpanTypeCount(); + hdfs.type = "HDFS"; + final SpanTypeCount accumulo = new SpanTypeCount(); + accumulo.type = "Accumulo"; + long numSpans = 0; + double maxSpanLength = 0; + double maxSpanLengthMS = 0; + for (Entry<Key,Value> entry : scanner) { + numSpans++; + RemoteSpan span = TraceFormatter.getRemoteSpan(entry); + String id = span.getSvc()+":"+span.getDescription().replaceAll("[0-9][0-9][0-9]+", ""); + SpanTypeCount stc = counts.get(id); + if (stc == null) { + stc = new SpanTypeCount(); + counts.put(id, stc); + if (span.description.startsWith("org.apache.hadoop") || span.svc.equals("NameNode") || span.svc.equals("DataNode") + || span.description.contains("DFSOutputStream") || span.description.contains("DFSInputStream") + || span.description.contains("BlockReader")) { + stc.type = hdfs.type; + } else { + stc.type = accumulo.type; + } + } + increment(stc, span); + if (stc.type.equals(hdfs.type)) { + increment(hdfs, span); + } else { + increment(accumulo, span); + } + maxSpanLength = Math.max(maxSpanLength, Math.log10(span.stop-span.start)); + maxSpanLengthMS = Math.max(maxSpanLengthMS, span.stop-span.start); + } + System.out.println(); + System.out.println("log10 max span length "+maxSpanLength+" "+maxSpanLengthMS); + System.out.println("Total spans "+numSpans); + System.out.println("Percentage Accumulo nonzero of total " + accumulo.nonzeroCount + "/" + numSpans + " = " + (accumulo.nonzeroCount * 1.0 / numSpans)); + System.out.println(hdfs + ", total " + (hdfs.nonzeroCount + hdfs.zeroCount)); + System.out.println(accumulo+ ", total " + (accumulo.nonzeroCount + accumulo.zeroCount)); + System.out.println(); + System.out.println("source:desc={stats}"); + for (Entry<String, SpanTypeCount> c : counts.entrySet()) { + System.out.println(c); + } + } + + private static void increment(SpanTypeCount stc, RemoteSpan span) { + stc.traceIds.add(span.getTraceId()); + if (span.stop == span.start) { + stc.zeroCount++; + incrementIndex(stc.log10SpanLength, 0); + } else { + stc.nonzeroCount++; + long ms = span.stop - span.start; + if (ms <= 10) + incrementIndex(stc.log10SpanLength, 1); + else if (ms <= 100) + incrementIndex(stc.log10SpanLength, 2); + else if (ms <= 1000) + incrementIndex(stc.log10SpanLength, 3); + else if (ms <= 10000) + incrementIndex(stc.log10SpanLength, 4); + else if (ms <= 100000) + incrementIndex(stc.log10SpanLength, 5); + else if (ms <= 1000000) + incrementIndex(stc.log10SpanLength, 6); + else + throw new IllegalArgumentException("unexpected span length "+ms); + } + } + + private static void incrementIndex(ArrayList<Long> hist, int i) { + hist.set(i, hist.get(i) + 1); + } +}
