Repository: accumulo Updated Branches: refs/heads/master 9c8dcaf0a -> 3a99300a2
ACCUMULO-3715 add configurable sampling to internal tracing Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/281c2f32 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/281c2f32 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/281c2f32 Branch: refs/heads/master Commit: 281c2f3243bf3f90043ca8f17f7bed6a01b60297 Parents: 9c8dcaf Author: Billie Rinaldi <billie.rina...@gmail.com> Authored: Wed Apr 8 08:35:32 2015 -0700 Committer: Billie Rinaldi <billie.rina...@gmail.com> Committed: Fri Apr 10 10:09:28 2015 -0700 ---------------------------------------------------------------------- .../org/apache/accumulo/core/conf/Property.java | 7 +++++ .../accumulo/core/trace/ProbabilitySampler.java | 27 ++++++++++++++++++++ .../accumulo/gc/SimpleGarbageCollector.java | 4 +-- .../master/replication/ReplicationDriver.java | 4 +-- .../replication/AccumuloReplicaSystem.java | 5 +++- .../tserver/tablet/MinorCompactionTask.java | 8 ++++-- .../apache/accumulo/tserver/tablet/Tablet.java | 10 +++++--- 7 files changed, 55 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/281c2f32/core/src/main/java/org/apache/accumulo/core/conf/Property.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index 9dceb1e..f9d8662 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -275,6 +275,10 @@ public enum Property { "The maximum number of concurrent major compactions for a tablet server"), TSERV_MINC_MAXCONCURRENT("tserver.compaction.minor.concurrent.max", "4", PropertyType.COUNT, "The maximum number of concurrent minor compactions for a tablet server"), + TSERV_MAJC_TRACE_PERCENT("tserver.compaction.major.trace.percent", "0.1", PropertyType.FRACTION, + "The percent of major compactions to trace"), + TSERV_MINC_TRACE_PERCENT("tserver.compaction.minor.trace.percent", "0.1", PropertyType.FRACTION, + "The percent of minor compactions to trace"), TSERV_COMPACTION_WARN_TIME("tserver.compaction.warn.time", "10m", PropertyType.TIMEDURATION, "When a compaction has not made progress for this time period, a warning will be logged"), TSERV_BLOOM_LOAD_MAXCONCURRENT("tserver.bloom.load.concurrent.max", "4", PropertyType.COUNT, @@ -344,6 +348,7 @@ public enum Property { GC_DELETE_THREADS("gc.threads.delete", "16", PropertyType.COUNT, "The number of threads used to delete files"), GC_TRASH_IGNORE("gc.trash.ignore", "false", PropertyType.BOOLEAN, "Do not use the Trash, even if it is configured"), GC_FILE_ARCHIVE("gc.file.archive", "false", PropertyType.BOOLEAN, "Archive any files/directories instead of moving to the HDFS trash or deleting"), + GC_TRACE_PERCENT("gc.trace.percent", "0.01", PropertyType.FRACTION, "Percent of gc cycles to trace"), // properties that are specific to the monitor server behavior MONITOR_PREFIX("monitor.", null, PropertyType.PREFIX, "Properties in this category affect the behavior of the monitor web server."), @@ -556,6 +561,8 @@ public enum Property { "Amount of time to wait before first checking for replication work, not useful outside of tests"), REPLICATION_WORK_PROCESSOR_PERIOD("replication.work.processor.period", "0s", PropertyType.TIMEDURATION, "Amount of time to wait before re-checking for replication work, not useful outside of tests"), + REPLICATION_TRACE_PERCENT("replication.trace.percent", "0.1", PropertyType.FRACTION, + "The sampling percentage to use for replication traces"), ; http://git-wip-us.apache.org/repos/asf/accumulo/blob/281c2f32/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java b/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java new file mode 100644 index 0000000..ff02fd7 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.trace; + +import org.apache.htrace.HTraceConfiguration; + +import java.util.Collections; + +public class ProbabilitySampler extends org.apache.htrace.impl.ProbabilitySampler { + public ProbabilitySampler(double d) { + super(HTraceConfiguration.fromMap(Collections.singletonMap(ProbabilitySampler.SAMPLER_FRACTION_CONF_KEY, Double.toString(d)))); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/281c2f32/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java ---------------------------------------------------------------------- diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java index 04f21a1..ffa59cd 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java @@ -64,8 +64,8 @@ import org.apache.accumulo.core.replication.ReplicationTable; import org.apache.accumulo.core.replication.ReplicationTableOfflineException; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.thrift.TCredentials; -import org.apache.accumulo.core.trace.CountSampler; import org.apache.accumulo.core.trace.DistributedTrace; +import org.apache.accumulo.core.trace.ProbabilitySampler; import org.apache.accumulo.core.trace.Span; import org.apache.accumulo.core.trace.Trace; import org.apache.accumulo.core.trace.thrift.TInfo; @@ -526,7 +526,7 @@ public class SimpleGarbageCollector extends AccumuloServerContext implements Ifa return; } - CountSampler sampler = new CountSampler(100); + ProbabilitySampler sampler = new ProbabilitySampler(getConfiguration().getFraction(Property.GC_TRACE_PERCENT)); while (true) { Trace.on("gc", sampler); http://git-wip-us.apache.org/repos/asf/accumulo/blob/281c2f32/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java ---------------------------------------------------------------------- diff --git a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java index 63c6b20..72b3bbd 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java +++ b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java @@ -21,7 +21,7 @@ import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.trace.CountSampler; +import org.apache.accumulo.core.trace.ProbabilitySampler; import org.apache.accumulo.core.trace.Trace; import org.apache.accumulo.core.util.Daemon; import org.apache.accumulo.fate.util.UtilWaitThread; @@ -53,7 +53,7 @@ public class ReplicationDriver extends Daemon { @Override public void run() { - CountSampler sampler = new CountSampler(10); + ProbabilitySampler sampler = new ProbabilitySampler(conf.getFraction(Property.REPLICATION_TRACE_PERCENT)); long millisToWait = conf.getTimeInMillis(Property.REPLICATION_DRIVER_DELAY); log.debug("Waiting " + millisToWait + "ms before starting main replication loop"); http://git-wip-us.apache.org/repos/asf/accumulo/blob/281c2f32/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java index cb8ae13..d18e434 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java @@ -55,6 +55,7 @@ import org.apache.accumulo.core.replication.thrift.ReplicationServicer.Client; import org.apache.accumulo.core.replication.thrift.WalEdits; import org.apache.accumulo.core.security.Credentials; import org.apache.accumulo.core.security.thrift.TCredentials; +import org.apache.accumulo.core.trace.ProbabilitySampler; import org.apache.accumulo.core.trace.Span; import org.apache.accumulo.core.trace.Trace; import org.apache.accumulo.core.util.UtilWaitThread; @@ -232,7 +233,9 @@ public class AccumuloReplicaSystem implements ReplicaSystem { private Status _replicate(final Path p, final Status status, final ReplicationTarget target, final ReplicaSystemHelper helper, final AccumuloConfiguration localConf, final ClientContext peerContext, final UserGroupInformation accumuloUgi) { try { - Trace.on("AccumuloReplicaSystem"); + double tracePercent = localConf.getFraction(Property.REPLICATION_TRACE_PERCENT); + ProbabilitySampler sampler = new ProbabilitySampler(tracePercent); + Trace.on("AccumuloReplicaSystem", sampler); // Remote identifier is an integer (table id) in this case. final String remoteTableId = target.getRemoteIdentifier(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/281c2f32/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java index 0f6a98d..53bf5c2 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java @@ -19,6 +19,7 @@ package org.apache.accumulo.tserver.tablet; import java.io.IOException; import org.apache.accumulo.core.metadata.schema.DataFileValue; +import org.apache.accumulo.core.trace.ProbabilitySampler; import org.apache.accumulo.core.trace.Span; import org.apache.accumulo.core.trace.Trace; import org.apache.accumulo.server.fs.FileRef; @@ -38,8 +39,9 @@ class MinorCompactionTask implements Runnable { private FileRef mergeFile; private long flushId; private MinorCompactionReason mincReason; + private double tracePercent; - MinorCompactionTask(Tablet tablet, FileRef mergeFile, CommitSession commitSession, long flushId, MinorCompactionReason mincReason) { + MinorCompactionTask(Tablet tablet, FileRef mergeFile, CommitSession commitSession, long flushId, MinorCompactionReason mincReason, double tracePercent) { this.tablet = tablet; queued = System.currentTimeMillis(); tablet.minorCompactionWaitingToStart(); @@ -47,12 +49,14 @@ class MinorCompactionTask implements Runnable { this.mergeFile = mergeFile; this.flushId = flushId; this.mincReason = mincReason; + this.tracePercent = tracePercent; } @Override public void run() { tablet.minorCompactionStarted(); - Span minorCompaction = Trace.on("minorCompaction"); + ProbabilitySampler sampler = new ProbabilitySampler(tracePercent); + Span minorCompaction = Trace.on("minorCompaction", sampler); try { FileRef newMapfileLocation = tablet.getNextMapFilename(mergeFile == null ? "F" : "M"); FileRef tmpFileRef = new FileRef(newMapfileLocation.path() + "_tmp"); http://git-wip-us.apache.org/repos/asf/accumulo/blob/281c2f32/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java index 43bd9d9..2342789 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java @@ -82,6 +82,7 @@ import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.tabletserver.thrift.TabletStats; +import org.apache.accumulo.core.trace.ProbabilitySampler; import org.apache.accumulo.core.trace.Span; import org.apache.accumulo.core.trace.Trace; import org.apache.accumulo.core.util.LocalityGroupUtil; @@ -996,7 +997,9 @@ public class Tablet implements TabletCommitter { mergeFile = getDatafileManager().reserveMergingMinorCompactionFile(); } - return new MinorCompactionTask(this, mergeFile, oldCommitSession, flushId, mincReason); + double tracePercent = tabletServer.getConfiguration().getFraction(Property.TSERV_MINC_TRACE_PERCENT); + + return new MinorCompactionTask(this, mergeFile, oldCommitSession, flushId, mincReason, tracePercent); } @@ -2075,8 +2078,9 @@ public class Tablet implements TabletCommitter { Span span = null; try { - // Always trace majC - span = Trace.on("majorCompaction"); + double tracePercent = tabletServer.getConfiguration().getFraction(Property.TSERV_MAJC_TRACE_PERCENT); + ProbabilitySampler sampler = new ProbabilitySampler(tracePercent); + span = Trace.on("majorCompaction", sampler); majCStats = _majorCompact(reason); if (reason == MajorCompactionReason.CHOP) {
