Repository: incubator-htrace Updated Branches: refs/heads/master 699c8cf80 -> 26518585d
HTRACE-303. Add client-side htraceDropped log file to track dropped spans (cmccabe) Project: http://git-wip-us.apache.org/repos/asf/incubator-htrace/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-htrace/commit/26518585 Tree: http://git-wip-us.apache.org/repos/asf/incubator-htrace/tree/26518585 Diff: http://git-wip-us.apache.org/repos/asf/incubator-htrace/diff/26518585 Branch: refs/heads/master Commit: 26518585d52c3b89f5967fa3776a075c700b6831 Parents: 699c8cf Author: Colin P. Mccabe <[email protected]> Authored: Sat Nov 21 13:27:23 2015 -0800 Committer: Colin P. Mccabe <[email protected]> Committed: Sat Nov 21 13:27:23 2015 -0800 ---------------------------------------------------------------------- .../go/src/org/apache/htrace/common/rpc.go | 19 -- .../src/org/apache/htrace/htraced/datastore.go | 4 +- .../org/apache/htrace/htraced/datastore_test.go | 4 +- .../go/src/org/apache/htrace/htraced/hrpc.go | 2 +- .../go/src/org/apache/htrace/htraced/metrics.go | 15 +- .../org/apache/htrace/htraced/reaper_test.go | 2 +- .../go/src/org/apache/htrace/htraced/rest.go | 2 +- .../go/src/org/apache/htrace/htracedTool/cmd.go | 6 +- .../main/java/org/apache/htrace/impl/Conf.java | 24 +++ .../apache/htrace/impl/HTracedSpanReceiver.java | 91 ++++++++-- .../apache/htrace/impl/TestDroppedSpans.java | 173 +++++++++++++++++++ .../src/main/webapp/app/server_info_view.js | 2 - .../src/main/webapp/app/server_stats.js | 1 - htrace-webapp/src/main/webapp/index.html | 4 - 14 files changed, 291 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/go/src/org/apache/htrace/common/rpc.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/rpc.go b/htrace-htraced/go/src/org/apache/htrace/common/rpc.go index 6375688..8028cc6 100644 --- a/htrace-htraced/go/src/org/apache/htrace/common/rpc.go +++ b/htrace-htraced/go/src/org/apache/htrace/common/rpc.go @@ -41,7 +41,6 @@ type WriteSpansReq struct { Addr string `json:",omitempty"` // This gets filled in by the RPC layer. DefaultTrid string `json:",omitempty"` Spans []*Span - ClientDropped uint64 `json:",omitempty"` } // Info returned by /server/version @@ -97,20 +96,6 @@ type SpanMetrics struct { // The total number of spans dropped by the server. ServerDropped uint64 - - // The total number of spans dropped by the client. - // - // This number is just an estimate and may be incorrect for many reasons. - // If the client can't contact the server at all, then obviously the server - // will never increment ClientDropped... even though spans are being - // dropped. The client may also tell the server about some new spans it - // has dropped, but then for some reason fail to get the acknowledgement - // from the server. In that case, the client would re-send its client - // dropped estimate and it would be double-counted by the server - // - // The intention here is to provide a rough estimate of how overloaded - // htraced clients are, not to provide strongly consistent numbers. - ClientDroppedEstimate uint64 } // A map from network address strings to SpanMetrics structures. @@ -145,10 +130,6 @@ type ServerStats struct { // The total number of spans dropped by the server since the server started. ServerDroppedSpans uint64 - // An estimate of the total number of spans dropped by the server since the server started. - // See SpanMetrics#ClientDroppedEstimate - ClientDroppedEstimate uint64 - // The maximum latency of a writeSpans request, in milliseconds. MaxWriteSpansLatencyMs uint32 http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go index 9310e6e..1dab5c8 100644 --- a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go +++ b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go @@ -792,7 +792,7 @@ func (ing *SpanIngestor) IngestSpan(span *common.Span) { } } -func (ing *SpanIngestor) Close(clientDropped int, startTime time.Time) { +func (ing *SpanIngestor) Close(startTime time.Time) { for shardIdx := range(ing.batches) { batch := ing.batches[shardIdx] if len(batch.incoming) > 0 { @@ -809,7 +809,7 @@ func (ing *SpanIngestor) Close(clientDropped int, startTime time.Time) { endTime := time.Now() ing.store.msink.UpdateIngested(ing.addr, ing.totalIngested, - ing.serverDropped, clientDropped, endTime.Sub(startTime)) + ing.serverDropped, endTime.Sub(startTime)) } func (store *dataStore) WriteSpans(shardIdx int, ispans []*IncomingSpan) { http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/go/src/org/apache/htrace/htraced/datastore_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore_test.go b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore_test.go index e6d1df7..d38c1b0 100644 --- a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore_test.go +++ b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore_test.go @@ -77,7 +77,7 @@ func createSpans(spans []common.Span, store *dataStore) { for idx := range spans { ing.IngestSpan(&spans[idx]) } - ing.Close(0, time.Now()) + ing.Close(time.Now()) store.WrittenSpans.Waits(int64(len(spans))) } @@ -364,7 +364,7 @@ func BenchmarkDatastoreWrites(b *testing.B) { for n := 0; n < b.N; n++ { ing.IngestSpan(allSpans[n]) } - ing.Close(0, time.Now()) + ing.Close(time.Now()) // Wait for all the spans to be written. ht.Store.WrittenSpans.Waits(int64(b.N)) assertNumWrittenEquals(b, ht.Store.msink, b.N) http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/go/src/org/apache/htrace/htraced/hrpc.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/hrpc.go b/htrace-htraced/go/src/org/apache/htrace/htraced/hrpc.go index a649420..0d569a0 100644 --- a/htrace-htraced/go/src/org/apache/htrace/htraced/hrpc.go +++ b/htrace-htraced/go/src/org/apache/htrace/htraced/hrpc.go @@ -270,7 +270,7 @@ func (hand *HrpcHandler) WriteSpans(req *common.WriteSpansReq, for spanIdx := range req.Spans { ing.IngestSpan(req.Spans[spanIdx]) } - ing.Close(int(req.ClientDropped), startTime) + ing.Close(startTime) return nil } http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/go/src/org/apache/htrace/htraced/metrics.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/metrics.go b/htrace-htraced/go/src/org/apache/htrace/htraced/metrics.go index 5ce3339..7bf42fd 100644 --- a/htrace-htraced/go/src/org/apache/htrace/htraced/metrics.go +++ b/htrace-htraced/go/src/org/apache/htrace/htraced/metrics.go @@ -55,9 +55,6 @@ type MetricsSink struct { // The total number of spans dropped by the server. ServerDropped uint64 - // The total number of spans dropped by the client (self-reported). - ClientDroppedEstimate uint64 - // Per-host Span Metrics HostSpanMetrics common.SpanMetricsMap @@ -80,13 +77,12 @@ func NewMetricsSink(cnf *conf.Config) *MetricsSink { // Update the total number of spans which were ingested, as well as other // metrics that get updated during span ingest. func (msink *MetricsSink) UpdateIngested(addr string, totalIngested int, - serverDropped int, clientDroppedEstimate int, wsLatency time.Duration) { + serverDropped int, wsLatency time.Duration) { msink.lock.Lock() defer msink.lock.Unlock() msink.IngestedSpans += uint64(totalIngested) msink.ServerDropped += uint64(serverDropped) - msink.ClientDroppedEstimate += uint64(clientDroppedEstimate) - msink.updateSpanMetrics(addr, 0, serverDropped, clientDroppedEstimate) + msink.updateSpanMetrics(addr, 0, serverDropped) wsLatencyMs := wsLatency.Nanoseconds() / 1000000 var wsLatency32 uint32 if wsLatencyMs > math.MaxUint32 { @@ -99,7 +95,7 @@ func (msink *MetricsSink) UpdateIngested(addr string, totalIngested int, // Update the per-host span metrics. Must be called with the lock held. func (msink *MetricsSink) updateSpanMetrics(addr string, numWritten int, - serverDropped int, clientDroppedEstimate int) { + serverDropped int) { mtx, found := msink.HostSpanMetrics[addr] if !found { // Ensure that the per-host span metrics map doesn't grow too large. @@ -117,7 +113,6 @@ func (msink *MetricsSink) updateSpanMetrics(addr string, numWritten int, } mtx.Written += uint64(numWritten) mtx.ServerDropped += uint64(serverDropped) - mtx.ClientDroppedEstimate += uint64(clientDroppedEstimate) } // Update the total number of spans which were persisted to disk. @@ -127,7 +122,7 @@ func (msink *MetricsSink) UpdatePersisted(addr string, totalWritten int, defer msink.lock.Unlock() msink.WrittenSpans += uint64(totalWritten) msink.ServerDropped += uint64(serverDropped) - msink.updateSpanMetrics(addr, totalWritten, serverDropped, 0) + msink.updateSpanMetrics(addr, totalWritten, serverDropped) } // Read the server stats. @@ -137,7 +132,6 @@ func (msink *MetricsSink) PopulateServerStats(stats *common.ServerStats) { stats.IngestedSpans = msink.IngestedSpans stats.WrittenSpans = msink.WrittenSpans stats.ServerDroppedSpans = msink.ServerDropped - stats.ClientDroppedEstimate = msink.ClientDroppedEstimate stats.MaxWriteSpansLatencyMs = msink.wsLatencyCircBuf.Max() stats.AverageWriteSpansLatencyMs = msink.wsLatencyCircBuf.Average() stats.HostSpanMetrics = make(common.SpanMetricsMap) @@ -145,7 +139,6 @@ func (msink *MetricsSink) PopulateServerStats(stats *common.ServerStats) { stats.HostSpanMetrics[k] = &common.SpanMetrics { Written: v.Written, ServerDropped: v.ServerDropped, - ClientDroppedEstimate: v.ClientDroppedEstimate, } } } http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/go/src/org/apache/htrace/htraced/reaper_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/reaper_test.go b/htrace-htraced/go/src/org/apache/htrace/htraced/reaper_test.go index 0140dbb..b354a2c 100644 --- a/htrace-htraced/go/src/org/apache/htrace/htraced/reaper_test.go +++ b/htrace-htraced/go/src/org/apache/htrace/htraced/reaper_test.go @@ -56,7 +56,7 @@ func TestReapingOldSpans(t *testing.T) { for spanIdx := range testSpans { ing.IngestSpan(testSpans[spanIdx]) } - ing.Close(0, time.Now()) + ing.Close(time.Now()) // Wait the spans to be created ht.Store.WrittenSpans.Waits(NUM_TEST_SPANS) // Set a reaper date that will remove all the spans except final one. http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/go/src/org/apache/htrace/htraced/rest.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/rest.go b/htrace-htraced/go/src/org/apache/htrace/htraced/rest.go index 1b90bd4..c327cdd 100644 --- a/htrace-htraced/go/src/org/apache/htrace/htraced/rest.go +++ b/htrace-htraced/go/src/org/apache/htrace/htraced/rest.go @@ -257,7 +257,7 @@ func (hand *writeSpansHandler) ServeHTTP(w http.ResponseWriter, req *http.Reques for spanIdx := range msg.Spans { ing.IngestSpan(msg.Spans[spanIdx]) } - ing.Close(int(msg.ClientDropped), startTime) + ing.Close(startTime) } type queryHandler struct { http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go b/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go index c81bbb7..9837e94 100644 --- a/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go +++ b/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go @@ -218,8 +218,6 @@ func printServerStats(hcl *htrace.Client) int { fmt.Fprintf(w, "Spans ingested\t%d\n", stats.IngestedSpans) fmt.Fprintf(w, "Spans written\t%d\n", stats.WrittenSpans) fmt.Fprintf(w, "Spans dropped by server\t%d\n", stats.ServerDroppedSpans) - fmt.Fprintf(w, "Estimated spans dropped by clients\t%d\n", - stats.ClientDroppedEstimate) dur := time.Millisecond * time.Duration(stats.AverageWriteSpansLatencyMs) fmt.Fprintf(w, "Average WriteSpan Latency\t%s\n", dur.String()) dur = time.Millisecond * time.Duration(stats.MaxWriteSpansLatencyMs) @@ -247,8 +245,8 @@ func printServerStats(hcl *htrace.Client) int { sort.Sort(keys) for k := range keys { mtx := mtxMap[keys[k]] - fmt.Fprintf(w, "%s\twritten: %d\tserver dropped: %d\tclient dropped estimate: %d\n", - keys[k], mtx.Written, mtx.ServerDropped, mtx.ClientDroppedEstimate) + fmt.Fprintf(w, "%s\twritten: %d\tserver dropped: %d\n", + keys[k], mtx.Written, mtx.ServerDropped) } w.Flush() return EXIT_SUCCESS http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/src/main/java/org/apache/htrace/impl/Conf.java ---------------------------------------------------------------------- diff --git a/htrace-htraced/src/main/java/org/apache/htrace/impl/Conf.java b/htrace-htraced/src/main/java/org/apache/htrace/impl/Conf.java index cdd176f..3206dd6 100644 --- a/htrace-htraced/src/main/java/org/apache/htrace/impl/Conf.java +++ b/htrace-htraced/src/main/java/org/apache/htrace/impl/Conf.java @@ -18,6 +18,7 @@ package org.apache.htrace.impl; import java.io.IOException; +import java.io.File; import java.net.InetSocketAddress; import com.fasterxml.jackson.annotation.JsonProperty; @@ -144,6 +145,18 @@ class Conf { "htraced.error.log.period.ms"; final static long ERROR_LOG_PERIOD_MS_DEFAULT = 30000L; + final static String DROPPED_SPANS_LOG_PATH_KEY = + "htraced.dropped.spans.log.path"; + + final static String DROPPED_SPANS_LOG_PATH_DEFAULT = + new File(System.getProperty("java.io.tmpdir", "/tmp"), "htraceDropped"). + getAbsolutePath(); + + final static String DROPPED_SPANS_LOG_MAX_SIZE_KEY = + "htraced.dropped.spans.log.max.size"; + + final static long DROPPED_SPANS_LOG_MAX_SIZE_DEFAULT = 1024L * 1024L; + @JsonProperty("ioTimeoutMs") final int ioTimeoutMs; @@ -180,6 +193,12 @@ class Conf { @JsonProperty("endpoint") final InetSocketAddress endpoint; + @JsonProperty("droppedSpansLogPath") + final String droppedSpansLogPath; + + @JsonProperty("droppedSpansLogMaxSize") + final long droppedSpansLogMaxSize; + private static int getBoundedInt(final HTraceConfiguration conf, String key, int defaultValue, int minValue, int maxValue) { int val = conf.getInt(key, defaultValue); @@ -341,6 +360,11 @@ class Conf { throw new IOException("Error reading " + ADDRESS_KEY + ": " + e.getMessage()); } + this.droppedSpansLogPath = conf.get( + DROPPED_SPANS_LOG_PATH_KEY, DROPPED_SPANS_LOG_PATH_DEFAULT); + this.droppedSpansLogMaxSize = getBoundedLong(conf, + DROPPED_SPANS_LOG_MAX_SIZE_KEY, DROPPED_SPANS_LOG_MAX_SIZE_DEFAULT, + 0, Long.MAX_VALUE); } @Override http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedSpanReceiver.java ---------------------------------------------------------------------- diff --git a/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedSpanReceiver.java b/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedSpanReceiver.java index f5f493c..22b64f6 100644 --- a/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedSpanReceiver.java +++ b/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedSpanReceiver.java @@ -17,7 +17,20 @@ */ package org.apache.htrace.impl; +import static java.nio.file.StandardOpenOption.APPEND; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.WRITE; + import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +//import java.nio.file.attribute.FileAttribute; +//import java.util.EnumSet; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; @@ -75,6 +88,8 @@ public class HTracedSpanReceiver extends SpanReceiver { private long lastBufferClearedTimeMs = 0; + private long unbufferableSpans = 0; + static class FaultInjector { static FaultInjector NO_OP = new FaultInjector(); public void handleContentLengthTrigger(int len) { } @@ -140,10 +155,12 @@ public class HTracedSpanReceiver extends SpanReceiver { } long deltaMs = TimeUtil.deltaMs(startTimeMs, TimeUtil.nowMs()); if (deltaMs > conf.spanDropTimeoutMs) { + StringBuilder bld = new StringBuilder(); spanDropLog.error("Dropping a span after unsuccessfully " + "attempting to add it for " + deltaMs + " ms. There is not " + "enough buffer space. Please increase " + Conf.BUFFER_SIZE_KEY + " or decrease the rate of spans being generated."); + unbufferableSpans++; return; } else if (LOG.isDebugEnabled()) { LOG.debug("Unable to write span to buffer #" + activeBuf + @@ -296,6 +313,16 @@ public class HTracedSpanReceiver extends SpanReceiver { return; } int flushTries = 0; + if (unbufferableSpans > 0) { + try { + appendToDroppedSpansLog("Dropped " + unbufferableSpans + + " spans because of lack of local buffer space.\n"); + } catch (IOException e) { + // Ignore. We already logged a message about the dropped spans + // earlier. + } + unbufferableSpans = 0; + } while (true) { Throwable exc; try { @@ -311,17 +338,22 @@ public class HTracedSpanReceiver extends SpanReceiver { return; } int numSpans = flushBufManager.getNumberOfSpans(); - String excMessage = "Failed to flush " + numSpans + " htrace " + - "spans to " + conf.endpointStr + " on try " + (flushTries + 1); - if (flushTries >= conf.flushRetryDelays.length) { - excMessage += ". Discarding all spans."; - } - if (LOG.isDebugEnabled()) { - LOG.error(excMessage, exc); - } else { - flushErrorLog.error(excMessage, exc); - } + flushErrorLog.error("Failed to flush " + numSpans + " htrace " + + "spans to " + conf.endpointStr + " on try " + (flushTries + 1), + exc); if (flushTries >= conf.flushRetryDelays.length) { + StringBuilder bld = new StringBuilder(); + bld.append("Failed to flush ").append(numSpans). + append(" spans to htraced at").append(conf.endpointStr). + append(" after ").append(flushTries).append(" tries: "). + append(exc.getMessage()); + try { + appendToDroppedSpansLog(bld.toString()); + } catch (IOException e) { + bld.append(". Failed to write to dropped spans log: "). + append(e.getMessage()); + } + spanDropLog.error(bld.toString()); return; } int delayMs = conf.flushRetryDelays[flushTries]; @@ -330,4 +362,43 @@ public class HTracedSpanReceiver extends SpanReceiver { } } } + + void appendToDroppedSpansLog(String text) throws IOException { + // Is the dropped spans log is disabled? + if (conf.droppedSpansLogPath.isEmpty() || + (conf.droppedSpansLogMaxSize == 0)) { + return; + } + FileLock lock = null; + ByteBuffer bb = ByteBuffer.wrap( + text.getBytes(StandardCharsets.UTF_8)); + // FileChannel locking corresponds to advisory locking on UNIX. It will + // protect multiple processes from attempting to write to the same dropped + // spans log at once. However, within a single process, we need this + // synchronized block to ensure that multiple HTracedSpanReceiver objects + // don't try to write to the same log at once. (It is unusal to configure + // multiple HTracedSpanReceiver objects, but possible.) + synchronized(HTracedSpanReceiver.class) { + FileChannel channel = FileChannel.open( + Paths.get(conf.droppedSpansLogPath), APPEND, CREATE, WRITE); + try { + lock = channel.lock(); + long size = channel.size(); + if (size > conf.droppedSpansLogMaxSize) { + throw new IOException("Dropped spans log " + + conf.droppedSpansLogPath + " is already " + size + + " bytes; will not add to it."); + } + channel.write(bb); + } finally { + try { + if (lock != null) { + lock.release(); + } + } finally { + channel.close(); + } + } + } + } } http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/src/test/java/org/apache/htrace/impl/TestDroppedSpans.java ---------------------------------------------------------------------- diff --git a/htrace-htraced/src/test/java/org/apache/htrace/impl/TestDroppedSpans.java b/htrace-htraced/src/test/java/org/apache/htrace/impl/TestDroppedSpans.java new file mode 100644 index 0000000..8947365 --- /dev/null +++ b/htrace-htraced/src/test/java/org/apache/htrace/impl/TestDroppedSpans.java @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.htrace.impl; + +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.net.ServerSocket; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.UUID; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.htrace.core.HTraceConfiguration; +import org.apache.htrace.core.MilliSpan; +import org.apache.htrace.core.TracerId; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestDroppedSpans { + private static final Log LOG = LogFactory.getLog(TestDroppedSpans.class); + + private static Path tempDir; + + @BeforeClass + public static void beforeClass() throws IOException { + // Allow setting really small buffer sizes for testing purposes. + // We do not allow setting such small sizes in production. + Conf.BUFFER_SIZE_MIN = 0; + + // Create a temporary directory to hold the dropped spans logs. + String tmp = System.getProperty("java.io.tmpdir", "/tmp"); + File dir = new File(tmp, + "TestDroppedSpans." + UUID.randomUUID().toString()); + Files.createDirectory(dir.toPath()); + tempDir = dir.toPath(); + } + + @BeforeClass + public static void afterClass() throws IOException { + if (tempDir != null) { + try (DirectoryStream<Path> stream = Files.newDirectoryStream(tempDir)) { + for (final Iterator<Path> it = stream.iterator(); it.hasNext();) { + Files.delete(it.next()); + } + } + Files.delete(tempDir); + tempDir = null; + } + } + + /** + * Test that we can disable the dropped spans log. + */ + @Test(timeout = 60000) + public void testDisableDroppedSpansLog() throws Exception { + HTraceConfiguration conf = HTraceConfiguration.fromMap( + new HashMap<String, String>() {{ + put(Conf.ADDRESS_KEY, "127.0.0.1:8080"); + put(TracerId.TRACER_ID_KEY, "testAppendToDroppedSpansLog"); + put(Conf.DROPPED_SPANS_LOG_PATH_KEY, "/"); + put(Conf.DROPPED_SPANS_LOG_MAX_SIZE_KEY, "0"); + }}); + HTracedSpanReceiver rcvr = new HTracedSpanReceiver(conf); + try { + rcvr.appendToDroppedSpansLog("this won't get written"); + } finally { + rcvr.close(); + } + } + + /** + * Test that we can write to the dropped spans log. + */ + @Test(timeout = 60000) + public void testWriteToDroppedSpansLog() throws Exception { + final String logPath = new File( + tempDir.toFile(), "testWriteToDroppedSpansLog").getAbsolutePath(); + HTraceConfiguration conf = HTraceConfiguration.fromMap( + new HashMap<String, String>() {{ + put(Conf.ADDRESS_KEY, "127.0.0.1:8080"); + put(TracerId.TRACER_ID_KEY, "testWriteToDroppedSpansLog"); + put(Conf.DROPPED_SPANS_LOG_PATH_KEY, logPath); + put(Conf.DROPPED_SPANS_LOG_MAX_SIZE_KEY, "78"); + }}); + HTracedSpanReceiver rcvr = new HTracedSpanReceiver(conf); + try { + final String LINE1 = "This is a test of the dropped spans log."; + rcvr.appendToDroppedSpansLog(LINE1 + "\n"); + final String LINE2 = "These lines should appear in the log."; + rcvr.appendToDroppedSpansLog(LINE2 + "\n"); + try { + rcvr.appendToDroppedSpansLog("This line won't be written because we're " + + "out of space."); + Assert.fail("expected append to fail because of lack of space"); + } catch (IOException e) { + // ignore + } + List<String> lines = + Files.readAllLines(Paths.get(logPath), StandardCharsets.UTF_8); + Assert.assertEquals(2, lines.size()); + Assert.assertEquals(LINE1, lines.get(0)); + Assert.assertEquals(LINE2, lines.get(1)); + } finally { + rcvr.close(); + } + } + + /** + * Test that we write to the dropped spans log when htraced is unreachable. + */ + @Test(timeout = 60000) + public void testSpansDroppedBecauseOfUnreachableHTraced() throws Exception { + final String logPath = new File(tempDir.toFile(), + "testSpansDroppedBecauseOfUnreachableHTraced").getAbsolutePath(); + // Open a local socket. We know that nobody is listening on this socket, so + // all attempts to send to it will fail. + final ServerSocket serverSocket = new ServerSocket(0); + HTracedSpanReceiver rcvr = null; + try { + HTraceConfiguration conf = HTraceConfiguration.fromMap( + new HashMap<String, String>() {{ + put(Conf.ADDRESS_KEY, "127.0.0.1:" + serverSocket.getLocalPort()); + put(TracerId.TRACER_ID_KEY, + "testSpansDroppedBecauseOfUnreachableHTraced"); + put(Conf.DROPPED_SPANS_LOG_PATH_KEY, logPath); + put(Conf.DROPPED_SPANS_LOG_MAX_SIZE_KEY, "78"); + put(Conf.CONNECT_TIMEOUT_MS_KEY, "1"); + put(Conf.IO_TIMEOUT_MS_KEY, "1"); + put(Conf.FLUSH_RETRY_DELAYS_KEY, "1,1"); + }}); + rcvr = new HTracedSpanReceiver(conf); + rcvr.receiveSpan(new MilliSpan.Builder(). + begin(123).end(456).description("FooBar").build()); + HTracedSpanReceiver tmpRcvr = rcvr; + rcvr = null; + tmpRcvr.close(); + List<String> lines = + Files.readAllLines(Paths.get(logPath), StandardCharsets.UTF_8); + Assert.assertTrue(lines.size() >= 1); + Assert.assertTrue(lines.get(0).contains("Failed to flush ")); + } finally { + serverSocket.close(); + if (rcvr != null) { + rcvr.close(); + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-webapp/src/main/webapp/app/server_info_view.js ---------------------------------------------------------------------- diff --git a/htrace-webapp/src/main/webapp/app/server_info_view.js b/htrace-webapp/src/main/webapp/app/server_info_view.js index efb7545..43533d4 100644 --- a/htrace-webapp/src/main/webapp/app/server_info_view.js +++ b/htrace-webapp/src/main/webapp/app/server_info_view.js @@ -50,7 +50,6 @@ htrace.ServerInfoView = Backbone.View.extend({ '<th>Remote</th>' + '<th>Written</th>' + '<th>ServerDropped</th>' + - '<th>ClientDroppedEstimate</th>' + '</tr>' + '</thead>'; var remotes = []; @@ -69,7 +68,6 @@ htrace.ServerInfoView = Backbone.View.extend({ "<td>" + remote + "</td>" + "<td>" + smtx.Written + "</td>" + "<td>" + smtx.ServerDropped + "</td>" + - "<td>" + smtx.ClientDroppedEstimate + "</td>" + "</tr>"; } out = out + '</table>'; http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-webapp/src/main/webapp/app/server_stats.js ---------------------------------------------------------------------- diff --git a/htrace-webapp/src/main/webapp/app/server_stats.js b/htrace-webapp/src/main/webapp/app/server_stats.js index 4cfea92..65d8c60 100644 --- a/htrace-webapp/src/main/webapp/app/server_stats.js +++ b/htrace-webapp/src/main/webapp/app/server_stats.js @@ -26,7 +26,6 @@ htrace.ServerStats = Backbone.Model.extend({ "IngestedSpans": "(unknown)", "WrittenSpans": "(unknown)", "ServerDroppedSpans": "(unknown)", - "ClientDroppedSpans": "(unknown)", "MaxWriteSpansLatencyMs": "(unknown)", "AverageWriteSpansLatencyMs": "(unknown)" }, http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-webapp/src/main/webapp/index.html ---------------------------------------------------------------------- diff --git a/htrace-webapp/src/main/webapp/index.html b/htrace-webapp/src/main/webapp/index.html index 1e20ec0..edbfd26 100644 --- a/htrace-webapp/src/main/webapp/index.html +++ b/htrace-webapp/src/main/webapp/index.html @@ -92,10 +92,6 @@ <td><%= model.stats.get("ServerDroppedSpans") %></td> </tr> <tr> - <td>Estimated Client Dropped Spans</td> - <td><%= model.stats.get("ClientDroppedEstimate") %></td> - </tr> - <tr> <td>Maximum WriteSpans Latency (ms)</td> <td><%= model.stats.get("MaxWriteSpansLatencyMs") %></td> </tr>
