Michael Blow has submitted this change and it was merged. Change subject: [ASTERIXDB-2132] Replace ITracer.check ......................................................................
[ASTERIXDB-2132] Replace ITracer.check - user model changes: no - storage format changes: no - interface changes: yes Trace categories in the ITracer interface are represented by long values where each bit represents a category. Introduce ITraceCategoryRegistry, an interface for a registry/mapping from Strings to bit positions representing categories. - add simple tests for Tracer and TraceCategoryRegistry Change-Id: I3f26ae988d3786ef4ddb17e8fe78f911e7e628f1 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2087 Sonar-Qube: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> --- M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ConnectorDescriptorWithMessagingTest.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCServiceContext.java M hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/AbstractFrameAppender.java M hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/util/FrameUtils.java M hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/PartitionDataWriter.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ExternalIndexHarness.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java M hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java A hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/ITraceCategoryRegistry.java M hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/ITracer.java A hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/TraceCategoryRegistry.java M hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java A hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/trace/TraceCategoryRegistryTest.java A hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/trace/TraceTest.java 18 files changed, 437 insertions(+), 89 deletions(-) Approvals: Jenkins: Verified; No violations found; Verified Michael Blow: Looks good to me, approved Objections: Anon. E. Moose #1000171: diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ConnectorDescriptorWithMessagingTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ConnectorDescriptorWithMessagingTest.java index 56a45c6..ecc14b1 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ConnectorDescriptorWithMessagingTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ConnectorDescriptorWithMessagingTest.java @@ -49,6 +49,7 @@ import org.apache.hyracks.dataflow.std.connectors.MToNPartitioningWithMessageConnectorDescriptor; import org.apache.hyracks.dataflow.std.connectors.PartitionWithMessageDataWriter; import org.apache.hyracks.test.support.TestUtils; +import org.apache.hyracks.util.trace.ITraceCategoryRegistry; import org.apache.hyracks.util.trace.ITracer; import org.junit.Assert; import org.junit.Test; @@ -92,7 +93,7 @@ for (IFrameWriter writer : partitionWriterFactory.getWriters().values()) { recipients.add((TestFrameWriter) writer); } - partitioner.flush(ITracer.NONE, null, null, null); + partitioner.flush(ITracer.NONE, null, ITraceCategoryRegistry.CATEGORIES_NONE, null); for (TestFrameWriter writer : recipients) { Assert.assertEquals(writer.nextFrameCount(), 1); fta.reset(writer.getLastFrame()); @@ -104,7 +105,7 @@ message.getBuffer().clear(); message.getBuffer().put(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE); message.getBuffer().flip(); - partitioner.flush(ITracer.NONE, null, null, null);; + partitioner.flush(ITracer.NONE, null, ITraceCategoryRegistry.CATEGORIES_NONE, null);; for (TestFrameWriter writer : recipients) { Assert.assertEquals(writer.nextFrameCount(), 2); fta.reset(writer.getLastFrame()); @@ -117,7 +118,7 @@ message.getBuffer().clear(); message.getBuffer().put(MessagingFrameTupleAppender.NULL_FEED_MESSAGE); message.getBuffer().flip(); - partitioner.flush(ITracer.NONE, null, null, null);; + partitioner.flush(ITracer.NONE, null, ITraceCategoryRegistry.CATEGORIES_NONE, null);; for (TestFrameWriter writer : recipients) { Assert.assertEquals(writer.nextFrameCount(), 3); fta.reset(writer.getLastFrame()); @@ -170,7 +171,7 @@ for (IFrameWriter writer : partitionWriterFactory.getWriters().values()) { recipients.add((TestFrameWriter) writer); } - partitioner.flush(ITracer.NONE, null, null, null);; + partitioner.flush(ITracer.NONE, null, ITraceCategoryRegistry.CATEGORIES_NONE, null);; for (TestFrameWriter writer : recipients) { Assert.assertEquals(writer.nextFrameCount(), 1); fta.reset(writer.getLastFrame()); @@ -182,7 +183,7 @@ message.getBuffer().clear(); message.getBuffer().put(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE); message.getBuffer().flip(); - partitioner.flush(ITracer.NONE, null, null, null);; + partitioner.flush(ITracer.NONE, null, ITraceCategoryRegistry.CATEGORIES_NONE, null);; for (TestFrameWriter writer : recipients) { Assert.assertEquals(writer.nextFrameCount(), 2); fta.reset(writer.getLastFrame()); @@ -194,7 +195,7 @@ message.getBuffer().clear(); message.getBuffer().put(MessagingFrameTupleAppender.NULL_FEED_MESSAGE); message.getBuffer().flip(); - partitioner.flush(ITracer.NONE, null, null, null);; + partitioner.flush(ITracer.NONE, null, ITraceCategoryRegistry.CATEGORIES_NONE, null);; for (TestFrameWriter writer : recipients) { Assert.assertEquals(writer.nextFrameCount(), 3); fta.reset(writer.getLastFrame()); @@ -265,7 +266,7 @@ tuple = ttg.next(); } partitioner.nextFrame(frame.getBuffer()); - partitioner.flush(ITracer.NONE, null, null, null);; + partitioner.flush(ITracer.NONE, null, ITraceCategoryRegistry.CATEGORIES_NONE, null);; Assert.assertEquals(1, partitionWriterFactory.getWriters().get(0).nextFrameCount()); Assert.assertEquals(2, partitionWriterFactory.getWriters().get(1).nextFrameCount()); Assert.assertEquals(1, partitionWriterFactory.getWriters().get(2).nextFrameCount()); @@ -324,7 +325,7 @@ appender.append(tuple); } partitioner.nextFrame(frame.getBuffer()); - partitioner.flush(ITracer.NONE, null, null, null);; + partitioner.flush(ITracer.NONE, null, ITraceCategoryRegistry.CATEGORIES_NONE, null);; Assert.assertEquals(partitionWriterFactory.getWriters().get(0).nextFrameCount(), 1); Assert.assertEquals(partitionWriterFactory.getWriters().get(1).nextFrameCount(), 1); Assert.assertEquals(partitionWriterFactory.getWriters().get(2).nextFrameCount(), 1); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java index 3791f61..459a3ea 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java @@ -88,6 +88,8 @@ private final ITracer tracer; + private final long traceCategory; + public FeedMetaStoreNodePushable(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions, IOperatorDescriptor coreOperator, FeedConnectionId feedConnectionId, Map<String, String> feedPolicyProperties, FeedMetaOperatorDescriptor feedMetaOperatorDescriptor) @@ -105,6 +107,7 @@ this.recordDescProvider = recordDescProvider; this.opDesc = feedMetaOperatorDescriptor; tracer = ctx.getJobletContext().getServiceContext().getTracer(); + traceCategory = tracer.getRegistry().get("Process-Frame"); } @Override @@ -140,7 +143,7 @@ @Override public void nextFrame(ByteBuffer buffer) throws HyracksDataException { - long tid = tracer.durationB("Ingestion-Store", "Process-Frame", null); + long tid = tracer.durationB("Ingestion-Store", traceCategory, null); try { FeedUtils.processFeedMessage(buffer, message, fta); writer.nextFrame(buffer); @@ -148,7 +151,7 @@ LOGGER.log(Level.WARNING, "Failure Processing a frame at store side", e); throw HyracksDataException.create(e); } finally { - tracer.durationE(tid, null); + tracer.durationE(tid, traceCategory, null); } } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java index 75d76c0..7a01b87 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java @@ -605,15 +605,17 @@ private class TraceCurrentTimeTask extends TimerTask { private ITracer tracer; + private long traceCategory; public TraceCurrentTimeTask(ITracer tracer) { this.tracer = tracer; + this.traceCategory = tracer.getRegistry().get("Timestamp"); } @Override public void run() { try { - ITracer.check(tracer).instant("CurrentTime", "Timestamp", Tracer.Scope.p, Tracer.dateTimeStamp()); + tracer.instant("CurrentTime", traceCategory, Tracer.Scope.p, Tracer.dateTimeStamp()); } catch (Exception e) { LOGGER.log(Level.WARNING, "Exception tracing current time", e); } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCServiceContext.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCServiceContext.java index 8e35ed4..6a75471 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCServiceContext.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCServiceContext.java @@ -35,6 +35,7 @@ import org.apache.hyracks.control.nc.io.IOManager; import org.apache.hyracks.control.nc.resources.memory.MemoryManager; import org.apache.hyracks.util.trace.ITracer; +import org.apache.hyracks.util.trace.TraceCategoryRegistry; import org.apache.hyracks.util.trace.Tracer; public class NCServiceContext extends ServiceContext implements INCServiceContext { @@ -57,7 +58,7 @@ this.memoryManager = memoryManager; this.ncs = ncs; this.sdh = lccm::dumpState; - this.tracer = new Tracer(nodeId, ncs.getConfiguration().getTraceCategories()); + this.tracer = new Tracer(nodeId, ncs.getConfiguration().getTraceCategories(), new TraceCategoryRegistry()); } @Override @@ -95,7 +96,7 @@ @Override public ITracer getTracer() { - return ITracer.check(tracer); + return tracer; } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/AbstractFrameAppender.java b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/AbstractFrameAppender.java index a377d75..18ddf3e 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/AbstractFrameAppender.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/AbstractFrameAppender.java @@ -117,13 +117,13 @@ writer.flush(); } - public void flush(IFrameWriter writer, ITracer tracer, String name, String cat, String args) + public void flush(IFrameWriter writer, ITracer tracer, String name, long traceCategory, String args) throws HyracksDataException { - final long tid = ITracer.check(tracer).durationB(name, cat, args); + final long tid = tracer.durationB(name, traceCategory, args); try { flush(writer); } finally { - ITracer.check(tracer).durationE(tid, args); + tracer.durationE(tid, traceCategory, args); } } } diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/util/FrameUtils.java b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/util/FrameUtils.java index ea53c25..ec45437 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/util/FrameUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/util/FrameUtils.java @@ -150,14 +150,14 @@ } public static int appendToWriter(IFrameWriter writer, IFrameTupleAppender frameTupleAppender, - IFrameTupleAccessor tupleAccessor, int tIndex, ITracer tracer, String name, String cat, String args) + IFrameTupleAccessor tupleAccessor, int tIndex, ITracer tracer, String name, long cat, String args) throws HyracksDataException { int flushedBytes = 0; if (!frameTupleAppender.append(tupleAccessor, tIndex)) { flushedBytes = frameTupleAppender.getBuffer().capacity(); - long tid = ITracer.check(tracer).durationB(name, cat, args); + long tid = tracer.durationB(name, cat, args); frameTupleAppender.write(writer, true); - ITracer.check(tracer).durationE(tid, args); + tracer.durationE(tid, cat, args); if (!frameTupleAppender.append(tupleAccessor, tIndex)) { throw HyracksDataException.create(ErrorCode.TUPLE_CANNOT_FIT_INTO_EMPTY_FRAME, tupleAccessor.getTupleLength(tIndex)); diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/PartitionDataWriter.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/PartitionDataWriter.java index 4705001..5e33275 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/PartitionDataWriter.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/PartitionDataWriter.java @@ -164,7 +164,7 @@ } } - public void flush(ITracer tracer, String name, String cat, String args) throws HyracksDataException { + public void flush(ITracer tracer, String name, long cat, String args) throws HyracksDataException { for (int i = 0; i < consumerPartitionCount; i++) { if (allocatedFrames[i]) { appenders[i].flush(pWriters[i], tracer, name, cat, args); diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ExternalIndexHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ExternalIndexHarness.java index 11f2441..2f4dcc2 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ExternalIndexHarness.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ExternalIndexHarness.java @@ -39,13 +39,14 @@ import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType; import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.ISearchPredicate; +import org.apache.hyracks.util.trace.ITracer; public class ExternalIndexHarness extends LSMHarness { private static final Logger LOGGER = Logger.getLogger(ExternalIndexHarness.class.getName()); public ExternalIndexHarness(ILSMIndex lsmIndex, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, boolean replicationEnabled) { - super(lsmIndex, mergePolicy, opTracker, replicationEnabled, null); + super(lsmIndex, mergePolicy, opTracker, replicationEnabled, ITracer.NONE); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java index ac38cba..efd7f8c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java @@ -68,6 +68,7 @@ protected final boolean replicationEnabled; protected List<ILSMDiskComponent> componentsToBeReplicated; protected ITracer tracer; + protected long traceCategory; public LSMHarness(ILSMIndex lsmIndex, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, boolean replicationEnabled, ITracer tracer) { @@ -75,6 +76,7 @@ this.opTracker = opTracker; this.mergePolicy = mergePolicy; this.tracer = tracer; + this.traceCategory = tracer.getRegistry().get("release-memory-component"); fullMergeIsRequested = new AtomicBoolean(); //only durable indexes are replicated this.replicationEnabled = replicationEnabled && lsmIndex.isDurable(); @@ -253,13 +255,13 @@ } break; case INACTIVE: - ITracer.check(tracer).instant(c.toString(), "release-memory-component", Scope.p, - lsmIndex.toString()); + tracer.instant(c.toString(), traceCategory, Scope.p, lsmIndex.toString()); ((AbstractLSMMemoryComponent) c).reset(); - // Notify all waiting threads whenever the mutable component's has change to inactive. This is important because - // even though we switched the mutable components, it is possible that the component that we just switched - // to is still busy flushing its data to disk. Thus, the notification that was issued upon scheduling the flush - // is not enough. + // Notify all waiting threads whenever the mutable component's state has changed to + // inactive. This is important because even though we switched the mutable + // components, it is possible that the component that we just switched to is still + // busy flushing its data to disk. Thus, the notification that was issued upon + // scheduling the flush is not enough. opTracker.notifyAll(); break; default: diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java index a7ec8c3..88f3231 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java @@ -37,22 +37,25 @@ protected final ILSMIOOperation ioOp; private final LSMIOOpertionType ioOpType; private final ITracer tracer; - private final String cat; + private final long traceCategory; - protected TracedIOOperation(ILSMIOOperation ioOp, ITracer tracer) { + protected TracedIOOperation(ILSMIOOperation ioOp, ITracer tracer, long traceCategory) { this.ioOp = ioOp; this.tracer = tracer; this.ioOpType = ioOp.getIOOpertionType(); - this.cat = ioOpType.name().toLowerCase(); + this.traceCategory = traceCategory; } public static ILSMIOOperation wrap(final ILSMIOOperation ioOp, final ITracer tracer) { - if (tracer != null && tracer.isEnabled()) { - tracer.instant(ioOp.getTarget().getRelativePath(), - ioOp.getIOOpertionType() == LSMIOOpertionType.FLUSH ? "schedule-flush" : "schedule-merge", Scope.p, - null); - return ioOp instanceof Comparable ? new ComparableTracedIOOperation(ioOp, tracer) - : new TracedIOOperation(ioOp, tracer); + final String ioOpName = ioOp.getIOOpertionType().name().toLowerCase(); + final long traceCategorySchedule = tracer.getRegistry().get("schedule-" + ioOpName); + if (tracer.isEnabled(traceCategorySchedule)) { + tracer.instant(ioOp.getTarget().getRelativePath(), traceCategorySchedule, Scope.p, null); + } + final long traceCategoryExec = tracer.getRegistry().get(ioOpName); + if (tracer.isEnabled(traceCategoryExec)) { + return ioOp instanceof Comparable ? new ComparableTracedIOOperation(ioOp, tracer, traceCategoryExec) + : new TracedIOOperation(ioOp, tracer, traceCategoryExec); } return ioOp; } @@ -83,12 +86,12 @@ @Override public Boolean call() throws HyracksDataException { - final long tid = tracer.durationB(getTarget().getRelativePath(), cat, null); + final String name = getTarget().getRelativePath(); + final long tid = tracer.durationB(name, traceCategory, null); try { return ioOp.call(); } finally { - tracer.durationE(getTarget().getRelativePath(), cat, tid, - "{\"size\":" + getTarget().getFile().length() + "}"); + tracer.durationE(name, traceCategory, tid, "{\"size\":" + getTarget().getFile().length() + "}"); } } @@ -105,8 +108,8 @@ class ComparableTracedIOOperation extends TracedIOOperation implements Comparable<ILSMIOOperation> { - protected ComparableTracedIOOperation(ILSMIOOperation ioOp, ITracer trace) { - super(ioOp, trace); + protected ComparableTracedIOOperation(ILSMIOOperation ioOp, ITracer trace, long traceCategory) { + super(ioOp, trace, traceCategory); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java index ef5a0d2..84eae32 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java @@ -41,7 +41,9 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker; import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.util.trace.ITraceCategoryRegistry; import org.apache.hyracks.util.trace.ITracer; +import org.apache.hyracks.util.trace.TraceCategoryRegistry; import org.apache.hyracks.util.trace.Tracer; @SuppressWarnings("rawtypes") @@ -102,7 +104,8 @@ lsmTree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache, typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck, null, null, null, null, true, - metadataPageManagerFactory, updateAware, Tracer.ALL); + metadataPageManagerFactory, updateAware, new Tracer(LSMBTreeTestContext.class.getSimpleName(), + ITraceCategoryRegistry.CATEGORIES_ALL, new TraceCategoryRegistry())); } LSMBTreeTestContext testCtx = new LSMBTreeTestContext(fieldSerdes, lsmTree, filtered); return testCtx; diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java index b3f5c98..3edba21 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java @@ -23,17 +23,19 @@ final class Event { private static final long NANOTIME_DELTA_TO_EPOCH = System.nanoTime() - TimeUnit.MILLISECONDS.toNanos(System.currentTimeMillis()); + public final String name; - public final String cat; + public final long cat; public final ITracer.Phase ph; public final long ts; public final int pid; public final long tid; public final ITracer.Scope scope; public final String args; + public final TraceCategoryRegistry registry; - private Event(String name, String cat, ITracer.Phase ph, long ts, int pid, long tid, ITracer.Scope scope, - String args) { + private Event(String name, long cat, ITracer.Phase ph, long ts, int pid, long tid, ITracer.Scope scope, String args, + TraceCategoryRegistry registry) { this.name = name; this.cat = cat; this.ph = ph; @@ -42,15 +44,16 @@ this.tid = tid; this.scope = scope; this.args = args; + this.registry = registry; } private static long timestamp() { return (System.nanoTime() - NANOTIME_DELTA_TO_EPOCH) / 1000; } - public static Event create(String name, String cat, ITracer.Phase ph, int pid, long tid, ITracer.Scope scope, - String args) { - return new Event(name, cat, ph, timestamp(), pid, tid, scope, args); + public static Event create(String name, long cat, ITracer.Phase ph, int pid, long tid, ITracer.Scope scope, + String args, TraceCategoryRegistry registry) { + return new Event(name, cat, ph, timestamp(), pid, tid, scope, args, registry); } public String toJson() { @@ -62,8 +65,9 @@ if (name != null) { sb.append("\"name\":\"").append(name).append("\","); } - if (cat != null) { - sb.append("\"cat\":\"").append(cat).append("\","); + if (cat != 0L) { + final String catName = registry.getName(cat); + sb.append("\"cat\":\"").append(catName).append("\","); } sb.append("\"ph\":\"").append(ph).append("\","); sb.append("\"pid\":\"").append(pid).append("\","); diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/ITraceCategoryRegistry.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/ITraceCategoryRegistry.java new file mode 100644 index 0000000..efec0dc --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/ITraceCategoryRegistry.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.util.trace; + +public interface ITraceCategoryRegistry { + + int NO_CATEGORIES = 64; + + long CATEGORIES_ALL = -1L; + long CATEGORIES_NONE = 0L; + + ITraceCategoryRegistry NONE = new TraceCategoryRegistry() { + @Override + public long get(String name) { + return CATEGORIES_NONE; + } + + @Override + public long get(String... names) { + return CATEGORIES_NONE; + } + + @Override + public String getName(long categoryCode) { + return ""; + } + }; + + long get(String name); + + long get(String... names); + + String getName(long categoryCode); +} diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/ITracer.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/ITracer.java index 8342c5b..4f1c2b9 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/ITracer.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/ITracer.java @@ -58,49 +58,49 @@ } @Override - public boolean isEnabled() { + public ITraceCategoryRegistry getRegistry() { + return ITraceCategoryRegistry.NONE; + } + + @Override + public boolean isEnabled(long cat) { return false; } @Override - public long durationB(String name, String cat, String args) { + public long durationB(String name, long cat, String args) { return -1; } @Override - public void durationE(long tid, String args) { + public void durationE(String name, long cat, long tid, String args) { // nothing to do here } @Override - public void instant(String name, String cat, Scope scope, String args) { + public void durationE(long tid, long cat, String args) { // nothing to do here } @Override - public void durationE(String name, String cat, long tid, String args) { + public void instant(String name, long cat, Scope scope, String args) { // nothing to do here } }; - static ITracer check(ITracer tracer) { - if (tracer == null) { - throw new IllegalArgumentException("Tracer cannot be null"); - } - return tracer.isEnabled() ? tracer : NONE; - } - String getName(); - boolean isEnabled(); + ITraceCategoryRegistry getRegistry(); - long durationB(String name, String cat, String args); + boolean isEnabled(long cat); - void durationE(long tid, String args); + long durationB(String name, long cat, String args); - void durationE(String name, String cat, long tid, String args); + void durationE(long tid, long cat, String args); - void instant(String name, String cat, Scope scope, String args); + void durationE(String name, long cat, long tid, String args); + + void instant(String name, long cat, Scope scope, String args); @Override String toString(); diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/TraceCategoryRegistry.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/TraceCategoryRegistry.java new file mode 100644 index 0000000..db11285 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/TraceCategoryRegistry.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.util.trace; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +public class TraceCategoryRegistry implements ITraceCategoryRegistry { + + private Map<String, Long> categories = Collections.synchronizedMap(new HashMap<>()); + private int bitPos = 0; + + public TraceCategoryRegistry() { + categories.put("*", ITraceCategoryRegistry.CATEGORIES_ALL); + } + + @Override + public long get(String name) { + return categories.computeIfAbsent(name, this::nextCode); + } + + private long nextCode(String name) { + if (bitPos > NO_CATEGORIES - 1) { + throw new IllegalStateException("Cannot add category " + name); + } + return 1L << bitPos++; + } + + @Override + public long get(String... names) { + long result = 0; + for (String name : names) { + result |= get(name); + } + return result; + } + + private Optional<Map.Entry<String, Long>> findEntry(long categoryCode) { + return categories.entrySet().stream().filter(e -> e.getValue() == categoryCode).findFirst(); + } + + @Override + public String getName(long categoryCode) { + Optional<Map.Entry<String, Long>> entry = findEntry(categoryCode); + if (!entry.isPresent()) { + throw new IllegalArgumentException("No category for code " + categoryCode); + } + return entry.get().getKey(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (int pos = 0; pos < NO_CATEGORIES; ++pos) { + long categoryCode = 1L << pos; + Optional<Map.Entry<String, Long>> entry = findEntry(categoryCode); + if (!entry.isPresent()) { + continue; + } + String name = entry.get().getKey(); + String codeString = Long.toBinaryString(categoryCode); + sb.append(name).append(" -> ").append(codeString).append(' '); + } + return sb.toString(); + } +} diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java index d540dda..72a9989 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java @@ -40,16 +40,25 @@ protected static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); protected final Logger traceLog; - protected String[] categories; + protected long categories; + protected TraceCategoryRegistry registry; protected static final int pid = PidHelper.getPid(); - public Tracer(String name, String[] categories) { + public Tracer(String name, long categories, TraceCategoryRegistry registry) { final String traceLoggerName = Tracer.class.getName() + "@" + name; - LOGGER.info("Initialize Tracer " + traceLoggerName + " " + Arrays.toString(categories)); + LOGGER.info("Initialize Tracer " + traceLoggerName); this.traceLog = Logger.getLogger(traceLoggerName); this.categories = categories; - instant("Trace-Start", CAT, Scope.p, dateTimeStamp()); + this.registry = registry; + final long traceCategory = getRegistry().get(CAT); + instant("Trace-Start", traceCategory, Scope.p, dateTimeStamp()); + } + + public Tracer(String name, String[] categories, TraceCategoryRegistry registry) { + this(name, ITraceCategoryRegistry.CATEGORIES_ALL, registry); + LOGGER.info("Set categories for Tracer " + this.traceLog.getName() + " to " + Arrays.toString(categories)); + this.categories = getRegistry().get(categories); } public static String dateTimeStamp() { @@ -58,11 +67,9 @@ } } - public static final Tracer ALL = new Tracer("All", new String[] { "*" }); - @Override public String toString() { - return getName() + Arrays.toString(categories) + (isEnabled() ? "enabled" : "disabled"); + return getName() + Long.toHexString(categories); } @Override @@ -71,32 +78,46 @@ } @Override - public boolean isEnabled() { - return categories.length > 0; + public TraceCategoryRegistry getRegistry() { + return registry; } @Override - public long durationB(String name, String cat, String args) { - Event e = Event.create(name, cat, Phase.B, pid, Thread.currentThread().getId(), null, args); - traceLog.log(TRACE_LOG_LEVEL, e.toJson()); - return e.tid; + public boolean isEnabled(long cat) { + return (categories & cat) != 0L; } @Override - public void durationE(long tid, String args) { - Event e = Event.create(null, null, Phase.E, pid, tid, null, args); - traceLog.log(TRACE_LOG_LEVEL, e.toJson()); + public long durationB(String name, long cat, String args) { + if (isEnabled(cat)) { + Event e = Event.create(name, cat, Phase.B, pid, Thread.currentThread().getId(), null, args, getRegistry()); + traceLog.log(TRACE_LOG_LEVEL, e.toJson()); + return e.tid; + } + return -1; } @Override - public void durationE(String name, String cat, long tid, String args) { - Event e = Event.create(name, cat, Phase.E, pid, tid, null, args); - traceLog.log(TRACE_LOG_LEVEL, e.toJson()); + public void durationE(String name, long cat, long tid, String args) { + if (isEnabled(cat)) { + Event e = Event.create(name, cat, Phase.E, pid, tid, null, args, getRegistry()); + traceLog.log(TRACE_LOG_LEVEL, e.toJson()); + } } @Override - public void instant(String name, String cat, Scope scope, String args) { - Event e = Event.create(name, cat, Phase.i, pid, Thread.currentThread().getId(), scope, args); - traceLog.log(TRACE_LOG_LEVEL, e.toJson()); + public void durationE(long tid, long cat, String args) { + if (isEnabled(cat)) { + Event e = Event.create(null, 0L, Phase.E, pid, tid, null, args, getRegistry()); + traceLog.log(TRACE_LOG_LEVEL, e.toJson()); + } + } + + @Override + public void instant(String name, long cat, Scope scope, String args) { + if (isEnabled(cat)) { + Event e = Event.create(name, cat, Phase.i, pid, Thread.currentThread().getId(), scope, args, getRegistry()); + traceLog.log(TRACE_LOG_LEVEL, e.toJson()); + } } } diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/trace/TraceCategoryRegistryTest.java b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/trace/TraceCategoryRegistryTest.java new file mode 100644 index 0000000..2207aba --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/trace/TraceCategoryRegistryTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.util.trace; + +import org.junit.Assert; +import org.junit.Test; + +public class TraceCategoryRegistryTest { + + @Test + public void limit() { + TraceCategoryRegistry registry = new TraceCategoryRegistry(); + + int success = 0; + int fail = 0; + String expectedError = "Cannot add category"; + String actualError = null; + for (int i = 0; i < ITraceCategoryRegistry.NO_CATEGORIES + 1; ++i) { + try { + final String name = "cat" + Integer.toHexString(i); + registry.get(name); + ++success; + } catch (IllegalStateException ise) { + actualError = ise.getMessage(); + ++fail; + } + } + Assert.assertEquals(64, success); + Assert.assertEquals(1, fail); + Assert.assertTrue(actualError.contains(expectedError)); + } + + @Test + public void getNameValidCode() { + TraceCategoryRegistry registry = new TraceCategoryRegistry(); + registry.get("catA"); + registry.get("catB"); + registry.get("catC"); + + int success = 0; + int fail = 0; + String expectedError = "No category for code"; + String actualError = null; + String expectedResult = "catC"; + String actualResult = null; + for (int i = 0; i < ITraceCategoryRegistry.NO_CATEGORIES; ++i) { + try { + final long code = 1L << i; + actualResult = registry.getName(code); + ++success; + } catch (IllegalArgumentException iae) { + actualError = iae.getMessage(); + ++fail; + } + } + + Assert.assertEquals(3, success); + Assert.assertEquals(61, fail); + Assert.assertEquals(actualResult, expectedResult); + Assert.assertTrue(actualError.contains(expectedError)); + } + + @Test + public void getNameInvalidCode() { + TraceCategoryRegistry registry = new TraceCategoryRegistry(); + + final String expectedError = "No category for code"; + String actualError = null; + try { + registry.getName(11); + } catch (IllegalArgumentException iae) { + actualError = iae.getMessage(); + } + Assert.assertTrue(actualError.contains(expectedError)); + } +} diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/trace/TraceTest.java b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/trace/TraceTest.java new file mode 100644 index 0000000..4b36013 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/trace/TraceTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.util.trace; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.logging.Formatter; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.logging.StreamHandler; + +import org.junit.Assert; +import org.junit.Test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class TraceTest { + + private final ObjectMapper mapper = new ObjectMapper(); + private final String name = "test"; + + private StreamHandler redirectTraceLog(OutputStream os) throws IOException { + final Logger logger = Logger.getLogger(Tracer.class.getName() + "@" + name); + final StreamHandler handler = new StreamHandler(os, new Formatter() { + @Override + public String format(LogRecord record) { + return record.getMessage() + "\n"; + } + }); + logger.addHandler(handler); + return handler; + } + + private JsonNode validate(String line) throws IOException { + return mapper.readTree(line); + } + + @Test + public void testInstant() throws IOException { + final ByteArrayOutputStream os = new ByteArrayOutputStream(); + final StreamHandler handler = redirectTraceLog(os); + + TraceCategoryRegistry registry = new TraceCategoryRegistry(); + long cat1 = registry.get("CAT1"); + long cat2 = registry.get("CAT2"); + long cat3 = registry.get("CAT3"); + + ITracer tracer = new Tracer(name, new String[] { "CAT1", "CAT2" }, registry); + tracer.instant("test1", cat1, ITracer.Scope.p, null); + tracer.instant("test2", cat2, ITracer.Scope.p, null); + tracer.instant("test3", cat3, ITracer.Scope.p, null); + + handler.flush(); + + String[] lines = os.toString().split("\n"); + for (String line : lines) { + Assert.assertFalse(validate(line).get("cat").equals("CAT3")); + } + } +} -- To view, visit https://asterix-gerrit.ics.uci.edu/2087 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I3f26ae988d3786ef4ddb17e8fe78f911e7e628f1 Gerrit-PatchSet: 10 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Till Westmann <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
