Author: mduerig
Date: Tue Mar 27 14:34:31 2018
New Revision: 1827841
URL: http://svn.apache.org/viewvc?rev=1827841&view=rev
Log:
OAK-5655: TarMK: Analyse locality of reference
Utility for collecting IO traces for given read access patterns
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/iotrace/
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/iotrace/BreathFirstTrace.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/iotrace/IOTracer.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/iotrace/Trace.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/tool/
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/tool/iotrace/
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/tool/iotrace/BreathFirstTraceTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/tool/iotrace/IOTracerRunner.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/tool/iotrace/IOTracerTest.java
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/iotrace/BreathFirstTrace.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/iotrace/BreathFirstTrace.java?rev=1827841&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/iotrace/BreathFirstTrace.java
(added)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/iotrace/BreathFirstTrace.java
Tue Mar 27 14:34:31 2018
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.jackrabbit.oak.segment.tool.iotrace;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.Lists.newLinkedList;
+import static java.lang.String.valueOf;
+import static java.util.Collections.singleton;
+import static org.apache.jackrabbit.oak.commons.PathUtils.elements;
+
+import java.io.Writer;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+import javax.annotation.Nonnull;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+
+/**
+ * A breath first traversal trace.
+ * <p>
+ * When {@link Trace#run(NodeState) run} this trace performs a breath first
traversal starting
+ * from the passed node up to a certain depth. It logs the current depth and
the number of
+ * traversed nodes as additional {@link IOTracer#setContext(List) context}.
+ */
+public class BreathFirstTrace implements Trace {
+
+ /**
+ * The context specification of this trace.
+ * @see IOTracer#newIOTracer(Function, Writer, String)
+ */
+ @Nonnull
+ public static final String CONTEXT_SPEC = "depth,count";
+
+ private final int depth;
+
+ @Nonnull
+ private final String path;
+
+ @Nonnull
+ private final Consumer<List<String>> context;
+
+ @Nonnull
+ private final AtomicInteger nodeCount = new AtomicInteger();
+
+ /**
+ * Create a new instance of a breath first traversal trace.
+ * @param depth maximal depth of the nodes to traverse
+ * @param path path of the root node where to start traversing
+ * @param context consumer to pass the additional context to
+ */
+ public BreathFirstTrace(int depth, @Nonnull String path, @Nonnull
Consumer<List<String>> context) {
+ checkArgument(depth >= 0);
+
+ this.depth = depth;
+ this.path = path;
+ this.context = context;
+ }
+
+ @Override
+ public void run(@Nonnull NodeState node) {
+ updateContext(context, 0, nodeCount.incrementAndGet());
+ traverse(newLinkedList(singleton(getNode(node, path))), 0);
+ }
+
+ @Nonnull
+ private static NodeState getNode(@Nonnull NodeState root, @Nonnull String
path) {
+ NodeState node = root;
+ for (String name : elements(path)) {
+ node = node.getChildNode(name);
+ }
+ return node;
+ }
+
+ private void traverse(Queue<NodeState> nodes, int depth) {
+ if (!nodes.isEmpty()) {
+ Queue<NodeState> children = newLinkedList();
+ while (!nodes.isEmpty()) {
+ NodeState head = nodes.poll();
+ assert head != null;
+ if (depth < this.depth) {
+ head.getChildNodeEntries().forEach(
+ cse -> {
+ updateContext(context, depth + 1,
nodeCount.incrementAndGet());
+ NodeState child = cse.getNodeState();
+ if (depth + 1 < this.depth) {
+ // Only add to children queue if not at last
level to save memory
+ children.offer(child);
+ }
+ });
+ }
+ }
+ traverse(children, depth + 1);
+ }
+ }
+
+ private static void updateContext(@Nonnull Consumer<List<String>> context,
int depth, int count) {
+ context.accept(ImmutableList.of(valueOf(depth), valueOf(count)));
+ }
+
+}
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/iotrace/IOTracer.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/iotrace/IOTracer.java?rev=1827841&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/iotrace/IOTracer.java
(added)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/iotrace/IOTracer.java
Tue Mar 27 14:34:31 2018
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.jackrabbit.oak.segment.tool.iotrace;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.collect.Queues.newConcurrentLinkedQueue;
+import static java.lang.String.join;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.Flushable;
+import java.io.PrintWriter;
+import java.io.Writer;
+import java.util.List;
+import java.util.Objects;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Function;
+
+import javax.annotation.Nonnull;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitor;
+import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter;
+
+/**
+ * This utility class allows collecting IO traces of read accesses to segments
+ * caused by reading specific items.
+ * <p>
+ * An instance of {@link Trace} is used to specify a read pattern. Segment
reads
+ * are recorded in CSV format:
+ <pre>
+ timestamp,file,segmentId,length,elapsed
+
1522147945084,data01415a.tar,f81378df-b3f8-4b25-0000-00000002c450,181328,171849
+
1522147945096,data01415a.tar,f81378df-b3f8-4b25-0000-00000002c450,181328,131272
+
1522147945097,data01415a.tar,f81378df-b3f8-4b25-0000-00000002c450,181328,142766
+ ...
+ </pre>
+ * {@link Trace} implementations can specify an additional context, which is
recorded
+ * with each line of the CSV output. A context is simply a list of additional
fields
+ * as specified during instantiation of an {@code IOTracer}.
+ */
+public class IOTracer {
+ @Nonnull
+ private final Function<IOMonitor, FileStore> fileStoreFactory;
+
+ @Nonnull
+ private final IOTraceMonitor ioMonitor;
+
+ private IOTracer(
+ @Nonnull Function<IOMonitor, FileStore> fileStoreFactory,
+ @Nonnull Writer output,
+ @Nonnull String contextSpec) {
+ this.fileStoreFactory = checkNotNull(fileStoreFactory);
+ ioMonitor = new IOTraceMonitor(checkNotNull(output),
checkNotNull(contextSpec));
+ }
+
+ /**
+ * Create a new {@code IOTracer} instance.
+ * @param fileStoreFactory A factory for creating a {@link FileStore}
with the
+ * passed {@link IOMonitor} for monitoring
segment IO.
+ * @param output The target for the CSV formatted IO trace.
+ * @param contextSpec The specification of additional context
provided by
+ * the {@link Trace traces} being {@link
IOTracer#collectTrace(Trace) run}.
+ * A trace consists of a comma separated list of
values, which must match
+ * the list of values passed to {@link
IOTracer#setContext(List)}.
+ * @return A new {@code IOTracer} instance.
+ */
+ @Nonnull
+ public static IOTracer newIOTracer(
+ @Nonnull Function<IOMonitor, FileStore> fileStoreFactory,
+ @Nonnull Writer output,
+ @Nonnull String contextSpec) {
+ return new IOTracer(fileStoreFactory, output, contextSpec);
+ }
+
+ /**
+ * Collect a IO trace.
+ * @param trace
+ */
+ public void collectTrace(@Nonnull Trace trace) {
+ checkNotNull(trace);
+ try (FileStore fileStore =
checkNotNull(fileStoreFactory).apply(checkNotNull(ioMonitor))) {
+ trace.run(fileStore.getHead());
+ } finally {
+ ioMonitor.flush();
+ }
+ }
+
+ /**
+ * Set the {@code context} to be added to each line of the IOTrace going
forward. The list
+ * of values needs to match the context specification passed to
+ * {@link IOTracer#newIOTracer(Function, Writer, String)}.
+ * @param context
+ */
+ public void setContext(@Nonnull List<String> context) {
+ ioMonitor.setContext(context);
+ }
+
+ private static class IOTraceMonitor extends IOMonitorAdapter implements
Flushable {
+ @Nonnull
+ private final AtomicReference<List<String>> context =
+ new AtomicReference<>(ImmutableList.of());
+
+ @Nonnull
+ private final PrintWriter out;
+
+ @Nonnull
+ private final Lock ioLock = new ReentrantLock();
+
+ @Nonnull
+ private final ConcurrentLinkedQueue<IOEvent> ioEvents =
newConcurrentLinkedQueue();
+
+ public IOTraceMonitor(@Nonnull Writer writer, @Nonnull String
contextSpec) {
+ out = new PrintWriter(new BufferedWriter(writer));
+ out.println(IOEvent.getFields(contextSpec));
+ }
+
+ public void setContext(@Nonnull List<String> context) {
+ this.context.set(context);
+ }
+
+ @Override
+ public void afterSegmentRead(@Nonnull File file, long msb, long lsb,
int length, long elapsed) {
+ ioEvents.add(new IOEvent(
+ file.getName(), msb, lsb, length, elapsed, context.get()));
+ if (ioLock.tryLock()) {
+ try {
+ flush();
+ } finally {
+ ioLock.unlock();
+ }
+ }
+ }
+
+ @Override
+ public void flush() {
+ ioLock.lock();
+ try {
+ while (!ioEvents.isEmpty()) {
+ out.println(Objects.toString(ioEvents.poll()));
+ }
+ out.flush();
+ } finally {
+ ioLock.unlock();
+ }
+ }
+
+ private static class IOEvent {
+ @Nonnull private final String fileName;
+ private final long msb;
+ private final long lsb;
+ private final int length;
+ private final long elapsed;
+ @Nonnull private final List<String> context;
+
+ private IOEvent(
+ @Nonnull String fileName,
+ long msb,
+ long lsb,
+ int length,
+ long elapsed,
+ @Nonnull List<String> context)
+ {
+ this.fileName = fileName;
+ this.msb = msb;
+ this.lsb = lsb;
+ this.length = length;
+ this.elapsed = elapsed;
+ this.context = context;
+ }
+
+ public static String getFields(@Nonnull String contextSpec) {
+ return "timestamp,file,segmentId,length,elapsed," +
contextSpec;
+ }
+
+ @Override
+ public String toString() {
+ return System.currentTimeMillis() + "," + fileName + "," +
+ new UUID(msb, length) + "," + length + "," + elapsed +
"," +
+ join(",", context);
+ }
+ }
+ }
+}
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/iotrace/Trace.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/iotrace/Trace.java?rev=1827841&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/iotrace/Trace.java
(added)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/iotrace/Trace.java
Tue Mar 27 14:34:31 2018
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.jackrabbit.oak.segment.tool.iotrace;
+
+import javax.annotation.Nonnull;
+
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+
+/**
+ * An instance of a {@code Trace} specifies a read pattern for tracing
+ * IO reads of segments with an {@link IOTracer} instance.
+ */
+interface Trace {
+
+ /**
+ * Run this trace on the passed {@code node}.
+ * @param node
+ */
+ void run(@Nonnull NodeState node);
+}
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/tool/iotrace/BreathFirstTraceTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/tool/iotrace/BreathFirstTraceTest.java?rev=1827841&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/tool/iotrace/BreathFirstTraceTest.java
(added)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/tool/iotrace/BreathFirstTraceTest.java
Tue Mar 27 14:34:31 2018
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.jackrabbit.oak.segment.tool.iotrace;
+
+import static com.google.common.collect.Lists.newArrayList;
+import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import static org.junit.Assert.assertEquals;
+
+import java.util.List;
+
+import javax.annotation.Nonnull;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.junit.Test;
+
+public class BreathFirstTraceTest {
+
+ @Nonnull
+ private static NodeState createTree(int depth) {
+ NodeBuilder root = EMPTY_NODE.builder();
+ NodeBuilder child = root;
+ for (int k = 0 ; k < depth; k++) {
+ child = child.setChildNode("node-" + k);
+ }
+ return root.getNodeState();
+ }
+
+ @Test
+ public void testTraverseEmptyTree() {
+ List<List<String>> trace = newArrayList();
+ new BreathFirstTrace(4, "/", trace::add).run(createTree(0));
+ assertEquals(1, trace.size());
+ assertEquals(ImmutableList.of("0", "1"), trace.get(0));
+ }
+
+ @Test
+ public void testTraverseDepth1Tree() {
+ List<List<String>> trace = newArrayList();
+ new BreathFirstTrace(4, "/", trace::add).run(createTree(1));
+ assertEquals(2, trace.size());
+ assertEquals(ImmutableList.of("0", "1"), trace.get(0));
+ assertEquals(ImmutableList.of("1", "2"), trace.get(1));
+ }
+
+ @Test
+ public void testTraverseDepth2Tree() {
+ List<List<String>> trace = newArrayList();
+ new BreathFirstTrace(4, "/", trace::add).run(createTree(2));
+ assertEquals(3, trace.size());
+ assertEquals(ImmutableList.of("0", "1"), trace.get(0));
+ assertEquals(ImmutableList.of("1", "2"), trace.get(1));
+ assertEquals(ImmutableList.of("2", "3"), trace.get(2));
+ }
+
+ @Test
+ public void testTraverseDepth3TreeWithLimit2() {
+ List<List<String>> trace = newArrayList();
+ new BreathFirstTrace(2, "/", trace::add).run(createTree(3));
+ assertEquals(3, trace.size());
+ assertEquals(ImmutableList.of("0", "1"), trace.get(0));
+ assertEquals(ImmutableList.of("1", "2"), trace.get(1));
+ assertEquals(ImmutableList.of("2", "3"), trace.get(2));
+ }
+
+}
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/tool/iotrace/IOTracerRunner.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/tool/iotrace/IOTracerRunner.java?rev=1827841&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/tool/iotrace/IOTracerRunner.java
(added)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/tool/iotrace/IOTracerRunner.java
Tue Mar 27 14:34:31 2018
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.jackrabbit.oak.segment.tool.iotrace;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.lang.Boolean.parseBoolean;
+import static java.lang.Integer.getInteger;
+import static java.lang.String.format;
+import static java.lang.System.getProperty;
+import static
org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
+import static
org.apache.jackrabbit.oak.segment.tool.iotrace.IOTracer.newIOTracer;
+import static org.junit.Assume.assumeTrue;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.function.Function;
+
+import javax.annotation.Nonnull;
+
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
+import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
+import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitor;
+import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * This test case can be used to collect {@link IOTracer io traces}. It is
disabled
+ * by default and needs to be enabled via {@code -Dtest=IOTraceRunner}.
+ * <br>
+ * The test accepts the following properties:<br>
+ * {@code -Dinput=/path/to/segmentstore}. Required.<br>
+ * {@code -Doutput=/path/to/trace.cvs}. Default: {@code iotrace.csv}<br>
+ * {@code -Dmmap=true|false}. Default {@code true}<br>
+ * {@code -Dsegment-cache=n}. Default {@code 256}<br>
+ * {@code -Ddepth=n}. Default {@code 5}<br>
+ * {@code -Dpath=/path/to/start/node}. Default {@code /root}
+ * <p>
+ * FIXME OAK-5655: Turn this into a development tool and move to the right
place.
+ */
+public class IOTracerRunner extends IOMonitorAdapter {
+ private static final boolean ENABLED =
+ IOTracerRunner.class.getSimpleName().equals(getProperty("test"));
+
+ private static final String OUTPUT = getProperty("output", "iotrace.csv");
+
+ private static final String INPUT = getProperty("segmentstore");
+
+ private static final boolean MMAP = parseBoolean(getProperty("mmap",
"true"));
+
+ private static final int SEGMENT_CACHE = getInteger("segment-cache", 256);
+
+ private static final int DEPTH = getInteger("depth", 5);
+
+ private static final String PATH = getProperty("path", "/root");
+
+ @Before
+ public void setup() throws Exception {
+ assumeTrue(
+ format("Test disabled. Specify -Dtest=%s to enable it",
IOTracerRunner.class.getSimpleName()),
+ ENABLED);
+ }
+
+ @Test
+ public void collectTrace() throws IOException,
InvalidFileStoreVersionException {
+ checkArgument(INPUT != null, "No segment store directory specified");
+ System.out.println(format(
+ "Breath first traversing %d levels of %s starting at %s",
DEPTH, INPUT, PATH));
+ System.out.println(
+ format("mmap=%b, segment cache=%d", MMAP, SEGMENT_CACHE));
+ System.out.println(format("Writing trace to %s", OUTPUT));
+
+ collectTrace(INPUT, MMAP, SEGMENT_CACHE, PATH, DEPTH, OUTPUT);
+ }
+
+ public void collectTrace(
+ @Nonnull String segmentStore,
+ boolean mmap,
+ int segmentCacheSize,
+ @Nonnull String path,
+ int depth,
+ @Nonnull String output)
+ throws IOException, InvalidFileStoreVersionException {
+ checkNotNull(segmentStore);
+ checkNotNull(path);
+ checkNotNull(output);
+
+ try (PrintWriter out = new PrintWriter(new BufferedWriter(new
FileWriter(output, true)))) {
+ Function<IOMonitor, FileStore> factory = ioMonitor ->
+ newFileStore(
+ fileStoreBuilder(new File(segmentStore))
+ .withMemoryMapping(mmap)
+ .withSegmentCacheSize(segmentCacheSize)
+ .withIOMonitor(ioMonitor));
+
+ IOTracer ioTracer = newIOTracer(factory, out,
BreathFirstTrace.CONTEXT_SPEC);
+ ioTracer.collectTrace(
+ new BreathFirstTrace(depth, path, ioTracer::setContext));
+ }
+ }
+
+ @Nonnull
+ private static FileStore newFileStore(FileStoreBuilder fileStoreBuilder) {
+ try {
+ return fileStoreBuilder.build();
+ } catch (InvalidFileStoreVersionException | IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+}
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/tool/iotrace/IOTracerTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/tool/iotrace/IOTracerTest.java?rev=1827841&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/tool/iotrace/IOTracerTest.java
(added)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/tool/iotrace/IOTracerTest.java
Tue Mar 27 14:34:31 2018
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.jackrabbit.oak.segment.tool.iotrace;
+
+import static java.lang.Integer.parseInt;
+import static java.lang.Long.parseLong;
+import static java.lang.System.currentTimeMillis;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
+import static
org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
+import static
org.apache.jackrabbit.oak.segment.tool.iotrace.IOTracer.newIOTracer;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
+
+import javax.annotation.Nonnull;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.jackrabbit.oak.segment.SegmentNodeBuilder;
+import org.apache.jackrabbit.oak.segment.SegmentNodeState;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
+import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitor;
+import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class IOTracerTest extends IOMonitorAdapter {
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
+ @Before
+ public void setup() throws IOException, InvalidFileStoreVersionException {
+ try (FileStore fileStore = fileStoreBuilder(folder.getRoot()).build())
{
+ SegmentNodeState currentHead = fileStore.getHead();
+ SegmentNodeBuilder root = currentHead.builder();
+ root.setChildNode("0a");
+ root.setChildNode("0b");
+ NodeBuilder builder = root.setChildNode("0c");
+
+ builder.setChildNode("0d");
+ builder.setChildNode("0e");
+ builder = builder.setChildNode("0f");
+
+ builder.setChildNode("0g");
+ builder.setChildNode("0h");
+ builder.setChildNode("0i").setChildNode("0j");
+ SegmentNodeState newHead = root.getNodeState();
+ fileStore.getRevisions().setHead(currentHead.getRecordId(),
newHead.getRecordId());
+ }
+ }
+
+ @Nonnull
+ private FileStore createFileStore(IOMonitor ioMonitor) {
+ try {
+ return fileStoreBuilder(folder.getRoot())
+ .withSegmentCacheSize(0)
+ .withIOMonitor(ioMonitor).build();
+ } catch (InvalidFileStoreVersionException | IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Test
+ public void collectTrace() throws IOException,
InvalidFileStoreVersionException {
+ try (StringWriter out = new StringWriter()) {
+ Function<IOMonitor, FileStore> factory = this::createFileStore;
+
+ IOTracer ioTracer = newIOTracer(factory, out,
BreathFirstTrace.CONTEXT_SPEC);
+ ioTracer.collectTrace(
+ new BreathFirstTrace(2, "/", ioTracer::setContext));
+
+ try (BufferedReader reader = new BufferedReader(new
StringReader(out.toString()))) {
+ Optional<String> header = reader.lines().findFirst();
+ List<String[]> entries = reader.lines()
+ .map(line -> line.split(","))
+ .collect(toList());
+
+ assertTrue(header.isPresent());
+
assertEquals("timestamp,file,segmentId,length,elapsed,depth,count",
header.get());
+
+ long now = currentTimeMillis();
+ assertTrue("The timestamps of all entries must be in the past",
+ entries.stream()
+ .map(row -> parseLong(row[0])) // ts
+ .allMatch(ts -> ts <= now));
+
+ assertEquals("Expected depths 0, 1 and 2",
+ ImmutableSet.of(0, 1, 2),
+ entries.stream()
+ .map(row -> parseInt(row[5])) // depth
+ .distinct().collect(toSet()));
+
+ assertEquals("Expected max 10 nodes",
+ Optional.of(true),
+ entries.stream()
+ .map(row -> parseInt(row[6])) // count
+ .max(Comparator.naturalOrder())
+ .map(max -> max <= 10));
+ }
+ }
+ }
+
+}
