Author: mduerig
Date: Tue Dec 15 09:05:19 2015
New Revision: 1720094
URL: http://svn.apache.org/viewvc?rev=1720094&view=rev
Log:
OAK-3774: Tool for detecting references to pre compacted segments
Generalised segment graph parsing, implement gc graph, add tests
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraph.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraphTest.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/package-info.java
jackrabbit/oak/trunk/oak-run/README.md
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/segment/FileStoreHelper.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraph.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraph.java?rev=1720094&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraph.java
(added)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraph.java
Tue Dec 15 09:05:19 2015
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.collect.Maps.newHashMap;
+import static com.google.common.collect.Sets.newHashSet;
+import static
org.apache.jackrabbit.oak.plugins.segment.SegmentId.isDataSegmentId;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.UUID;
+
+import javax.annotation.CheckForNull;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.json.JsonObject;
+import org.apache.jackrabbit.oak.commons.json.JsopTokenizer;
+import org.apache.jackrabbit.oak.plugins.segment.file.FileStore.ReadOnlyStore;
+
+/**
+ * Utility graph for parsing a segment graph.
+ */
+public final class SegmentGraph {
+ private SegmentGraph() { }
+
+ /**
+ * Visitor for receiving call backs while traversing the
+ * segment graph.
+ */
+ public interface SegmentGraphVisitor {
+
+ /**
+ * A call to this method indicates that the {@code from} segment
+ * references the {@code to} segment. Or if {@code to} is {@code null}
+ * that the {@code from} has no references.
+ *
+ * @param from
+ * @param to
+ */
+ void accept(@Nonnull UUID from, @CheckForNull UUID to);
+ }
+
+ /**
+ * A simple graph representation for a graph with node of type {@code T}.
+ */
+ public static class Graph<T> {
+ /** The vertices of this graph */
+ public final Set<T> vertices = newHashSet();
+
+ /** The edges of this graph */
+ public final Map<T, Set<T>> edges = newHashMap();
+
+ private void addVertex(T vertex) {
+ vertices.add(vertex);
+ }
+
+ private void addEdge(T from, T to) {
+ Set<T> tos = edges.get(from);
+ if (tos == null) {
+ tos = newHashSet();
+ edges.put(from, tos);
+ }
+ tos.add(to);
+ }
+ }
+
+ /**
+ * Write the segment graph of a file store to a stream.
+ * <p>
+ * The graph is written in
+ * <a
href="https://gephi.github.io/users/supported-graph-formats/gdf-format/";>the
Guess GDF format</a>,
+ * which is easily imported into <a
href="https://gephi.github.io/";>Gephi</a>.
+ * As GDF only supports integers but the segment time stamps are encoded
as long
+ * the {@code epoch} argument is used as a negative offset translating all
timestamps
+ * into a valid int range.
+ *
+ * @param fileStore file store to graph
+ * @param out stream to write the graph to
+ * @param epoch epoch (in milliseconds)
+ * @throws Exception
+ */
+ public static void writeSegmentGraph(
+ @Nonnull ReadOnlyStore fileStore,
+ @Nonnull OutputStream out,
+ @Nonnull Date epoch) throws Exception {
+
+ checkNotNull(epoch);
+ PrintWriter writer = new PrintWriter(checkNotNull(out));
+ try {
+ SegmentNodeState root = checkNotNull(fileStore).getHead();
+
+ Graph<UUID> segmentGraph = parseSegmentGraph(fileStore);
+ Graph<UUID> headGraph = parseHeadGraph(root.getRecordId());
+
+ writer.write("nodedef>name VARCHAR, label VARCHAR, type VARCHAR,
wid VARCHAR, gc INT, t INT, head BOOLEAN\n");
+ for (UUID segment : segmentGraph.vertices) {
+ writeNode(segment, writer,
headGraph.vertices.contains(segment), epoch, fileStore.getTracker());
+ }
+
+ writer.write("edgedef>node1 VARCHAR, node2 VARCHAR, head
BOOLEAN\n");
+ for (Entry<UUID, Set<UUID>> edge : segmentGraph.edges.entrySet()) {
+ UUID from = edge.getKey();
+ for (UUID to : edge.getValue()) {
+ if (!from.equals(to)) {
+ Set<UUID> he = headGraph.edges.get(from);
+ boolean inHead = he != null && he.contains(to);
+ writer.write(from + "," + to + "," + inHead + "\n");
+ }
+ }
+ }
+ } finally {
+ writer.close();
+ }
+ }
+
+ /**
+ * Parse the segment graph of a file store.
+ *
+ * @param fileStore file store to parse
+ * @return the segment graph rooted as the segment containing the head node
+ * state of {@code fileStore}.
+ * @throws IOException
+ */
+ @Nonnull
+ public static Graph<UUID> parseSegmentGraph(@Nonnull ReadOnlyStore
fileStore) throws IOException {
+ SegmentNodeState root = checkNotNull(fileStore).getHead();
+ HashSet<UUID> roots = newHashSet(root.getRecordId().asUUID());
+ return parseSegmentGraph(fileStore, roots, Functions.<UUID>identity());
+ }
+
+ /**
+ * Write the gc generation graph of a file store to a stream.
+ * <p>
+ * The graph is written in
+ * <a
href="https://gephi.github.io/users/supported-graph-formats/gdf-format/";>the
Guess GDF format</a>,
+ * which is easily imported into <a
href="https://gephi.github.io/";>Gephi</a>.
+ *
+ * @param fileStore file store to graph
+ * @param out stream to write the graph to
+ * @throws Exception
+ */
+ public static void writeGCGraph(@Nonnull ReadOnlyStore fileStore, @Nonnull
OutputStream out)
+ throws Exception {
+ PrintWriter writer = new PrintWriter(checkNotNull(out));
+ try {
+ Graph<Integer> gcGraph = parseGCGraph(checkNotNull(fileStore));
+
+ writer.write("nodedef>name VARCHAR, out-degree INT\n");
+ for (Integer gen : gcGraph.vertices) {
+ writer.write(gen + "," + getOutDegree(gcGraph, gen) + "\n");
+ }
+
+ writer.write("edgedef>node1 VARCHAR, node2 VARCHAR\n");
+ for (Entry<Integer, Set<Integer>> edge : gcGraph.edges.entrySet())
{
+ Integer from = edge.getKey();
+ for (Integer to : edge.getValue()) {
+ if (!from.equals(to) && to != -1) {
+ writer.write(from + "," + to + "\n");
+ }
+ }
+ }
+ } finally {
+ writer.close();
+ }
+ }
+
+ private static int getOutDegree(Graph<Integer> gcGraph, Integer gen) {
+ Set<Integer> tos = gcGraph.edges.get(gen);
+ if (tos == null) {
+ return 0;
+ } else {
+ int degree = tos.size();
+ if (tos.contains(-1)) { // don't count edges to bulk segments
+ degree--;
+ }
+ if (tos.contains(gen)) { // don't call edges to self
+ degree--;
+ }
+ return degree;
+ }
+ }
+
+ /**
+ * Parse the gc generation graph of a file store.
+ *
+ * @param fileStore file store to parse
+ * @return the gc generation graph rooted ad the segment containing the
head node
+ * state of {@code fileStore}.
+ * @throws IOException
+ */
+ @Nonnull
+ public static Graph<Integer> parseGCGraph(@Nonnull final ReadOnlyStore
fileStore)
+ throws IOException {
+ SegmentNodeState root = checkNotNull(fileStore).getHead();
+ HashSet<UUID> roots = newHashSet(root.getRecordId().asUUID());
+ return parseSegmentGraph(fileStore, roots, new Function<UUID,
Integer>() {
+ @Override @Nullable
+ public Integer apply(UUID segmentId) {
+ Map<String, String> info = getSegmentInfo(segmentId,
fileStore.getTracker());
+ if (info != null) {
+ return toInt(info.get("gc"), -1);
+ } else {
+ return -1;
+ }
+ }
+ });
+ }
+
+ private static int toInt(String number, int defaultValue) {
+ if (number == null) {
+ return defaultValue;
+ } else {
+ try {
+ return Integer.parseInt(number);
+ } catch (NumberFormatException e) {
+ return defaultValue;
+ }
+ }
+ }
+
+ /**
+ * Parse the segment graph of a file store starting with a given set of
root segments.
+ * The full segment graph is mapped through the passed {@code
homomorphism} to the
+ * graph returned by this function.
+ *
+ * @param fileStore file store to parse
+ * @param roots the initial set of segments
+ * @param homomorphism map from the segment graph into the returned graph
+ * @return the segment graph of {@code fileStore} rooted at {@code
roots} and mapped
+ * by {@code homomorphism}
+ * @throws IOException
+ */
+ @Nonnull
+ public static <T> Graph<T> parseSegmentGraph(
+ @Nonnull ReadOnlyStore fileStore,
+ @Nonnull Set<UUID> roots,
+ @Nonnull final Function<UUID, T> homomorphism) throws IOException {
+ final Graph<T> graph = new Graph<T>();
+
+ checkNotNull(homomorphism);
+ checkNotNull(fileStore).traverseSegmentGraph(checkNotNull(roots),
+ new SegmentGraphVisitor() {
+ @Override
+ public void accept(@Nonnull UUID from, @CheckForNull UUID to) {
+ graph.addVertex(homomorphism.apply(from));
+ if (to != null) {
+ graph.addVertex((homomorphism.apply(to)));
+ graph.addEdge(homomorphism.apply(from),
homomorphism.apply(to));
+ }
+ }
+ });
+ return graph;
+ }
+
+ /**
+ * Parser the head graph. The head graph is the sub graph of the segment
+ * graph containing the {@code root}.
+ * @param root
+ * @return the head graph of {@code root}.
+ */
+ @Nonnull
+ public static Graph<UUID> parseHeadGraph(@Nonnull RecordId root) {
+ final Graph<UUID> graph = new Graph<UUID>();
+
+ new SegmentParser() {
+ private void addEdge(RecordId from, RecordId to) {
+ graph.addVertex(from.asUUID());
+ graph.addVertex(to.asUUID());
+ graph.addEdge(from.asUUID(), to.asUUID());
+ }
+ @Override
+ protected void onNode(RecordId parentId, RecordId nodeId) {
+ super.onNode(parentId, nodeId);
+ addEdge(parentId, nodeId);
+ }
+ @Override
+ protected void onTemplate(RecordId parentId, RecordId templateId) {
+ super.onTemplate(parentId, templateId);
+ addEdge(parentId, templateId);
+ }
+ @Override
+ protected void onMap(RecordId parentId, RecordId mapId, MapRecord
map) {
+ super.onMap(parentId, mapId, map);
+ addEdge(parentId, mapId);
+ }
+ @Override
+ protected void onMapDiff(RecordId parentId, RecordId mapId,
MapRecord map) {
+ super.onMapDiff(parentId, mapId, map);
+ addEdge(parentId, mapId);
+ }
+ @Override
+ protected void onMapLeaf(RecordId parentId, RecordId mapId,
MapRecord map) {
+ super.onMapLeaf(parentId, mapId, map);
+ addEdge(parentId, mapId);
+ }
+ @Override
+ protected void onMapBranch(RecordId parentId, RecordId mapId,
MapRecord map) {
+ super.onMapBranch(parentId, mapId, map);
+ addEdge(parentId, mapId);
+ }
+ @Override
+ protected void onProperty(RecordId parentId, RecordId propertyId,
PropertyTemplate template) {
+ super.onProperty(parentId, propertyId, template);
+ addEdge(parentId, propertyId);
+ }
+ @Override
+ protected void onValue(RecordId parentId, RecordId valueId,
Type<?> type) {
+ super.onValue(parentId, valueId, type);
+ addEdge(parentId, valueId);
+ }
+ @Override
+ protected void onBlob(RecordId parentId, RecordId blobId) {
+ super.onBlob(parentId, blobId);
+ addEdge(parentId, blobId);
+ }
+ @Override
+ protected void onString(RecordId parentId, RecordId stringId) {
+ super.onString(parentId, stringId);
+ addEdge(parentId, stringId);
+ }
+ @Override
+ protected void onList(RecordId parentId, RecordId listId, int
count) {
+ super.onList(parentId, listId, count);
+ addEdge(parentId, listId);
+ }
+ @Override
+ protected void onListBucket(RecordId parentId, RecordId listId,
int index, int count, int capacity) {
+ super.onListBucket(parentId, listId, index, count, capacity);
+ addEdge(parentId, listId);
+ }
+ }.parseNode(checkNotNull(root));
+ return graph;
+ }
+
+ private static void writeNode(UUID node, PrintWriter writer, boolean
inHead, Date epoch, SegmentTracker tracker) {
+ Map<String, String> sInfo = getSegmentInfo(node, tracker);
+ if (sInfo == null) {
+ writer.write(node + ",b,bulk,b,-1,-1," + inHead + "\n");
+ } else {
+ long t = asLong(sInfo.get("t"));
+ long ts = t - epoch.getTime();
+ checkArgument(ts >= Integer.MIN_VALUE && ts <= Integer.MAX_VALUE,
+ "Time stamp (" + new Date(t) + ") not in epoch (" +
+ new Date(epoch.getTime() + Integer.MIN_VALUE) + " - " +
+ new Date(epoch.getTime() + Integer.MAX_VALUE) + ")");
+ writer.write(node +
+ "," + sInfo.get("sno") +
+ ",data" +
+ "," + sInfo.get("wid") +
+ "," + sInfo.get("gc") +
+ "," + ts +
+ "," + inHead + "\n");
+ }
+ }
+
+ private static long asLong(String string) {
+ return Long.valueOf(string);
+ }
+
+ private static Map<String, String> getSegmentInfo(UUID node,
SegmentTracker tracker) {
+ if (isDataSegmentId(node.getLeastSignificantBits())) {
+ SegmentId id = tracker.getSegmentId(node.getMostSignificantBits(),
node.getLeastSignificantBits());
+ String info = id.getSegment().getSegmentInfo();
+ if (info != null) {
+ JsopTokenizer tokenizer = new JsopTokenizer(info);
+ tokenizer.read('{');
+ return JsonObject.create(tokenizer).getProperties();
+ } else {
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+
+}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1720094&r1=1720093&r2=1720094&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Tue Dec 15 09:05:19 2015
@@ -69,6 +69,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.segment.PersistedCompactionMap;
import org.apache.jackrabbit.oak.plugins.segment.RecordId;
import org.apache.jackrabbit.oak.plugins.segment.Segment;
+import
org.apache.jackrabbit.oak.plugins.segment.SegmentGraph.SegmentGraphVisitor;
import org.apache.jackrabbit.oak.plugins.segment.SegmentId;
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState;
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
@@ -1370,15 +1371,16 @@ public class FileStore implements Segmen
/**
* Build the graph of segments reachable from an initial set of
segments
- * @param referencedIds the initial set of segments
+ * @param roots the initial set of segments
+ * @param visitor visitor receiving call back while following the
segment graph
* @throws IOException
*/
- public Map<UUID, Set<UUID>> getSegmentGraph(Set<UUID> referencedIds)
throws IOException {
- Map<UUID, Set<UUID>> graph = newHashMap();
+ public void traverseSegmentGraph(
+ @Nonnull Set<UUID> roots,
+ @Nonnull SegmentGraphVisitor visitor) throws IOException {
for (TarReader reader : super.readers) {
- graph.putAll(reader.getReferenceGraph(referencedIds));
+ reader.traverseSegmentGraph(checkNotNull(roots),
checkNotNull(visitor));
}
- return graph;
}
@Override
@@ -1417,7 +1419,6 @@ public class FileStore implements Segmen
public boolean maybeCompact(boolean cleanup) {
throw new UnsupportedOperationException("Read Only Store");
}
-
}
private class SetHead implements Callable<Boolean> {
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java?rev=1720094&r1=1720093&r2=1720094&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
Tue Dec 15 09:05:19 2015
@@ -17,6 +17,7 @@
package org.apache.jackrabbit.oak.plugins.segment.file;
import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Lists.newArrayListWithCapacity;
import static com.google.common.collect.Maps.newHashMap;
@@ -36,7 +37,6 @@ import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Collections;
-import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -52,6 +52,7 @@ import javax.annotation.Nonnull;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.plugins.segment.CompactionMap;
+import
org.apache.jackrabbit.oak.plugins.segment.SegmentGraph.SegmentGraphVisitor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -675,32 +676,37 @@ class TarReader implements Closeable {
/**
* Build the graph of segments reachable from an initial set of segments
- * @param referencedIds the initial set of segments
+ * @param roots the initial set of segments
+ * @param visitor visitor receiving call back while following the
segment graph
* @throws IOException
*/
- Map<UUID, Set<UUID>> getReferenceGraph(Set<UUID> referencedIds) throws
IOException {
+ public void traverseSegmentGraph(
+ @Nonnull Set<UUID> roots,
+ @Nonnull SegmentGraphVisitor visitor) throws IOException {
+ checkNotNull(roots);
+ checkNotNull(visitor);
Map<UUID, List<UUID>> graph = getGraph();
- Map<UUID, Set<UUID>> refGraph = newHashMap();
TarEntry[] entries = getEntries();
for (int i = entries.length - 1; i >= 0; i--) {
TarEntry entry = entries[i];
UUID id = new UUID(entry.msb(), entry.lsb());
- if (!referencedIds.remove(id)) {
- // this segment is not referenced anywhere
- entries[i] = null;
- } else {
- if (isDataSegmentId(entry.lsb())) {
- // this is a referenced data segment, so follow the graph
- List<UUID> refIds = getReferences(entry, id, graph);
- if (refIds != null) {
- refGraph.put(id, new HashSet<UUID>(refIds));
- referencedIds.addAll(refIds);
+ if (roots.remove(id) && isDataSegmentId(entry.lsb())) {
+ // this is a referenced data segment, so follow the graph
+ List<UUID> refIds = getReferences(entry, id, graph);
+ if (refIds != null) {
+ for (UUID refId : refIds) {
+ visitor.accept(id, refId);
+ roots.add(refId);
}
+ } else {
+ visitor.accept(id, null);
}
+ } else {
+ // this segment is not referenced anywhere
+ visitor.accept(id, null);
}
}
- return refGraph;
}
/**
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/package-info.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/package-info.java?rev=1720094&r1=1720093&r2=1720094&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/package-info.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/package-info.java
Tue Dec 15 09:05:19 2015
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-@Version("3.2.0")
+@Version("4.0.0")
@Export(optional = "provide:=true")
package org.apache.jackrabbit.oak.plugins.segment.file;
Added:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraphTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraphTest.java?rev=1720094&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraphTest.java
(added)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraphTest.java
Tue Dec 15 09:05:19 2015
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment;
+
+
+import static com.google.common.collect.Maps.newHashMap;
+import static com.google.common.collect.Sets.newHashSet;
+import static java.io.File.createTempFile;
+import static java.util.Collections.singleton;
+import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import static
org.apache.jackrabbit.oak.plugins.segment.SegmentGraph.parseSegmentGraph;
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+
+import javax.annotation.Nonnull;
+
+import org.apache.jackrabbit.oak.plugins.segment.SegmentGraph.Graph;
+import org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy;
+import
org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy.CleanupType;
+import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
+import org.apache.jackrabbit.oak.plugins.segment.file.FileStore.ReadOnlyStore;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class SegmentGraphTest {
+ private final Set<UUID> segments = newHashSet();
+ private final Map<UUID, Set<UUID>> references = newHashMap();
+
+ private final Set<Integer> gcGenerations = newHashSet();
+ private final Map<Integer, Set<Integer>> gcReferences = newHashMap();
+
+ private File storeDir;
+
+ private void addSegment(SegmentNodeState node) {
+ segments.add(node.getRecordId().asUUID());
+ }
+
+ private void addReference(SegmentNodeState from, SegmentNodeState to) {
+ addReference(references, from, to);
+ }
+
+ private static void addReference(Map<UUID, Set<UUID>> map,
SegmentNodeState from, SegmentNodeState to) {
+ UUID fromUUID = from.getRecordId().asUUID();
+ UUID toUUID = to.getRecordId().asUUID();
+ Set<UUID> tos = map.get(fromUUID);
+ if (tos == null) {
+ tos = newHashSet();
+ map.put(fromUUID, tos);
+ }
+ tos.add(toUUID);
+ }
+
+ @Before
+ public void setup() throws IOException {
+ storeDir = createTempFile(SegmentGraph.class.getSimpleName(), null);
+ storeDir.delete();
+ storeDir.mkdir();
+
+ FileStore store = FileStore.newFileStore(storeDir).create();
+ CompactionStrategy strategy = new CompactionStrategy(false, false,
CleanupType.CLEAN_NONE, 0, (byte) 0) {
+ @Override
+ public boolean compacted(@Nonnull Callable<Boolean> setHead)
throws Exception {
+ return setHead.call();
+ }
+ };
+ strategy.setPersistCompactionMap(false);
+ store.setCompactionStrategy(strategy);
+
+ addSegment(store.getHead());
+
+ SegmentWriter writer = store.createSegmentWriter("testWriter");
+ SegmentNodeState node1 = writer.writeNode(EMPTY_NODE);
+ addSegment(node1);
+ writer.flush();
+
+ SegmentNodeState node2 = writer.writeNode(EMPTY_NODE);
+ addSegment(node2);
+ addReference(node2, node1); // Through the respective templates,
which are de-duplicated
+ writer.flush();
+
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setChildNode("foo", node1);
+ builder.setChildNode("bar", node2);
+
+ SegmentNodeState node3 = writer.writeNode(builder.getNodeState());
+ addSegment(node3);
+ addReference(node3, node1);
+ addReference(node3, node2);
+ writer.flush();
+
+ store.setHead(store.getHead(), node3);
+
+ store.compact();
+ addSegment(store.getHead());
+ builder = node3.builder();
+ builder.setProperty("p", 42);
+ SegmentNodeState node4 = writer.writeNode(builder.getNodeState());
+ addReference(node4, node3);
+ addSegment(node4);
+ store.setHead(store.getHead(), node4);
+
+ store.close();
+
+ gcGenerations.add(0);
+ gcGenerations.add(1);
+ gcReferences.put(0, singleton(0));
+ gcReferences.put(1, singleton(0));
+ }
+
+ @After
+ public void tearDown() {
+ storeDir.delete();
+ }
+
+ @Test
+ public void testSegmentGraph() throws IOException {
+ ReadOnlyStore store = new ReadOnlyStore(storeDir);
+ try {
+ Graph<UUID> segmentGraph = parseSegmentGraph(store);
+ assertEquals(segments, segmentGraph.vertices);
+ assertEquals(references, segmentGraph.edges);
+ } finally {
+ store.close();
+ }
+ }
+
+ @Test
+ public void testGCGraph() throws IOException {
+ ReadOnlyStore store = new ReadOnlyStore(storeDir);
+ try {
+ Graph<Integer> gcGraph = SegmentGraph.parseGCGraph(store);
+ assertEquals(gcGenerations, gcGraph.vertices);
+ assertEquals(gcReferences, gcGraph.edges);
+ } finally {
+ store.close();
+ }
+ }
+}
Modified: jackrabbit/oak/trunk/oak-run/README.md
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/README.md?rev=1720094&r1=1720093&r2=1720094&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/README.md (original)
+++ jackrabbit/oak/trunk/oak-run/README.md Tue Dec 15 09:05:19 2015
@@ -120,6 +120,7 @@ a negative offset translating all timest
(derived from journal.log if not
given)
--output <File> Output file (default: segments.gdf)
+ --gc Write the gc generation graph instead of the full graph
History
-------
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/segment/FileStoreHelper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/segment/FileStoreHelper.java?rev=1720094&r1=1720093&r2=1720094&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/segment/FileStoreHelper.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/segment/FileStoreHelper.java
Tue Dec 15 09:05:19 2015
@@ -18,35 +18,23 @@
*/
package org.apache.jackrabbit.oak.plugins.segment;
-import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.Lists.newArrayList;
-import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Sets.newHashSet;
import static java.util.Collections.reverseOrder;
-import static java.util.Collections.singleton;
import static java.util.Collections.sort;
-import static
org.apache.jackrabbit.oak.plugins.segment.SegmentId.isDataSegmentId;
import java.io.File;
import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintWriter;
import java.util.AbstractMap.SimpleImmutableEntry;
import java.util.ArrayDeque;
-import java.util.Date;
import java.util.Deque;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
-import org.apache.jackrabbit.oak.api.Type;
-import org.apache.jackrabbit.oak.commons.json.JsonObject;
-import org.apache.jackrabbit.oak.commons.json.JsopTokenizer;
import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
-import org.apache.jackrabbit.oak.plugins.segment.file.FileStore.ReadOnlyStore;
import org.apache.jackrabbit.oak.plugins.segment.file.JournalReader;
public final class FileStoreHelper {
@@ -107,176 +95,6 @@ public final class FileStoreHelper {
}
}
- /**
- * Write the segment graph of a file store to a stream.
- * <p>
- * The graph is written in
- * <a
href="https://gephi.github.io/users/supported-graph-formats/gdf-format/";>the
Guess GDF format</a>,
- * which is easily imported into <a
href="https://gephi.github.io/";>Gephi</a>.
- * As GDF only supports integers but the segment time stamps are encoded
as long
- * the {@code epoch} argument is used as a negative offset translating all
timestamps
- * into a valid int range.
- *
- * @param fileStore file store to graph
- * @param out stream to write the graph to
- * @param epoch epoch (in milliseconds)
- * @throws Exception
- */
- public static void writeSegmentGraph(ReadOnlyStore fileStore, OutputStream
out, Date epoch) throws Exception {
- PrintWriter writer = new PrintWriter(out);
- try {
- SegmentNodeState root = fileStore.getHead();
-
- // Segment graph starting from the segment containing root
- Map<UUID, Set<UUID>> segmentGraph = fileStore.getSegmentGraph(new
HashSet<UUID>(singleton(root.getRecordId().asUUID())));
-
- // All segment in the segment graph
- Set<UUID> segments = newHashSet();
- segments.addAll(segmentGraph.keySet());
- for (Set<UUID> tos : segmentGraph.values()) {
- segments.addAll(tos);
- }
-
- // Graph of segments containing the head state
- final Map<UUID, Set<UUID>> headGraph = newHashMap();
- final Set<UUID> headSegments = newHashSet();
- new SegmentParser() {
- private void addEdge(RecordId from, RecordId to) {
- UUID fromUUID = from.asUUID();
- UUID toUUID = to.asUUID();
- if (!fromUUID.equals(toUUID)) {
- Set<UUID> tos = headGraph.get(fromUUID);
- if (tos == null) {
- tos = newHashSet();
- headGraph.put(fromUUID, tos);
- }
- tos.add(toUUID);
- headSegments.add(fromUUID);
- headSegments.add(toUUID);
- }
- }
- @Override
- protected void onNode(RecordId parentId, RecordId nodeId) {
- super.onNode(parentId, nodeId);
- addEdge(parentId, nodeId);
- }
- @Override
- protected void onTemplate(RecordId parentId, RecordId
templateId) {
- super.onTemplate(parentId, templateId);
- addEdge(parentId, templateId);
- }
- @Override
- protected void onMap(RecordId parentId, RecordId mapId,
MapRecord map) {
- super.onMap(parentId, mapId, map);
- addEdge(parentId, mapId);
- }
- @Override
- protected void onMapDiff(RecordId parentId, RecordId mapId,
MapRecord map) {
- super.onMapDiff(parentId, mapId, map);
- addEdge(parentId, mapId);
- }
- @Override
- protected void onMapLeaf(RecordId parentId, RecordId mapId,
MapRecord map) {
- super.onMapLeaf(parentId, mapId, map);
- addEdge(parentId, mapId);
- }
- @Override
- protected void onMapBranch(RecordId parentId, RecordId mapId,
MapRecord map) {
- super.onMapBranch(parentId, mapId, map);
- addEdge(parentId, mapId);
- }
- @Override
- protected void onProperty(RecordId parentId, RecordId
propertyId, PropertyTemplate template) {
- super.onProperty(parentId, propertyId, template);
- addEdge(parentId, propertyId);
- }
- @Override
- protected void onValue(RecordId parentId, RecordId valueId,
Type<?> type) {
- super.onValue(parentId, valueId, type);
- addEdge(parentId, valueId);
- }
- @Override
- protected void onBlob(RecordId parentId, RecordId blobId) {
- super.onBlob(parentId, blobId);
- addEdge(parentId, blobId);
- }
- @Override
- protected void onString(RecordId parentId, RecordId stringId) {
- super.onString(parentId, stringId);
- addEdge(parentId, stringId);
- }
- @Override
- protected void onList(RecordId parentId, RecordId listId, int
count) {
- super.onList(parentId, listId, count);
- addEdge(parentId, listId);
- }
- @Override
- protected void onListBucket(RecordId parentId, RecordId
listId, int index, int count, int capacity) {
- super.onListBucket(parentId, listId, index, count,
capacity);
- addEdge(parentId, listId);
- }
- }.parseNode(root.getRecordId());
-
- writer.write("nodedef>name VARCHAR, label VARCHAR, type VARCHAR,
wid VARCHAR, gc INT, t INT, head BOOLEAN\n");
- for (UUID segment : segments) {
- writeNode(segment, writer, headSegments.contains(segment),
epoch, fileStore.getTracker());
- }
-
- writer.write("edgedef>node1 VARCHAR, node2 VARCHAR, head
BOOLEAN\n");
- for (Entry<UUID, Set<UUID>> edge : segmentGraph.entrySet()) {
- UUID from = edge.getKey();
- for (UUID to : edge.getValue()) {
- Set<UUID> he = headGraph.get(from);
- boolean inHead = he != null && he.contains(to);
- writer.write(from + "," + to + "," + inHead + "\n");
- }
- }
- } finally {
- writer.close();
- }
- }
-
- private static void writeNode(UUID node, PrintWriter writer, boolean
inHead, Date epoch, SegmentTracker tracker) {
- Map<String, String> sInfo = getSegmentInfo(node, tracker);
- if (sInfo == null) {
- writer.write(node + ",b,bulk,b,-1,-1," + inHead + "\n");
- } else {
- long t = asLong(sInfo.get("t"));
- long ts = t - epoch.getTime();
- checkArgument(ts >= Integer.MIN_VALUE && ts <= Integer.MAX_VALUE,
- "Time stamp (" + new Date(t) + ") not in epoch (" +
- new Date(epoch.getTime() + Integer.MIN_VALUE) + " - " +
- new Date(epoch.getTime() + Integer.MAX_VALUE) + ")");
- writer.write(node +
- "," + sInfo.get("sno") +
- ",data" +
- "," + sInfo.get("wid") +
- "," + sInfo.get("gc") +
- "," + ts +
- "," + inHead + "\n");
- }
- }
-
- private static long asLong(String string) {
- return Long.valueOf(string);
- }
-
- private static Map<String, String> getSegmentInfo(UUID node,
SegmentTracker tracker) {
- if (isDataSegmentId(node.getLeastSignificantBits())) {
- SegmentId id = tracker.getSegmentId(node.getMostSignificantBits(),
node.getLeastSignificantBits());
- String info = id.getSegment().getSegmentInfo();
- if (info != null) {
- JsopTokenizer tokenizer = new JsopTokenizer(info);
- tokenizer.read('{');
- return JsonObject.create(tokenizer).getProperties();
- } else {
- return null;
- }
- } else {
- return null;
- }
- }
-
public static List<String> readRevisions(File store) {
File journal = new File(store, "journal.log");
if (!journal.exists()) {
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java?rev=1720094&r1=1720093&r2=1720094&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
Tue Dec 15 09:05:19 2015
@@ -21,6 +21,8 @@ import static java.util.Arrays.asList;
import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
import static org.apache.jackrabbit.oak.checkpoint.Checkpoints.CP;
import static org.apache.jackrabbit.oak.plugins.segment.RecordType.NODE;
+import static
org.apache.jackrabbit.oak.plugins.segment.SegmentGraph.writeGCGraph;
+import static
org.apache.jackrabbit.oak.plugins.segment.SegmentGraph.writeSegmentGraph;
import static
org.apache.jackrabbit.oak.plugins.segment.file.FileStore.newFileStore;
import static
org.apache.jackrabbit.oak.plugins.segment.file.tooling.ConsistencyChecker.checkConsistency;
import static org.slf4j.LoggerFactory.getLogger;
@@ -99,7 +101,6 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.document.util.MapFactory;
import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
import org.apache.jackrabbit.oak.plugins.segment.FileStoreDiff;
-import org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper;
import org.apache.jackrabbit.oak.plugins.segment.PCMAnalyser;
import org.apache.jackrabbit.oak.plugins.segment.RecordId;
import org.apache.jackrabbit.oak.plugins.segment.RecordUsageAnalyser;
@@ -811,6 +812,9 @@ public final class Main {
OptionSpec<Long> epochArg = parser.accepts(
"epoch", "Epoch of the segment time stamps (derived from
journal.log if not given)")
.withRequiredArg().ofType(Long.class);
+ OptionSpec<Void> gcGraphArg = parser.accepts(
+ "gc", "Write the gc generation graph instead of the full
graph");
+
OptionSet options = parser.parse(args);
File directory = directoryArg.value(options);
@@ -847,7 +851,13 @@ public final class Main {
System.out.println("Setting epoch to " + epoch);
System.out.println("Writing graph to " + outFile);
- FileStoreHelper.writeSegmentGraph(fileStore, new
FileOutputStream(outFile), epoch);
+
+ FileOutputStream out = new FileOutputStream(outFile);
+ if (options.has(gcGraphArg)) {
+ writeGCGraph(fileStore, out);
+ } else {
+ writeSegmentGraph(fileStore, out, epoch);
+ }
}
private static void check(String[] args) throws IOException {
