Author: tomekr
Date: Mon Aug 28 10:01:59 2017
New Revision: 1806418
URL: http://svn.apache.org/viewvc?rev=1806418&view=rev
Log:
OAK-6560: Sidegrade uses too much memory
Fixed the progress logging.
Modified:
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/PersistingDiff.java
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java
Modified:
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/PersistingDiff.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/PersistingDiff.java?rev=1806418&r1=1806417&r2=1806418&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/PersistingDiff.java
(original)
+++
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/PersistingDiff.java
Mon Aug 28 10:01:59 2017
@@ -16,6 +16,7 @@
*/
package org.apache.jackrabbit.oak.upgrade;
+import com.google.common.collect.Lists;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeBuilder;
import org.apache.jackrabbit.oak.segment.RecordId;
@@ -26,11 +27,16 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStateDiff;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
@@ -38,6 +44,8 @@ import static org.apache.jackrabbit.oak.
public class PersistingDiff implements NodeStateDiff {
+ private static final Logger LOG =
LoggerFactory.getLogger(PersistingDiff.class);
+
/**
* Number of content updates that need to happen before the updates
* are automatically purged to the underlying segments.
@@ -51,6 +59,12 @@ public class PersistingDiff implements N
private final BlobStore blobStore;
+ private final PersistingDiff parent;
+
+ private final String nodeName;
+
+ private final Reporter reporter;
+
@Nonnull
private MemoryNodeBuilder builder;
@@ -62,12 +76,26 @@ public class PersistingDiff implements N
private long modCount;
+ private PersistingDiff(PersistingDiff parent, String nodeName, @Nonnull
NodeState base) {
+ this.writer = parent.writer;
+ this.reader = parent.reader;
+ this.blobStore = parent.blobStore;
+ this.reporter = parent.reporter;
+ this.builder = new MemoryNodeBuilder(checkNotNull(base));
+ this.parent = parent;
+ this.base = base;
+ this.nodeName = nodeName;
+ }
+
private PersistingDiff(SegmentWriter writer, SegmentReader reader,
BlobStore blobStore, @Nonnull NodeState base) {
this.writer = writer;
this.reader = reader;
this.blobStore = blobStore;
+ this.reporter = new Reporter();
this.builder = new MemoryNodeBuilder(checkNotNull(base));
+ this.parent = null;
this.base = base;
+ this.nodeName = null;
}
public static SegmentNodeState applyDiffOnNodeState(
@@ -78,23 +106,32 @@ public class PersistingDiff implements N
return new PersistingDiff(fileStore.getWriter(),
fileStore.getReader(), fileStore.getBlobStore(), onto).diff(before, after);
}
- private SegmentNodeState copyToSegmentNodeState(NodeState state) throws
IOException {
- return applyDiffOnNodeState(EMPTY_NODE, state, EMPTY_NODE);
- }
-
- private SegmentNodeState applyDiffOnNodeState(
- @Nonnull NodeState before,
- @Nonnull NodeState after,
- @Nonnull NodeState onto) throws IOException {
- return new PersistingDiff(writer, reader, blobStore,
onto).diff(before, after);
- }
-
private void updated() throws IOException {
- if (++modCount % UPDATE_LIMIT == 0) {
+ if (modCount % UPDATE_LIMIT == 0) {
RecordId newBaseId = writer.writeNode(builder.getNodeState(),
null);
SegmentNodeState newBase = new SegmentNodeState(reader, writer,
blobStore, newBaseId);
builder = new MemoryNodeBuilder(newBase);
}
+ modCount++;
+ }
+
+ private String getPath() {
+ List<String> segments = new ArrayList<>();
+ PersistingDiff currentDiff = this;
+ while (currentDiff != null) {
+ if (currentDiff.nodeName != null) {
+ segments.add(currentDiff.nodeName);
+ }
+ currentDiff = currentDiff.parent;
+ }
+ segments = Lists.reverse(segments);
+
+ StringBuilder path = new StringBuilder();
+ for (String segment : segments) {
+ path.append("/");
+ path.append(segment);
+ }
+ return path.toString();
}
@CheckForNull
@@ -106,6 +143,7 @@ public class PersistingDiff implements N
NodeState nodeState = builder.getNodeState();
checkState(modCount == 0 || !(nodeState instanceof
SegmentNodeState));
RecordId nodeId = writer.writeNode(nodeState,
getStableIdBytes(after));
+ reporter.reportNode(this::getPath);
return new SegmentNodeState(reader, writer, blobStore, nodeId);
} else {
return null;
@@ -133,7 +171,7 @@ public class PersistingDiff implements N
@Override
public boolean childNodeAdded(@Nonnull String name, @Nonnull NodeState
after) {
try {
- SegmentNodeState segmentNodeState = copyToSegmentNodeState(after);
+ SegmentNodeState segmentNodeState = new PersistingDiff(this, name,
EMPTY_NODE).diff(EMPTY_NODE, after);
if (segmentNodeState != null) {
updated();
builder.setChildNode(name, segmentNodeState);
@@ -150,7 +188,7 @@ public class PersistingDiff implements N
@Override
public boolean childNodeChanged(@Nonnull String name, @Nonnull NodeState
before, @Nonnull NodeState after) {
try {
- SegmentNodeState compacted = applyDiffOnNodeState(before, after,
base.getChildNode(name));
+ SegmentNodeState compacted = new PersistingDiff(this, name,
base.getChildNode(name)).diff(before, after);
if (compacted != null) {
updated();
builder.setChildNode(name, compacted);
@@ -184,4 +222,17 @@ public class PersistingDiff implements N
return null;
}
}
+
+ private static class Reporter {
+
+ private long count = 0;
+
+ public void reportNode(Supplier<String> pathSupplier) {
+ if (count > 0 && count % RepositorySidegrade.LOG_NODE_COPY == 0) {
+ LOG.info("Copying node {}: {}", count, pathSupplier.get());
+ }
+ count++;
+ }
+
+ }
}
Modified:
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java?rev=1806418&r1=1806417&r2=1806418&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java
(original)
+++
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java
Mon Aug 28 10:01:59 2017
@@ -83,7 +83,7 @@ public class RepositorySidegrade {
private static final Logger LOG =
LoggerFactory.getLogger(RepositorySidegrade.class);
- private static final int LOG_NODE_COPY =
Integer.getInteger("oak.upgrade.logNodeCopy", 10000);
+ static final int LOG_NODE_COPY =
Integer.getInteger("oak.upgrade.logNodeCopy", 10000);
private static final String WORKSPACE_NAME_PROP =
"oak.upgrade.workspaceName";
@@ -423,13 +423,17 @@ public class RepositorySidegrade {
}
private NodeState copyDiffToTarget(NodeState before, NodeState after,
NodeState targetRoot, boolean tracePaths) throws IOException,
CommitFailedException {
- NodeState currentRoot = wrapNodeState(after, tracePaths, true);
- NodeState baseRoot = wrapNodeState(before, false, true);
NodeBuilder targetBuilder = targetRoot.builder();
if (targetFileStore == null) {
+ NodeState currentRoot = wrapNodeState(after, tracePaths, true);
+ NodeState baseRoot = wrapNodeState(before, false, true);
+
currentRoot.compareAgainstBaseState(baseRoot, new
ApplyDiff(targetBuilder));
} else {
+ NodeState currentRoot = wrapNodeState(after, false, true);
+ NodeState baseRoot = wrapNodeState(before, false, true);
+
SegmentNodeState state =
PersistingDiff.applyDiffOnNodeState(targetFileStore, baseRoot, currentRoot,
targetRoot);
state.compareAgainstBaseState(targetRoot, new
ApplyDiff(targetBuilder));
}
