Author: adulceanu
Date: Mon Feb 13 15:56:39 2017
New Revision: 1782796
URL: http://svn.apache.org/viewvc?rev=1782796&view=rev
Log:
OAK-5620 - Simplify consistency check
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java?rev=1782796&r1=1782795&r2=1782796&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java
Mon Feb 13 15:56:39 2017
@@ -36,53 +36,48 @@ class CheckCommand implements Command {
ArgumentAcceptingOptionSpec<String> journal = parser.accepts(
"journal", "journal file")
.withRequiredArg().ofType(String.class).defaultsTo("journal.log");
- OptionSpec deep = parser.accepts(
- "deep", "<deprecated> enable deep consistency checking. ");
+ OptionSpec<?> deep = parser.accepts(
+ "deep", "<deprecated> enable deep consistency checking.");
ArgumentAcceptingOptionSpec<Long> notify = parser.accepts(
"notify", "number of seconds between progress notifications")
.withRequiredArg().ofType(Long.class).defaultsTo(Long.MAX_VALUE);
- OptionSpec bin = parser.accepts("bin", "read the content of binary
properties");
- OptionSpec segment = parser.accepts("segment", "Use oak-segment
instead of oak-segment-tar");
- OptionSpec ioStatistics = parser.accepts("io-stats", "Print I/O
statistics (only for oak-segment-tar)");
+ OptionSpec<?> bin = parser.accepts("bin", "read the content of binary
properties");
+ OptionSpec<?> segment = parser.accepts("segment", "Use oak-segment
instead of oak-segment-tar");
+ OptionSpec<?> ioStatistics = parser.accepts("io-stats", "Print I/O
statistics (only for oak-segment-tar)");
OptionSet options = parser.parse(args);
+
+ PrintWriter out = new PrintWriter(System.out, true);
+ PrintWriter err = new PrintWriter(System.err, true);
if (options.nonOptionArguments().size() != 1) {
- printUsage(parser);
+ printUsage(parser, err);
}
File dir = isValidFileStoreOrFail(new
File(options.nonOptionArguments().get(0).toString()));
String journalFileName = journal.value(options);
long debugLevel = notify.value(options);
-
- long binLen = 0L;
- if (options.has(bin)) {
- binLen = -1L;
- }
-
if (options.has(deep)) {
- printUsage(parser, "The --deep option was deprecated! Please do
not use it in the future!"
+ printUsage(parser, err, "The --deep option was deprecated! Please
do not use it in the future!"
, "A deep scan of the content tree, traversing every node,
will be performed by default.");
}
- PrintWriter out = new PrintWriter(System.out, true);
- PrintWriter err = new PrintWriter(System.err, true);
if (options.has(segment)) {
- SegmentUtils.check(dir, journalFileName, true, debugLevel, binLen);
+ SegmentUtils.check(dir, journalFileName, debugLevel,
options.has(bin));
} else {
- SegmentTarUtils.check(dir, journalFileName, true, debugLevel,
binLen, options.has(ioStatistics), out, err);
+ SegmentTarUtils.check(dir, journalFileName, debugLevel,
options.has(bin), options.has(ioStatistics), out, err);
}
}
- private void printUsage(OptionParser parser, String... messages) throws
IOException {
+ private void printUsage(OptionParser parser, PrintWriter err, String...
messages) throws IOException {
for (String message : messages) {
- System.err.println(message);
+ err.println(message);
}
- System.err.println("usage: check path/to/segmentstore <options>");
- parser.printHelpOn(System.err);
+ err.println("usage: check path/to/segmentstore <options>");
+ parser.printHelpOn(err);
System.exit(1);
}
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java?rev=1782796&r1=1782795&r2=1782796&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
Mon Feb 13 15:56:39 2017
@@ -185,14 +185,13 @@ final class SegmentTarUtils {
.run();
}
- static void check(File dir, String journalFileName, boolean fullTraversal,
long debugLevel, long binLen,
- boolean ioStatistics, PrintWriter outWriter, PrintWriter
errWriter) {
+ static void check(File dir, String journalFileName, long debugLevel,
boolean checkBinaries, boolean ioStatistics,
+ PrintWriter outWriter, PrintWriter errWriter) {
Check.builder()
.withPath(dir)
.withJournal(journalFileName)
- .withFullTraversal(fullTraversal)
.withDebugInterval(debugLevel)
- .withMinimumBinaryLength(binLen)
+ .withCheckBinaries(checkBinaries)
.withIOStatistics(ioStatistics)
.withOutWriter(outWriter)
.withErrWriter(errWriter)
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java?rev=1782796&r1=1782795&r2=1782796&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java
Mon Feb 13 15:56:39 2017
@@ -162,8 +162,8 @@ class SegmentUtils {
}
}
- static void check(File dir, String journalFileName, boolean fullTraversal,
long debugLevel, long binLen) throws IOException,
InvalidFileStoreVersionException {
- checkConsistency(dir, journalFileName, fullTraversal, debugLevel,
binLen);
+ static void check(File dir, String journalFileName, long debugLevel,
boolean checkBin) throws IOException, InvalidFileStoreVersionException {
+ checkConsistency(dir, journalFileName, true, debugLevel, checkBin ?
-1L : 0L);
}
static void compact(File directory, boolean force) throws IOException,
InvalidFileStoreVersionException {
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java?rev=1782796&r1=1782795&r2=1782796&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java
Mon Feb 13 15:56:39 2017
@@ -20,7 +20,6 @@
package org.apache.jackrabbit.oak.segment.file.tooling;
import static com.google.common.collect.Sets.newHashSet;
-import static java.lang.Math.min;
import static org.apache.jackrabbit.oak.api.Type.BINARIES;
import static org.apache.jackrabbit.oak.api.Type.BINARY;
import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
@@ -88,16 +87,18 @@ public class ConsistencyChecker implemen
private final PrintWriter errWriter;
+ private int nodeCount;
+
+ private int propertyCount;
+
/**
- * Run a consistency check.
+ * Run a full traversal consistency check.
*
* @param directory directory containing the tar files
* @param journalFileName name of the journal file containing the
revision history
- * @param fullTraversal full traversal consistency check if {@code
true}. Only try
- * to access the root node otherwise.
* @param debugInterval number of seconds between printing progress
information to
* the console during the full traversal phase.
- * @param binLen number of bytes to read from binary properties.
-1 for all.
+ * @param checkBinaries if {@code true} full content of binary
properties will be scanned
* @param ioStatistics if {@code true} prints I/O statistics gathered
while consistency
* check was performed
* @param outWriter text output stream writer
@@ -107,9 +108,8 @@ public class ConsistencyChecker implemen
public static void checkConsistency(
File directory,
String journalFileName,
- boolean fullTraversal,
long debugInterval,
- long binLen,
+ boolean checkBinaries,
boolean ioStatistics,
PrintWriter outWriter,
PrintWriter errWriter
@@ -118,26 +118,23 @@ public class ConsistencyChecker implemen
JournalReader journal = new JournalReader(new File(directory,
journalFileName));
ConsistencyChecker checker = new ConsistencyChecker(directory,
debugInterval, ioStatistics, outWriter, errWriter)
) {
- checker.print("Searching for last good revision in {0}",
journalFileName);
- Set<String> badPaths = newHashSet();
+ Set<String> corruptPaths = newHashSet();
String latestGoodRevision = null;
int revisionCount = 0;
while (journal.hasNext() && latestGoodRevision == null) {
String revision = journal.next();
try {
- checker.print("Checking revision {0}", revision);
revisionCount++;
- String badPath = checker.check(revision, badPaths, binLen);
- if (badPath == null && fullTraversal) {
- badPath = checker.traverse(revision, binLen);
- }
- if (badPath == null) {
+
+ String corruptPath = checker.checkRevision(revision,
corruptPaths, "/", checkBinaries);
+
+ if (corruptPath == null) {
checker.print("Found latest good revision {0}",
revision);
checker.print("Searched through {0} revisions",
revisionCount);
latestGoodRevision = revision;
} else {
- badPaths.add(badPath);
+ corruptPaths.add(corruptPath);
checker.print("Broken revision {0}", revision);
}
} catch (IllegalArgumentException e) {
@@ -191,68 +188,61 @@ public class ConsistencyChecker implemen
this.errWriter = errWriter;
}
+
/**
- * Check whether the nodes and all its properties of all given
- * {@code paths} are consistent at the given {@code revision}.
- *
- * @param revision revision to check
- * @param paths paths to check
- * @param binLen number of bytes to read from binary properties. -1 for
all.
- * @return Path of the first inconsistency detected or {@code null} if
none.
+ * Checks the consistency of the supplied {@code path} at the given {@code
revision},
+ * starting first with already known {@code corruptPaths}.
+ *
+ * @param revision revision to be checked
+ * @param corruptPaths already known corrupt paths from previous revisions
+ * @param path initial path from which to start the consistency
check,
+ * provided there are no corrupt paths.
+ * @param checkBinaries if {@code true} full content of binary properties
will be scanned
+ * @return {@code null}, if the content tree rooted at path
is consistent
+ * in this revision or the path of the first
inconsistency otherwise.
*/
- public String check(String revision, Set<String> paths, long binLen) {
- store.setRevision(revision);
- for (String path : paths) {
- String err = checkPath(path, binLen);
- if (err != null) {
- return err;
- }
- }
- return null;
- }
-
- private String checkPath(String path, long binLen) {
+ public String checkRevision(String revision, Set<String> corruptPaths,
String path, boolean checkBinaries) {
+ print("Checking revision {0}", revision);
+ String result = null;
+
try {
print("Checking {0}", path);
+ store.setRevision(revision);
NodeState root =
SegmentNodeStoreBuilders.builder(store).build().getRoot();
- String parentPath = getParentPath(path);
- String name = getName(path);
- NodeState parent = getNode(root, parentPath);
- if (!denotesRoot(path) && parent.hasChildNode(name)) {
- return traverse(parent.getChildNode(name), path, false,
binLen);
- } else {
- return traverse(parent, parentPath, false, binLen);
- }
- } catch (RuntimeException e) {
- printError("Error while checking {0}: {1}", path, e.getMessage());
- return path;
- }
- }
- private int nodeCount;
- private int propertyCount;
+ for (String corruptPath : corruptPaths) {
+ NodeWrapper wrapper =
NodeWrapper.deriveTraversableNodeOnPath(root, corruptPath);
+ result = checkNode(wrapper.node, wrapper.path, checkBinaries);
+
+ if (result != null) {
+ return result;
+ }
+ }
- /**
- * Traverse the given {@code revision}
- * @param revision revision to travers
- * @param binLen number of bytes to read from binary properties. -1 for
all.
- */
- public String traverse(String revision, long binLen) {
- try {
- store.setRevision(revision);
nodeCount = 0;
propertyCount = 0;
- String result =
traverse(SegmentNodeStoreBuilders.builder(store).build()
- .getRoot(), "/", true, binLen);
+
+ NodeWrapper wrapper =
NodeWrapper.deriveTraversableNodeOnPath(root, path);
+ result = checkNodeAndDescendants(wrapper.node, wrapper.path,
checkBinaries);
print("Checked {0} nodes and {1} properties", nodeCount,
propertyCount);
+
return result;
} catch (RuntimeException e) {
- printError("Error while traversing {0}", revision, e.getMessage());
- return "/";
+ printError("Error while traversing {0}: {1}", revision,
e.getMessage());
+ return path;
}
}
-
- private String traverse(NodeState node, String path, boolean deep, long
binLen) {
+
+ /**
+ * Checks the consistency of a node and its properties at the given path.
+ *
+ * @param node node to be checked
+ * @param path path of the node
+ * @param checkBinaries if {@code true} full content of binary
properties will be scanned
+ * @return {@code null}, if the node is consistent,
+ * or the path of the first inconsistency
otherwise.
+ */
+ private String checkNode(NodeState node, String path, boolean
checkBinaries) {
try {
debug("Traversing {0}", path);
nodeCount++;
@@ -260,44 +250,85 @@ public class ConsistencyChecker implemen
debug("Checking {0}/{1}", path, propertyState);
Type<?> type = propertyState.getType();
if (type == BINARY) {
- traverse(propertyState.getValue(BINARY), binLen);
+ traverse(propertyState.getValue(BINARY), checkBinaries);
} else if (type == BINARIES) {
for (Blob blob : propertyState.getValue(BINARIES)) {
- traverse(blob, binLen);
+ traverse(blob, checkBinaries);
}
} else {
propertyCount++;
propertyState.getValue(type);
}
}
+
+ return null;
+ } catch (RuntimeException | IOException e) {
+ printError("Error while traversing {0}: {1}", path,
e.getMessage());
+ return path;
+ }
+ }
+
+ /**
+ * Recursively checks the consistency of a node and its descendants at the
given path.
+ * @param node node to be checked
+ * @param path path of the node
+ * @param checkBinaries if {@code true} full content of binary properties
will be scanned
+ * @return {@code null}, if the node is consistent,
+ * or the path of the first inconsistency otherwise.
+ */
+ private String checkNodeAndDescendants(NodeState node, String path,
boolean checkBinaries) {
+ String result = checkNode(node, path, checkBinaries);
+ if (result != null) {
+ return result;
+ }
+
+ try {
for (ChildNodeEntry cne : node.getChildNodeEntries()) {
String childName = cne.getName();
NodeState child = cne.getNodeState();
- if (deep) {
- String result = traverse(child, concat(path, childName),
true, binLen);
- if (result != null) {
- return result;
- }
+ result = checkNodeAndDescendants(child, concat(path,
childName), checkBinaries);
+ if (result != null) {
+ return result;
}
}
+
return null;
- } catch (RuntimeException | IOException e) {
+ } catch (RuntimeException e) {
printError("Error while traversing {0}: {1}", path,
e.getMessage());
return path;
}
}
-
- private void traverse(Blob blob, long length) throws IOException {
- if (length < 0) {
- length = Long.MAX_VALUE;
+
+ static class NodeWrapper {
+ NodeState node;
+ String path;
+
+ NodeWrapper(NodeState node, String path) {
+ this.node = node;
+ this.path = path;
+ }
+
+ static NodeWrapper deriveTraversableNodeOnPath(NodeState root, String
path) {
+ String parentPath = getParentPath(path);
+ String name = getName(path);
+ NodeState parent = getNode(root, parentPath);
+
+ if (!denotesRoot(path) && parent.hasChildNode(name)) {
+ return new NodeWrapper(parent.getChildNode(name), path);
+ } else {
+ return new NodeWrapper(parent, parentPath);
+ }
}
- if (length > 0 && !isExternal(blob)) {
+ }
+
+ private void traverse(Blob blob, boolean checkBinaries) throws IOException
{
+ if (checkBinaries && !isExternal(blob)) {
InputStream s = blob.getNewStream();
try {
byte[] buffer = new byte[8192];
- int l = s.read(buffer, 0, (int) min(buffer.length, length));
- while (l >= 0 && (length -= l) > 0) {
- l = s.read(buffer, 0, (int) min(buffer.length, length));
+ int l = s.read(buffer, 0, buffer.length);
+ while (l >= 0) {
+ l = s.read(buffer, 0, buffer.length);
}
} finally {
s.close();
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java?rev=1782796&r1=1782795&r2=1782796&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
Mon Feb 13 15:56:39 2017
@@ -48,11 +48,9 @@ public class Check implements Runnable {
private String journal;
- private boolean fullTraversal;
-
private long debugInterval = Long.MAX_VALUE;
- private long minimumBinaryLength;
+ private boolean checkBinaries;
private boolean ioStatistics;
@@ -88,19 +86,6 @@ public class Check implements Runnable {
}
/**
- * Should a full traversal of the segment store be performed? This
- * parameter is not required and defaults to {@code false}.
- *
- * @param fullTraversal {@code true} if a full traversal should be
- * performed, {@code false} otherwise.
- * @return this builder.
- */
- public Builder withFullTraversal(boolean fullTraversal) {
- this.fullTraversal = fullTraversal;
- return this;
- }
-
- /**
* Number of seconds between successive debug print statements. This
* parameter is not required and defaults to an arbitrary large number.
*
@@ -115,17 +100,15 @@ public class Check implements Runnable {
}
/**
- * Minimum amount of bytes to read from binary properties. This
- * parameter is not required and defaults to zero.
+ * Instruct the command to scan the full content of binary properties.
+ * This parameter is not required and defaults to {@code false}.
*
- * @param minimumBinaryLength minimum amount of bytes to read from
- * binary properties. If this parameter is
- * set to {@code -1}, every binary property
- * is read in its entirety.
+ * @param checkBinaries {@code true} if binary properties should be
+ * scanned, {@code false} otherwise.
* @return this builder.
*/
- public Builder withMinimumBinaryLength(long minimumBinaryLength) {
- this.minimumBinaryLength = minimumBinaryLength;
+ public Builder withCheckBinaries(boolean checkBinaries) {
+ this.checkBinaries = checkBinaries;
return this;
}
@@ -182,11 +165,9 @@ public class Check implements Runnable {
private final String journal;
- private final boolean fullTraversal;
-
private final long debugInterval;
- private final long minimumBinaryLength;
+ private final boolean checkBinaries;
private final boolean ioStatistics;
@@ -197,9 +178,8 @@ public class Check implements Runnable {
private Check(Builder builder) {
this.path = builder.path;
this.journal = builder.journal;
- this.fullTraversal = builder.fullTraversal;
this.debugInterval = builder.debugInterval;
- this.minimumBinaryLength = builder.minimumBinaryLength;
+ this.checkBinaries = builder.checkBinaries;
this.ioStatistics = builder.ioStatistics;
this.outWriter = builder.outWriter;
this.errWriter = builder.errWriter;
@@ -208,7 +188,7 @@ public class Check implements Runnable {
@Override
public void run() {
try {
- ConsistencyChecker.checkConsistency(path, journal, fullTraversal,
debugInterval, minimumBinaryLength, ioStatistics, outWriter, errWriter);
+ ConsistencyChecker.checkConsistency(path, journal, debugInterval,
checkBinaries, ioStatistics, outWriter, errWriter);
} catch (Exception e) {
e.printStackTrace();
}
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java?rev=1782796&r1=1782795&r2=1782796&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java
Mon Feb 13 15:56:39 2017
@@ -87,9 +87,8 @@ public class CheckValidRepositoryTest {
Check.builder()
.withPath(new File(temporaryFolder.getRoot().getAbsolutePath()))
.withJournal("journal.log")
- .withFullTraversal(true)
.withDebugInterval(Long.MAX_VALUE)
- .withMinimumBinaryLength(Long.MAX_VALUE)
+ .withCheckBinaries(true)
.withIOStatistics(true)
.withOutWriter(outWriter)
.withErrWriter(errWriter)
@@ -114,9 +113,7 @@ public class CheckValidRepositoryTest {
Check.builder()
.withPath(new File(temporaryFolder.getRoot().getAbsolutePath()))
.withJournal("journal.log")
- .withFullTraversal(true)
.withDebugInterval(Long.MAX_VALUE)
- .withMinimumBinaryLength(0L)
.withIOStatistics(true)
.withOutWriter(outWriter)
.withErrWriter(errWriter)
