Author: frm
Date: Tue Nov 27 16:08:54 2018
New Revision: 1847563
URL: http://svn.apache.org/viewvc?rev=1847563&view=rev
Log:
OAK-7918 - Extract ConsistencyCheckerTemplate
The first step in making ConsistencyChecker usable in a programmatic way is to
separate the consistency checking logic from the code that takes care of
argument parsing, progress statements output and result reporting. The
consistency checking logic, unburdened by any other superfluous
responsibilities, has been extracted into a new class,
ConsistencyCheckerTemplate.
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyCheckerTemplate.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java?rev=1847563&r1=1847562&r2=1847563&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java
Tue Nov 27 16:08:54 2018
@@ -20,50 +20,34 @@
package org.apache.jackrabbit.oak.segment.file.tooling;
import static java.text.DateFormat.getDateTimeInstance;
-import static org.apache.jackrabbit.oak.api.Type.BINARIES;
-import static org.apache.jackrabbit.oak.api.Type.BINARY;
import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
-import static org.apache.jackrabbit.oak.commons.PathUtils.concat;
-import static org.apache.jackrabbit.oak.commons.PathUtils.denotesRoot;
-import static org.apache.jackrabbit.oak.commons.PathUtils.getName;
-import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath;
import static
org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
-import static org.apache.jackrabbit.oak.spi.state.NodeStateUtils.getNode;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
import java.io.PrintWriter;
import java.text.MessageFormat;
-import java.util.ArrayList;
import java.util.Date;
-import java.util.HashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
-import java.util.stream.Collectors;
+import com.google.common.base.Strings;
import com.google.common.collect.Sets;
-import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.PropertyState;
-import org.apache.jackrabbit.oak.api.Type;
-import org.apache.jackrabbit.oak.segment.SegmentBlob;
-import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
-import org.apache.jackrabbit.oak.segment.SegmentNotFoundException;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
-import org.apache.jackrabbit.oak.segment.file.JournalEntry;
import org.apache.jackrabbit.oak.segment.file.JournalReader;
import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
-import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter;
import org.apache.jackrabbit.oak.segment.file.tar.LocalJournalFile;
-import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
-import org.apache.jackrabbit.oak.spi.state.NodeState;
+import
org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyCheckerTemplate.ConsistencyCheckResult;
+import
org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyCheckerTemplate.Revision;
+import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter;
/**
* Utility for checking the files of a
@@ -72,10 +56,6 @@ import org.apache.jackrabbit.oak.spi.sta
*/
public class ConsistencyChecker implements Closeable {
- private static final String CHECKPOINT_INDENT = " ";
-
- private static final String NO_INDENT = "";
-
private static class StatisticsIOMonitor extends IOMonitorAdapter {
private final AtomicLong ioOperations = new AtomicLong(0);
@@ -106,226 +86,22 @@ public class ConsistencyChecker implemen
private int nodeCount;
private int propertyCount;
-
- private int checkCount;
-
- /**
- * Run a full traversal consistency check.
- *
- * @param directory directory containing the tar files
- * @param journalFileName name of the journal file containing the
revision history
- * @param debugInterval number of seconds between printing progress
information to
- * the console during the full traversal phase.
- * @param checkBinaries if {@code true} full content of binary
properties will be scanned
- * @param checkHead if {@code true} will check the head
- * @param checkpoints collection of checkpoints to be checked
- * @param filterPaths collection of repository paths to be checked
- * @param ioStatistics if {@code true} prints I/O statistics gathered
while consistency
- * check was performed
- * @param outWriter text output stream writer
- * @param errWriter text error stream writer
- * @throws IOException
- * @throws InvalidFileStoreVersionException
- */
- public static void checkConsistency(
- File directory,
- String journalFileName,
- long debugInterval,
- boolean checkBinaries,
- boolean checkHead,
- Set<String> checkpoints,
- Set<String> filterPaths,
- boolean ioStatistics,
- PrintWriter outWriter,
- PrintWriter errWriter
- ) throws IOException, InvalidFileStoreVersionException {
- try (
- JournalReader journal = new JournalReader(new
LocalJournalFile(directory, journalFileName));
- ConsistencyChecker checker = new ConsistencyChecker(directory,
debugInterval, ioStatistics, outWriter, errWriter)
- ) {
- Set<String> checkpointsSet = Sets.newLinkedHashSet();
- List<PathToCheck> headPaths = new ArrayList<>();
- Map<String, List<PathToCheck>> checkpointPaths = new HashMap<>();
-
- int revisionCount = 0;
-
- if (!checkpoints.isEmpty()) {
- checkpointsSet.addAll(checkpoints);
-
- if (checkpointsSet.remove("all")) {
- checkpointsSet = Sets
-
.newLinkedHashSet(SegmentNodeStoreBuilders.builder(checker.store).build().checkpoints());
- }
- }
-
- for (String path : filterPaths) {
- if (checkHead) {
- headPaths.add(new PathToCheck(path, null));
- checker.checkCount++;
- }
-
- for (String checkpoint : checkpointsSet) {
- List<PathToCheck> pathList =
checkpointPaths.get(checkpoint);
- if (pathList == null) {
- pathList = new ArrayList<>();
- checkpointPaths.put(checkpoint, pathList);
- }
-
- pathList.add(new PathToCheck(path, checkpoint));
- checker.checkCount++;
- }
- }
-
- int initialCount = checker.checkCount;
- JournalEntry lastValidJournalEntry = null;
-
- while (journal.hasNext() && checker.checkCount > 0) {
- JournalEntry journalEntry = journal.next();
- String revision = journalEntry.getRevision();
-
- try {
- revisionCount++;
- checker.store.setRevision(revision);
- boolean overallValid = true;
-
- SegmentNodeStore sns =
SegmentNodeStoreBuilders.builder(checker.store).build();
-
- checker.print("\nChecking revision {0}", revision);
-
- if (checkHead) {
- boolean mustCheck = headPaths.stream().anyMatch(p ->
p.journalEntry == null);
-
- if (mustCheck) {
- checker.print("\nChecking head\n");
- NodeState root = sns.getRoot();
- overallValid = overallValid &&
checker.checkPathsAtRoot(headPaths, root, journalEntry, checkBinaries);
- }
- }
-
- if (!checkpointsSet.isEmpty()) {
- Map<String, Boolean> checkpointsToCheck =
checkpointPaths.entrySet().stream().collect(Collectors.toMap(
- Map.Entry::getKey, e ->
e.getValue().stream().anyMatch(p -> p.journalEntry == null)));
- boolean mustCheck =
checkpointsToCheck.values().stream().anyMatch(v -> v == true);
-
- if (mustCheck) {
- checker.print("\nChecking checkpoints");
-
- for (String checkpoint : checkpointsSet) {
- if (checkpointsToCheck.get(checkpoint)) {
- checker.print("\nChecking checkpoint {0}",
checkpoint);
-
- List<PathToCheck> pathList =
checkpointPaths.get(checkpoint);
- NodeState root = sns.retrieve(checkpoint);
-
- if (root == null) {
- checker.printError("Checkpoint {0} not
found in this revision!", checkpoint);
- overallValid = false;
- } else {
- overallValid = overallValid &&
checker.checkPathsAtRoot(pathList, root,
- journalEntry, checkBinaries);
- }
- }
- }
- }
- }
-
- if (overallValid) {
- lastValidJournalEntry = journalEntry;
- }
- } catch (IllegalArgumentException | SegmentNotFoundException
e) {
- checker.printError("Skipping invalid record id {0}: {1}",
revision, e);
- }
- }
-
- checker.print("\nSearched through {0} revisions and {1}
checkpoints", revisionCount, checkpointsSet.size());
-
- if (initialCount == checker.checkCount) {
- checker.print("No good revision found");
- } else {
- if (checkHead) {
- checker.print("\nHead");
- checker.printResults(headPaths, NO_INDENT);
- }
-
- if (!checkpointsSet.isEmpty()) {
- checker.print("\nCheckpoints");
-
- for (String checkpoint : checkpointsSet) {
- List<PathToCheck> pathList =
checkpointPaths.get(checkpoint);
- checker.print("- {0}", checkpoint);
- checker.printResults(pathList, CHECKPOINT_INDENT);
- }
- }
-
- checker.print("\nOverall");
- checker.printOverallResults(lastValidJournalEntry);
- }
-
- if (ioStatistics) {
- checker.print(
- "[I/O] Segment read: Number of operations: {0}",
- checker.statisticsIOMonitor.ioOperations
- );
- checker.print(
- "[I/O] Segment read: Total size: {0} ({1} bytes)",
-
humanReadableByteCount(checker.statisticsIOMonitor.readBytes.get()),
- checker.statisticsIOMonitor.readBytes
- );
- checker.print(
- "[I/O] Segment read: Total time: {0} ns",
- checker.statisticsIOMonitor.readTime
- );
- }
- }
- }
-
- private void printResults(List<PathToCheck> pathList, String indent) {
- for (PathToCheck ptc : pathList) {
- String revision = ptc.journalEntry != null ?
ptc.journalEntry.getRevision() : null;
- long timestamp = ptc.journalEntry != null ?
ptc.journalEntry.getTimestamp() : -1L;
-
- print("{0}Latest good revision for path {1} is {2} from {3}",
indent, ptc.path,
- toString(revision), toString(timestamp));
- }
- }
-
- private void printOverallResults(JournalEntry journalEntry) {
- String revision = journalEntry != null ? journalEntry.getRevision() :
null;
- long timestamp = journalEntry != null ? journalEntry.getTimestamp() :
-1L;
-
- print("Latest good revision for paths and checkpoints checked is {0}
from {1}", toString(revision), toString(timestamp));
- }
- private static String toString(String revision) {
- if (revision != null) {
- return revision;
- } else {
- return "none";
- }
- }
-
- private static String toString(long timestamp) {
- if (timestamp != -1L) {
- return getDateTimeInstance().format(new Date(timestamp));
- } else {
- return "unknown date";
- }
- }
-
/**
* Create a new consistency checker instance
*
- * @param directory directory containing the tar files
- * @param debugInterval number of seconds between printing progress
information to
- * the console during the full traversal phase.
- * @param ioStatistics if {@code true} prints I/O statistics gathered
while consistency
- * check was performed
- * @param outWriter text output stream writer
- * @param errWriter text error stream writer
+ * @param directory directory containing the tar files
+ * @param debugInterval number of seconds between printing progress
+ * information to the console during the full
traversal
+ * phase.
+ * @param ioStatistics if {@code true} prints I/O statistics gathered
while
+ * consistency check was performed
+ * @param outWriter text output stream writer
+ * @param errWriter text error stream writer
* @throws IOException
*/
public ConsistencyChecker(File directory, long debugInterval, boolean
ioStatistics, PrintWriter outWriter,
- PrintWriter errWriter) throws IOException,
InvalidFileStoreVersionException {
+ PrintWriter errWriter) throws IOException,
InvalidFileStoreVersionException {
FileStoreBuilder builder = fileStoreBuilder(directory);
if (ioStatistics) {
builder.withIOMonitor(statisticsIOMonitor);
@@ -337,235 +113,205 @@ public class ConsistencyChecker implemen
}
/**
- * Checks for consistency a list of paths, relative to the same root.
- *
- * @param paths paths to check
- * @param root root relative to which the paths are retrieved
- * @param journalEntry entry containing the current revision checked
- * @param checkBinaries if {@code true} full content of binary
properties will be scanned
- * @return {@code true}, if the whole list of paths is
consistent
+ * Run a full traversal consistency check.
+ *
+ * @param directory directory containing the tar files
+ * @param journalFileName name of the journal file containing the revision
+ * history
+ * @param checkBinaries if {@code true} full content of binary properties
+ * will be scanned
+ * @param checkHead if {@code true} will check the head
+ * @param checkpointsToCheck collection of checkpoints to be checked
+ * @param filterPaths collection of repository paths to be checked
+ * @param ioStatistics if {@code true} prints I/O statistics gathered
+ * while consistency check was performed
*/
- private boolean checkPathsAtRoot(List<PathToCheck> paths, NodeState root,
JournalEntry journalEntry,
- boolean checkBinaries) {
- boolean result = true;
-
- for (PathToCheck ptc : paths) {
- if (ptc.journalEntry == null) {
- String corruptPath = checkPathAtRoot(ptc, root, checkBinaries);
-
- if (corruptPath == null) {
- print("Path {0} is consistent", ptc.path);
- ptc.journalEntry = journalEntry;
- checkCount--;
- } else {
- result = false;
- ptc.corruptPaths.add(corruptPath);
+ public void checkConsistency(
+ File directory,
+ String journalFileName,
+ boolean checkBinaries,
+ boolean checkHead,
+ Set<String> checkpointsToCheck,
+ Set<String> filterPaths,
+ boolean ioStatistics
+ ) throws IOException {
+ try (JournalReader journal = new JournalReader(new
LocalJournalFile(directory, journalFileName))) {
+ ConsistencyCheckerTemplate template = new
ConsistencyCheckerTemplate() {
+
+ @Override
+ void onCheckRevision(String revision) {
+ print("\nChecking revision {0}", revision);
}
- }
- }
-
- return result;
- }
-
- /**
- * Checks the consistency of the supplied {@code ptc} relative to the
given {@code root}.
- *
- * @param ptc path to check, provided there are no corrupt paths.
- * @param root root relative to which the path is retrieved
- * @param checkBinaries if {@code true} full content of binary properties
will be scanned
- * @return {@code null}, if the content tree rooted at path
(possibly under a checkpoint)
- * is consistent in this revision or the path of the
first inconsistency otherwise.
- */
- private String checkPathAtRoot(PathToCheck ptc, NodeState root, boolean
checkBinaries) {
- String result = null;
- for (String corruptPath : ptc.corruptPaths) {
- try {
- NodeWrapper wrapper =
NodeWrapper.deriveTraversableNodeOnPath(root, corruptPath);
- result = checkNode(wrapper.node, wrapper.path, checkBinaries);
+ @Override
+ void onCheckHead() {
+ print("\nChecking head\n");
+ }
- if (result != null) {
- return result;
+ @Override
+ void onCheckChekpoints() {
+ print("\nChecking checkpoints");
}
- } catch (IllegalArgumentException e) {
- debug("Path {0} not found", corruptPath);
- }
- }
- nodeCount = 0;
- propertyCount = 0;
+ @Override
+ void onCheckCheckpoint(String checkpoint) {
+ print("\nChecking checkpoint {0}", checkpoint);
+ }
- print("Checking {0}", ptc.path);
-
- try {
- NodeWrapper wrapper =
NodeWrapper.deriveTraversableNodeOnPath(root, ptc.path);
- result = checkNodeAndDescendants(wrapper.node, wrapper.path,
checkBinaries);
- print("Checked {0} nodes and {1} properties", nodeCount,
propertyCount);
-
- return result;
- } catch (IllegalArgumentException e) {
- printError("Path {0} not found", ptc.path);
- return ptc.path;
- }
- }
-
- /**
- * Checks the consistency of a node and its properties at the given path.
- *
- * @param node node to be checked
- * @param path path of the node
- * @param checkBinaries if {@code true} full content of binary
properties will be scanned
- * @return {@code null}, if the node is consistent,
- * or the path of the first inconsistency
otherwise.
- */
- private String checkNode(NodeState node, String path, boolean
checkBinaries) {
- try {
- debug("Traversing {0}", path);
- nodeCount++;
- for (PropertyState propertyState : node.getProperties()) {
- Type<?> type = propertyState.getType();
- boolean checked = false;
-
- if (type == BINARY) {
- checked = traverse(propertyState.getValue(BINARY),
checkBinaries);
- } else if (type == BINARIES) {
- for (Blob blob : propertyState.getValue(BINARIES)) {
- checked = checked | traverse(blob, checkBinaries);
- }
- } else {
- propertyState.getValue(type);
+ @Override
+ void onCheckpointNotFoundInRevision(String checkpoint) {
+ printError("Checkpoint {0} not found in this revision!",
checkpoint);
+ }
+
+ @Override
+ void onCheckRevisionError(String revision, Exception e) {
+ printError("Skipping invalid record id {0}: {1}",
revision, e);
+ }
+
+ @Override
+ void onConsistentPath(String path) {
+ print("Path {0} is consistent", path);
+ }
+
+ @Override
+ void onPathNotFound(String path) {
+ printError("Path {0} not found", path);
+ }
+
+ @Override
+ void onCheckTree(String path) {
+ nodeCount = 0;
+ propertyCount = 0;
+ print("Checking {0}", path);
+ }
+
+ @Override
+ void onCheckTreeEnd() {
+ print("Checked {0} nodes and {1} properties", nodeCount,
propertyCount);
+ }
+
+ @Override
+ void onCheckNode(String path) {
+ debug("Traversing {0}", path);
+ nodeCount++;
+ }
+
+ @Override
+ void onCheckProperty() {
propertyCount++;
- checked = true;
}
-
- if (checked) {
- debug("Checked {0}/{1}", path, propertyState);
+
+ @Override
+ void onCheckPropertyEnd(String path, PropertyState property) {
+ debug("Checked {0}/{1}", path, property);
}
- }
-
- return null;
- } catch (RuntimeException | IOException e) {
- printError("Error while traversing {0}: {1}", path, e);
- return path;
- }
- }
-
- /**
- * Recursively checks the consistency of a node and its descendants at the
given path.
- * @param node node to be checked
- * @param path path of the node
- * @param checkBinaries if {@code true} full content of binary properties
will be scanned
- * @return {@code null}, if the node is consistent,
- * or the path of the first inconsistency otherwise.
- */
- private String checkNodeAndDescendants(NodeState node, String path,
boolean checkBinaries) {
- String result = checkNode(node, path, checkBinaries);
- if (result != null) {
- return result;
- }
-
- try {
- for (ChildNodeEntry cne : node.getChildNodeEntries()) {
- String childName = cne.getName();
- NodeState child = cne.getNodeState();
- result = checkNodeAndDescendants(child, concat(path,
childName), checkBinaries);
- if (result != null) {
- return result;
+
+ @Override
+ void onCheckNodeError(String path, Exception e) {
+ printError("Error while traversing {0}: {1}", path, e);
}
- }
- return null;
- } catch (RuntimeException e) {
- printError("Error while traversing {0}: {1}", path,
e.getMessage());
- return path;
- }
- }
-
- static class NodeWrapper {
- final NodeState node;
- final String path;
-
- NodeWrapper(NodeState node, String path) {
- this.node = node;
- this.path = path;
- }
-
- static NodeWrapper deriveTraversableNodeOnPath(NodeState root, String
path) {
- String parentPath = getParentPath(path);
- String name = getName(path);
- NodeState parent = getNode(root, parentPath);
-
- if (!denotesRoot(path)) {
- if (!parent.hasChildNode(name)) {
- throw new IllegalArgumentException("Invalid path: " +
path);
+ @Override
+ void onCheckTreeError(String path, Exception e) {
+ printError("Error while traversing {0}: {1}", path,
e.getMessage());
}
-
- return new NodeWrapper(parent.getChildNode(name), path);
- } else {
- return new NodeWrapper(parent, parentPath);
+
+ };
+
+ Set<String> checkpoints = checkpointsToCheck;
+ if (checkpointsToCheck.contains("all")) {
+ checkpoints =
Sets.newLinkedHashSet(SegmentNodeStoreBuilders.builder(store).build().checkpoints());
}
- }
- }
-
- static class PathToCheck {
- final String path;
- final String checkpoint;
-
- JournalEntry journalEntry;
- Set<String> corruptPaths = new LinkedHashSet<>();
-
- PathToCheck(String path, String checkpoint) {
- this.path = path;
- this.checkpoint = checkpoint;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((checkpoint == null) ? 0 :
checkpoint.hashCode());
- result = prime * result + ((path == null) ? 0 : path.hashCode());
- return result;
- }
- @Override
- public boolean equals(Object object) {
- if (this == object) {
- return true;
- } else if (object instanceof PathToCheck) {
- PathToCheck that = (PathToCheck) object;
- return path.equals(that.path) &&
checkpoint.equals(that.checkpoint);
+ ConsistencyCheckResult result = template.checkConsistency(
+ store,
+ journal,
+ checkHead,
+ checkpoints,
+ filterPaths,
+ checkBinaries
+ );
+
+ print("\nSearched through {0} revisions and {1} checkpoints",
result.getCheckedRevisionsCount(), checkpoints.size());
+
+ if (hasAnyRevision(result)) {
+ if (checkHead) {
+ print("\nHead");
+ for (Entry<String, Revision> e :
result.getHeadRevisions().entrySet()) {
+ printRevision(0, e.getKey(), e.getValue());
+ }
+ }
+ if (checkpoints.size() > 0) {
+ print("\nCheckpoints");
+ for (String checkpoint :
result.getCheckpointRevisions().keySet()) {
+ print("- {0}", checkpoint);
+ for (Entry<String, Revision> e :
result.getCheckpointRevisions().get(checkpoint).entrySet()) {
+ printRevision(2, e.getKey(), e.getValue());
+ }
+
+ }
+ }
+ print("\nOverall");
+ printOverallRevision(result.getOverallRevision());
} else {
- return false;
+ print("No good revision found");
}
- }
- }
- private boolean traverse(Blob blob, boolean checkBinaries) throws
IOException {
- if (checkBinaries && !isExternal(blob)) {
- InputStream s = blob.getNewStream();
- try {
- byte[] buffer = new byte[8192];
- int l = s.read(buffer, 0, buffer.length);
- while (l >= 0) {
- l = s.read(buffer, 0, buffer.length);
- }
- } finally {
- s.close();
+ if (ioStatistics) {
+ print(
+ "[I/O] Segment read: Number of operations: {0}",
+ statisticsIOMonitor.ioOperations.get()
+ );
+ print(
+ "[I/O] Segment read: Total size: {0} ({1} bytes)",
+
humanReadableByteCount(statisticsIOMonitor.readBytes.get()),
+ statisticsIOMonitor.readBytes.get()
+ );
+ print(
+ "[I/O] Segment read: Total time: {0} ns",
+ statisticsIOMonitor.readTime.get()
+ );
}
-
- propertyCount++;
- return true;
}
-
- return false;
}
- private static boolean isExternal(Blob b) {
- if (b instanceof SegmentBlob) {
- return ((SegmentBlob) b).isExternal();
- }
- return false;
+ private void printRevision(int indent, String path, Revision revision) {
+ Optional<Revision> r = Optional.ofNullable(revision);
+ print(
+ "{0}Latest good revision for path {1} is {2} from {3}",
+ Strings.repeat(" ", indent),
+ path,
+ r.map(Revision::getRevision).orElse("none"),
+
r.map(Revision::getTimestamp).map(ConsistencyChecker::timestampToString).orElse("unknown
time")
+ );
+ }
+
+ private void printOverallRevision(Revision revision) {
+ Optional<Revision> r = Optional.ofNullable(revision);
+ print(
+ "Latest good revision for paths and checkpoints checked is {0}
from {1}",
+ r.map(Revision::getRevision).orElse("none"),
+
r.map(Revision::getTimestamp).map(ConsistencyChecker::timestampToString).orElse("unknown
time")
+ );
+ }
+
+ private static boolean hasAnyRevision(ConsistencyCheckResult result) {
+ return hasAnyHeadRevision(result) || hasAnyCheckpointRevision(result);
+ }
+
+ private static boolean hasAnyHeadRevision(ConsistencyCheckResult result) {
+ return result.getHeadRevisions()
+ .values()
+ .stream()
+ .anyMatch(Objects::nonNull);
+ }
+
+ private static boolean hasAnyCheckpointRevision(ConsistencyCheckResult
result) {
+ return result.getCheckpointRevisions()
+ .values()
+ .stream()
+ .flatMap(m -> m.values().stream())
+ .anyMatch(Objects::nonNull);
}
@Override
@@ -573,44 +319,22 @@ public class ConsistencyChecker implemen
store.close();
}
- private void print(String format) {
- outWriter.println(format);
+ private void print(String format, Object... arguments) {
+ outWriter.println(MessageFormat.format(format, arguments));
}
- private void print(String format, Object arg) {
- outWriter.println(MessageFormat.format(format, arg));
- }
-
- private void print(String format, Object arg1, Object arg2) {
- outWriter.println(MessageFormat.format(format, arg1, arg2));
- }
-
- private void print(String format, Object arg1, Object arg2, Object arg3,
Object arg4) {
- outWriter.println(MessageFormat.format(format, arg1, arg2, arg3,
arg4));
- }
-
- private void printError(String format, Object arg) {
- errWriter.println(MessageFormat.format(format, arg));
- }
-
- private void printError(String format, Object arg1, Object arg2) {
- errWriter.println(MessageFormat.format(format, arg1, arg2));
+ private void printError(String format, Object... args) {
+ errWriter.println(MessageFormat.format(format, args));
}
private long ts;
- private void debug(String format, Object arg) {
+ private void debug(String format, Object... arg) {
if (debug()) {
print(format, arg);
}
}
- private void debug(String format, Object arg1, Object arg2) {
- if (debug()) {
- print(format, arg1, arg2);
- }
- }
-
private boolean debug() {
// Avoid calling System.currentTimeMillis(), which is slow on some
systems.
if (debugInterval == Long.MAX_VALUE) {
@@ -627,4 +351,9 @@ public class ConsistencyChecker implemen
return false;
}
}
+
+ private static String timestampToString(long timestamp) {
+ return getDateTimeInstance().format(new Date(timestamp));
+ }
+
}
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyCheckerTemplate.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyCheckerTemplate.java?rev=1847563&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyCheckerTemplate.java
(added)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyCheckerTemplate.java
Tue Nov 27 16:08:54 2018
@@ -0,0 +1,557 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file.tooling;
+
+import static org.apache.jackrabbit.oak.api.Type.BINARIES;
+import static org.apache.jackrabbit.oak.api.Type.BINARY;
+import static org.apache.jackrabbit.oak.commons.PathUtils.concat;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.Set;
+
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.segment.SegmentBlob;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
+import org.apache.jackrabbit.oak.segment.SegmentNotFoundException;
+import org.apache.jackrabbit.oak.segment.file.JournalEntry;
+import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
+
+abstract class ConsistencyCheckerTemplate {
+
+ private static NodeState getDescendantOrNull(NodeState root, String path) {
+ NodeState descendant = NodeStateUtils.getNode(root, path);
+ if (descendant.exists()) {
+ return descendant;
+ }
+ return null;
+ }
+
+ private static class PathToCheck {
+
+ final String path;
+
+ JournalEntry journalEntry;
+
+ Set<String> corruptPaths = new LinkedHashSet<>();
+
+ PathToCheck(String path) {
+ this.path = path;
+ }
+
+ }
+
+ static class FoundRevision {
+
+ private final String path;
+
+ private final String revision;
+
+ private final Long timestamp;
+
+ private FoundRevision(String path, String revision, Long timestamp) {
+ this.path = path;
+ this.revision = revision;
+ this.timestamp = timestamp;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public Optional<String> getRevision() {
+ return Optional.ofNullable(revision);
+ }
+
+ public Optional<Long> getTimestamp() {
+ return Optional.ofNullable(timestamp);
+ }
+ }
+
+ void onCheckRevision(String revision) {
+ // Do nothing.
+ }
+
+ void onCheckHead() {
+ // Do nothing.
+ }
+
+ void onCheckChekpoints() {
+ // Do nothing.
+ }
+
+ void onCheckCheckpoint(String checkpoint) {
+ // Do nothing.
+ }
+
+ void onCheckpointNotFoundInRevision(String checkpoint) {
+ // Do nothing.
+ }
+
+ void onCheckRevisionError(String revision, Exception e) {
+ // Do nothing.
+ }
+
+ void onConsistentPath(String path) {
+ // Do nothing.
+ }
+
+ void onPathNotFound(String path) {
+ // Do nothing.
+ }
+
+ void onCheckTree(String path) {
+ // Do nothing.
+ }
+
+ void onCheckTreeEnd() {
+ // Do nothing.
+ }
+
+ void onCheckNode(String path) {
+ // Do nothing.
+ }
+
+ void onCheckProperty() {
+ // Do nothing.
+ }
+
+ void onCheckPropertyEnd(String path, PropertyState property) {
+ // Do nothing.
+ }
+
+ void onCheckNodeError(String path, Exception e) {
+ // Do nothing.
+ }
+
+ void onCheckTreeError(String path, Exception e) {
+ // Do nothing.
+ }
+
+ public static class Revision {
+
+ private final String revision;
+
+ private final long timestamp;
+
+ private Revision(String revision, long timestamp) {
+ this.revision = revision;
+ this.timestamp = timestamp;
+ }
+
+ public String getRevision() {
+ return revision;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ }
+
+ public static class ConsistencyCheckResult {
+
+ private ConsistencyCheckResult() {
+ // Prevent external instantiation.
+ }
+
+ private final Map<String, Revision> headRevisions = new HashMap<>();
+
+ private final Map<String, Map<String, Revision>> checkpointRevisions =
new HashMap<>();
+
+ private Revision overallRevision;
+
+ private int checkedRevisionsCount;
+
+ public int getCheckedRevisionsCount() {
+ return checkedRevisionsCount;
+ }
+
+ public Revision getOverallRevision() {
+ return overallRevision;
+ }
+
+ public Map<String, Revision> getHeadRevisions() {
+ return headRevisions;
+ }
+
+ public Map<String, Map<String, Revision>> getCheckpointRevisions() {
+ return checkpointRevisions;
+ }
+
+ }
+
+ private boolean isPathInvalid(NodeState root, String path, boolean
binaries) {
+ NodeState node = getDescendantOrNull(root, path);
+
+ if (node == null) {
+ onPathNotFound(path);
+ return true;
+ }
+
+ return checkNode(node, path, binaries) != null;
+ }
+
+ private String findFirstCorruptedPathInSet(NodeState root, Set<String>
corruptedPaths, boolean binaries) {
+ for (String corruptedPath : corruptedPaths) {
+ if (isPathInvalid(root, corruptedPath, binaries)) {
+ return corruptedPath;
+ }
+ }
+ return null;
+ }
+
+ private String findFirstCorruptedPathInTree(NodeState root, String path,
boolean binaries) {
+ NodeState node = getDescendantOrNull(root, path);
+
+ if (node == null) {
+ onPathNotFound(path);
+ return path;
+ }
+
+ return checkNodeAndDescendants(node, path, binaries);
+ }
+
+ private String checkTreeConsistency(NodeState root, String path,
Set<String> corruptedPaths, boolean binaries) {
+ String corruptedPath = findFirstCorruptedPathInSet(root,
corruptedPaths, binaries);
+
+ if (corruptedPath != null) {
+ return corruptedPath;
+ }
+
+ onCheckTree(path);
+ corruptedPath = findFirstCorruptedPathInTree(root, path, binaries);
+ onCheckTreeEnd();
+ return corruptedPath;
+ }
+
+ private boolean checkPathConsistency(NodeState root, PathToCheck ptc,
JournalEntry entry, boolean binaries) {
+ if (ptc.journalEntry != null) {
+ return true;
+ }
+
+ String corruptPath = checkTreeConsistency(root, ptc.path,
ptc.corruptPaths, binaries);
+
+ if (corruptPath != null) {
+ ptc.corruptPaths.add(corruptPath);
+ return false;
+ }
+
+ onConsistentPath(ptc.path);
+ ptc.journalEntry = entry;
+ return true;
+ }
+
+ private boolean checkAllPathsConsistency(NodeState root, List<PathToCheck>
paths, JournalEntry entry, boolean binaries) {
+ boolean result = true;
+
+ for (PathToCheck ptc : paths) {
+ if (!checkPathConsistency(root, ptc, entry, binaries)) {
+ result = false;
+ }
+ }
+
+ return result;
+ }
+
+ private boolean checkHeadConsistency(SegmentNodeStore store,
List<PathToCheck> paths, JournalEntry entry, boolean binaries) {
+ boolean allConsistent = paths.stream().allMatch(p -> p.journalEntry !=
null);
+
+ if (allConsistent) {
+ return true;
+ }
+
+ onCheckHead();
+ return checkAllPathsConsistency(store.getRoot(), paths, entry,
binaries);
+ }
+
+ private boolean checkCheckpointConsistency(SegmentNodeStore store, String
checkpoint, List<PathToCheck> paths, JournalEntry entry, boolean binaries) {
+ NodeState root = store.retrieve(checkpoint);
+
+ if (root == null) {
+ onCheckpointNotFoundInRevision(checkpoint);
+ return false;
+ }
+
+ return checkAllPathsConsistency(root, paths, entry, binaries);
+ }
+
+ private boolean checkCheckpointsConsistency(SegmentNodeStore store,
Map<String, List<PathToCheck>> paths, JournalEntry entry, boolean binaries) {
+ boolean result = true;
+ for (Entry<String, List<PathToCheck>> e : paths.entrySet()) {
+ if (!checkCheckpointConsistency(store, e.getKey(), e.getValue(),
entry, binaries)) {
+ result = false;
+ }
+ }
+ return result;
+ }
+
+ private boolean allPathsConsisten(List<PathToCheck> headPaths, Map<String,
List<PathToCheck>> checkpointPaths) {
+ for (PathToCheck path : headPaths) {
+ if (path.journalEntry == null) {
+ return false;
+ }
+ }
+
+ for (Entry<String, List<PathToCheck>> e : checkpointPaths.entrySet()) {
+ for (PathToCheck path : e.getValue()) {
+ if (path.journalEntry == null) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+ }
+
+ private boolean shouldCheckCheckpointsConsistency(Map<String,
List<PathToCheck>> paths) {
+ return paths
+ .values()
+ .stream()
+ .flatMap(List::stream)
+ .anyMatch(p -> p.journalEntry == null);
+ }
+
+ public final ConsistencyCheckResult checkConsistency(
+ ReadOnlyFileStore store,
+ Iterator<JournalEntry> journal,
+ boolean head,
+ Set<String> checkpoints,
+ Set<String> paths,
+ boolean binaries
+ ) {
+ List<PathToCheck> headPaths = new ArrayList<>();
+ Map<String, List<PathToCheck>> checkpointPaths = new HashMap<>();
+
+ int revisionCount = 0;
+
+ for (String path : paths) {
+ if (head) {
+ headPaths.add(new PathToCheck(path));
+ }
+
+ for (String checkpoint : checkpoints) {
+ checkpointPaths
+ .computeIfAbsent(checkpoint, k -> new ArrayList<>())
+ .add(new PathToCheck(path));
+ }
+ }
+
+ JournalEntry lastValidJournalEntry = null;
+
+ SegmentNodeStore sns = SegmentNodeStoreBuilders.builder(store).build();
+
+ while (journal.hasNext()) {
+ JournalEntry journalEntry = journal.next();
+ String revision = journalEntry.getRevision();
+
+ try {
+ revisionCount++;
+ store.setRevision(revision);
+ onCheckRevision(revision);
+
+ // Check the consistency of both the head and the checkpoints.
+ // If both are consistent, the current journal entry is the
+ // overall valid entry.
+
+ boolean overall = checkHeadConsistency(sns, headPaths,
journalEntry, binaries);
+
+ if (shouldCheckCheckpointsConsistency(checkpointPaths)) {
+ onCheckChekpoints();
+ overall = overall && checkCheckpointsConsistency(sns,
checkpointPaths, journalEntry, binaries);
+ }
+
+ if (overall) {
+ lastValidJournalEntry = journalEntry;
+ }
+
+ // If every PathToCheck is assigned to a JournalEntry, stop
+ // looping through the journal.
+
+ if (allPathsConsisten(headPaths, checkpointPaths)) {
+ break;
+ }
+ } catch (IllegalArgumentException | SegmentNotFoundException e) {
+ onCheckRevisionError(revision, e);
+ }
+ }
+
+ ConsistencyCheckResult result = new ConsistencyCheckResult();
+
+ result.checkedRevisionsCount = revisionCount;
+ result.overallRevision = newRevisionOrNull(lastValidJournalEntry);
+
+ for (PathToCheck path : headPaths) {
+ result.headRevisions.put(path.path,
newRevisionOrNull(path.journalEntry));
+ }
+
+ for (String checkpoint : checkpoints) {
+ for (PathToCheck path : checkpointPaths.get(checkpoint)) {
+ result.checkpointRevisions
+ .computeIfAbsent(checkpoint, s -> new HashMap<>())
+ .put(path.path, newRevisionOrNull(path.journalEntry));
+ }
+ }
+
+ return result;
+ }
+
+ private static Revision newRevisionOrNull(JournalEntry entry) {
+ if (entry == null) {
+ return null;
+ }
+ return new Revision(entry.getRevision(), entry.getTimestamp());
+ }
+
+ private List<FoundRevision> toFoundRevisionsList(List<PathToCheck>
pathsToCheck) {
+ List<FoundRevision> foundRevisions = new
ArrayList<>(pathsToCheck.size());
+ for (PathToCheck pathToCheck : pathsToCheck) {
+ if (pathToCheck.journalEntry == null) {
+ foundRevisions.add(new FoundRevision(pathToCheck.path, null,
null));
+ } else {
+ foundRevisions.add(new FoundRevision(pathToCheck.path,
pathToCheck.journalEntry.getRevision(),
pathToCheck.journalEntry.getTimestamp()));
+ }
+ }
+ return foundRevisions;
+ }
+
+ private Map<String, List<FoundRevision>> toFoundRevisionsMap(Map<String,
List<PathToCheck>> pathsToCheck) {
+ Map<String, List<FoundRevision>> foundRevisions = new HashMap<>();
+ for (Entry<String, List<PathToCheck>> entry : pathsToCheck.entrySet())
{
+ foundRevisions.put(entry.getKey(),
toFoundRevisionsList(entry.getValue()));
+ }
+ return foundRevisions;
+ }
+
+ /**
+ * Checks the consistency of a node and its properties at the given path.
+ *
+ * @param node node to be checked
+ * @param path path of the node
+ * @param checkBinaries if {@code true} full content of binary properties
+ * will be scanned
+ * @return {@code null}, if the node is consistent, or the path of the
first
+ * inconsistency otherwise.
+ */
+ private String checkNode(NodeState node, String path, boolean
checkBinaries) {
+ try {
+ onCheckNode(path);
+ for (PropertyState propertyState : node.getProperties()) {
+ Type<?> type = propertyState.getType();
+ boolean checked = false;
+
+ if (type == BINARY) {
+ checked = traverse(propertyState.getValue(BINARY),
checkBinaries);
+ } else if (type == BINARIES) {
+ for (Blob blob : propertyState.getValue(BINARIES)) {
+ checked = checked | traverse(blob, checkBinaries);
+ }
+ } else {
+ propertyState.getValue(type);
+ onCheckProperty();
+ checked = true;
+ }
+
+ if (checked) {
+ onCheckPropertyEnd(path, propertyState);
+ }
+ }
+
+ return null;
+ } catch (RuntimeException | IOException e) {
+ onCheckNodeError(path, e);
+ return path;
+ }
+ }
+
+ /**
+ * Recursively checks the consistency of a node and its descendants at the
+ * given path.
+ *
+ * @param node node to be checked
+ * @param path path of the node
+ * @param checkBinaries if {@code true} full content of binary properties
+ * will be scanned
+ * @return {@code null}, if the node is consistent, or the path of the
first
+ * inconsistency otherwise.
+ */
+ private String checkNodeAndDescendants(NodeState node, String path,
boolean checkBinaries) {
+ String result = checkNode(node, path, checkBinaries);
+ if (result != null) {
+ return result;
+ }
+
+ try {
+ for (ChildNodeEntry cne : node.getChildNodeEntries()) {
+ String childName = cne.getName();
+ NodeState child = cne.getNodeState();
+ result = checkNodeAndDescendants(child, concat(path,
childName), checkBinaries);
+ if (result != null) {
+ return result;
+ }
+ }
+
+ return null;
+ } catch (RuntimeException e) {
+ onCheckTreeError(path, e);
+ return path;
+ }
+ }
+
+ private boolean traverse(Blob blob, boolean checkBinaries) throws
IOException {
+ if (checkBinaries && !isExternal(blob)) {
+ try (InputStream s = blob.getNewStream()) {
+ byte[] buffer = new byte[8192];
+ int l = s.read(buffer, 0, buffer.length);
+ while (l >= 0) {
+ l = s.read(buffer, 0, buffer.length);
+ }
+ }
+ onCheckProperty();
+ return true;
+ }
+
+ return false;
+ }
+
+ private static boolean isExternal(Blob b) {
+ if (b instanceof SegmentBlob) {
+ return ((SegmentBlob) b).isExternal();
+ }
+ return false;
+ }
+
+}
+
+
Propchange:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyCheckerTemplate.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java?rev=1847563&r1=1847562&r2=1847563&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
Tue Nov 27 16:08:54 2018
@@ -239,18 +239,15 @@ public class Check {
}
public int run() {
- try {
- ConsistencyChecker.checkConsistency(
+ try (ConsistencyChecker checker = new ConsistencyChecker(path,
debugInterval, ioStatistics, outWriter, errWriter)) {
+ checker.checkConsistency(
path,
journal,
- debugInterval,
checkBinaries,
checkHead,
checkpoints,
filterPaths,
- ioStatistics,
- outWriter,
- errWriter
+ ioStatistics
);
return 0;
} catch (Exception e) {
