Author: amitj
Date: Wed Nov 29 06:24:11 2017
New Revision: 1816600
URL: http://svn.apache.org/viewvc?rev=1816600&view=rev
Log:
OAK-6865: Account for active deletion in oak-run datastorecheck
- Added a repoHome option to the command which is mandatory --consistency and
--track
- Now --track does not take any parameter
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java?rev=1816600&r1=1816599&r2=1816600&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
Wed Nov 29 06:24:11 2017
@@ -19,8 +19,10 @@ package org.apache.jackrabbit.oak.run;
import static com.google.common.base.StandardSystemProperty.JAVA_IO_TMPDIR;
import static com.google.common.base.Stopwatch.createStarted;
import static com.google.common.io.Closeables.close;
+import static java.io.File.createTempFile;
import static java.util.Arrays.asList;
import static org.apache.commons.io.FileUtils.forceDelete;
+import static org.apache.commons.io.FileUtils.listFiles;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.sort;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.writeAsLine;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.writeStrings;
@@ -29,6 +31,8 @@ import java.io.BufferedWriter;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Comparator;
import java.util.Iterator;
import java.util.Map;
@@ -39,6 +43,8 @@ import javax.annotation.Nullable;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.io.filefilter.FileFilterUtils;
+import org.apache.jackrabbit.oak.commons.FileIOUtils;
import
org.apache.jackrabbit.oak.commons.FileIOUtils.FileLineDifferenceIterator;
import org.apache.jackrabbit.oak.run.commons.Command;
import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
@@ -79,7 +85,8 @@ public class DataStoreCheckCommand imple
String helpStr =
"datastorecheck [--id] [--ref] [--consistency] [--store
<path>|<mongo_uri>] "
- + "[--s3ds <s3ds_config>|--fds <fds_config>|--azureblobds
<azureblobds_config>] [--dump <path>]";
+ + "[--s3ds <s3ds_config>|--fds <fds_config>|--azureblobds
<azureblobds_config>]"
+ + " [--dump <path>] [--repoHome <repo_home>] [--track]";
Closer closer = Closer.create();
try {
@@ -96,8 +103,12 @@ public class DataStoreCheckCommand imple
.withRequiredArg().ofType(String.class);
// Optional argument to specify tracking
- ArgumentAcceptingOptionSpec<String> track =
parser.accepts("track", "Local repository home folder")
- .withRequiredArg().ofType(String.class);
+ OptionSpecBuilder trackOverride = parser.accepts("track", "Force
override tracked ids");
+
+ // Required argument for --consistency to specify tracking folder
(otherwise can have inconsistencies)
+ ArgumentAcceptingOptionSpec<String> repoHome =
parser.accepts("repoHome", "Local repository home folder")
+ .requiredIf(trackOverride,
consistencyOp).withRequiredArg().ofType(String.class);
+
OptionSpec<?> help = parser.acceptsAll(asList("h", "?", "help"),
"show help").forHelp();
@@ -106,12 +117,16 @@ public class DataStoreCheckCommand imple
idOp.requiredUnless(refOp, consistencyOp);
refOp.requiredUnless(idOp, consistencyOp);
consistencyOp.requiredUnless(idOp, refOp);
+ trackOverride.availableIf(idOp, consistencyOp);
OptionSet options = null;
try {
options = parser.parse(args);
} catch (Exception e) {
System.err.println(e);
+ System.err.println(Arrays.toString(args));
+ System.err.println();
+ System.err.println("Options :");
parser.printHelpOn(System.err);
return;
}
@@ -163,9 +178,9 @@ public class DataStoreCheckCommand imple
File dumpFile = register.createFile(idOp, dumpPath);
retrieveBlobIds(blobStore, dumpFile);
- // If track path specified copy the file to the location
- if (options.has(track)) {
- String trackPath = options.valueOf(track);
+ // If track path and track override specified copy the file to
the location
+ if (options.has(repoHome) && options.has(trackOverride)) {
+ String trackPath = options.valueOf(repoHome);
File trackingFileParent = new
File(FilenameUtils.concat(trackPath, "blobids"));
File trackingFile = new File(trackingFileParent,
"blob-" + String.valueOf(System.currentTimeMillis()) +
".gen");
@@ -180,7 +195,7 @@ public class DataStoreCheckCommand imple
if (options.has(consistencyOp)) {
checkConsistency(register.get(idOp), register.get(refOp),
- register.createFile(consistencyOp, dumpPath));
+ register.createFile(consistencyOp, dumpPath),
options.valueOf(repoHome));
}
} catch (Throwable t) {
t.printStackTrace();
@@ -225,7 +240,7 @@ public class DataStoreCheckCommand imple
}
}
- private static void checkConsistency(File ids, File refs, File missing)
throws IOException {
+ private static void checkConsistency(File ids, File refs, File missing,
String trackRoot) throws IOException {
System.out.println("Starting consistency check");
Stopwatch watch = createStarted();
@@ -238,7 +253,39 @@ public class DataStoreCheckCommand imple
}
return "";
}});
- long candidates = writeStrings(iter, missing, true);
+
+
+ // write the candidates identified to a temp file
+ File candTemp = createTempFile("candTemp", null);
+ int candidates = writeStrings(iter, candTemp, true);
+
+ try {
+ // retrieve the .del file from track directory
+ File trackingFileParent = new File(FilenameUtils.concat(trackRoot,
"blobids"));
+ if (trackingFileParent.exists()) {
+ Collection<File> files =
+ listFiles(trackingFileParent,
FileFilterUtils.suffixFileFilter(".del"), null);
+
+ // If a delete file is present filter the tracked deleted ids
+ if (!files.isEmpty()) {
+ File delFile = files.iterator().next();
+ FileLineDifferenceIterator filteringIter = new
FileLineDifferenceIterator(delFile, candTemp, new Function<String, String>() {
+ @Nullable @Override public String apply(@Nullable
String input) {
+ if (input != null) {
+ return input.split(DELIM)[0];
+ }
+ return "";
+ }
+ });
+ candidates = FileIOUtils.writeStrings(filteringIter,
missing, true);
+ }
+ } else {
+ System.out.println("Skipping active deleted tracked as
parameter [repoHome] : [" + trackRoot + "] incorrect");
+ FileUtils.copyFile(candTemp, missing);
+ }
+ } finally {
+ FileUtils.forceDelete(candTemp);
+ }
System.out.println("Consistency check found " + candidates + " missing
blobs");
if (candidates > 0) {
Modified:
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java?rev=1816600&r1=1816599&r2=1816600&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
Wed Nov 29 06:24:11 2017
@@ -40,6 +40,7 @@ import java.util.Set;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import joptsimple.internal.Strings;
@@ -179,13 +180,15 @@ public class DataStoreCheckTest {
@Test
public void testCorrect() throws Exception {
File dump = temporaryFolder.newFolder();
+ File repoHome = temporaryFolder.newFolder();
setupDataStore.close();
- testAllParams(dump);
+ testAllParams(dump, repoHome);
}
@Test
public void testConsistency() throws Exception {
File dump = temporaryFolder.newFolder();
+ File repoHome = temporaryFolder.newFolder();
Random rand = new Random();
String deletedBlobId = Iterables.get(blobsAdded,
rand.nextInt(blobsAdded.size()));
@@ -194,18 +197,49 @@ public class DataStoreCheckTest {
assertEquals(1, count);
setupDataStore.close();
- testAllParams(dump);
+ testAllParams(dump, repoHome);
assertFileEquals(dump, "[id]", blobsAdded);
assertFileEquals(dump, "[ref]", Sets.union(blobsAdded,
Sets.newHashSet(deletedBlobId)));
assertFileEquals(dump, "[consistency]",
Sets.newHashSet(deletedBlobId));
}
- private void testAllParams(File dump) throws Exception {
+ @Test
+ public void testConsistencyWithDeleteTracker() throws Exception {
+ File dump = temporaryFolder.newFolder();
+ File repoHome = temporaryFolder.newFolder();
+
+ File trackerFolder = new File(repoHome, "blobids");
+ FileUtils.forceMkdir(trackerFolder);
+
+ File delTracker = new File(trackerFolder, "activedeletions.del");
+ Random rand = new Random();
+ String deletedBlobId = Iterables.get(blobsAdded,
rand.nextInt(blobsAdded.size()));
+ blobsAdded.remove(deletedBlobId);
+ long count =
setupDataStore.countDeleteChunks(ImmutableList.of(deletedBlobId), 0);
+
+ String activeDeletedBlobId = Iterables.get(blobsAdded,
rand.nextInt(blobsAdded.size()));
+ blobsAdded.remove(activeDeletedBlobId);
+ count +=
setupDataStore.countDeleteChunks(ImmutableList.of(activeDeletedBlobId), 0);
+ assertEquals(2, count);
+
+ // artificially put the deleted id in the tracked .del file
+
FileIOUtils.writeStrings(Iterators.singletonIterator(activeDeletedBlobId),
delTracker, false);
+
+ setupDataStore.close();
+
+ testAllParams(dump, repoHome);
+
+ assertFileEquals(dump, "[id]", blobsAdded);
+ assertFileEquals(dump, "[ref]", Sets.union(blobsAdded,
Sets.newHashSet(deletedBlobId, activeDeletedBlobId)));
+ assertFileEquals(dump, "[consistency]",
Sets.newHashSet(deletedBlobId));
+ }
+
+ private void testAllParams(File dump, File repoHome) throws Exception {
DataStoreCheckCommand checkCommand = new DataStoreCheckCommand();
List<String> argsList = Lists
.newArrayList("--id", "--ref", "--consistency", "--" + dsOption,
cfgFilePath, "--store", storePath,
- "--dump", dump.getAbsolutePath());
+ "--dump", dump.getAbsolutePath(), "--repoHome",
repoHome.getAbsolutePath());
checkCommand.execute(argsList.toArray(new String[0]));
}
@@ -227,7 +261,7 @@ public class DataStoreCheckTest {
File dump = temporaryFolder.newFolder();
List<String> argsList = Lists
.newArrayList("--id", "--ref", "--consistency", "--store",
storePath,
- "--dump", dump.getAbsolutePath());
+ "--dump", dump.getAbsolutePath(), "--repoHome",
temporaryFolder.newFolder().getAbsolutePath());
testIncorrectParams(argsList, Lists.newArrayList("Operation not
defined for SegmentNodeStore without external datastore"));
}
@@ -238,15 +272,46 @@ public class DataStoreCheckTest {
File dump = temporaryFolder.newFolder();
List<String> argsList = Lists
.newArrayList("--consistency", "--" + dsOption, cfgFilePath,
- "--dump", dump.getAbsolutePath());
+ "--dump", dump.getAbsolutePath(), "--repoHome",
temporaryFolder.newFolder().getAbsolutePath());
testIncorrectParams(argsList, Lists.newArrayList("Missing required
option(s) [store]"));
argsList = Lists
.newArrayList("--ref", "--" + dsOption, cfgFilePath,
- "--dump", dump.getAbsolutePath());
+ "--dump", dump.getAbsolutePath(), "--repoHome",
temporaryFolder.newFolder().getAbsolutePath());
testIncorrectParams(argsList, Lists.newArrayList("Missing required
option(s) [store]"));
}
+ @Test
+ public void testTrackWithRefs() throws Exception {
+ setupDataStore.close();
+ File dump = temporaryFolder.newFolder();
+ List<String> argsList = Lists
+ .newArrayList("--ref", "--store", storePath,
+ "--dump", dump.getAbsolutePath(), "--track", "--repoHome",
temporaryFolder.newFolder().getAbsolutePath());
+ testIncorrectParams(argsList,
+ Lists.newArrayList("Option(s) [track] are unavailable given other
options on the command line"));
+ }
+
+ @Test
+ public void testConsistencyNoRepo() throws Exception {
+ setupDataStore.close();
+ File dump = temporaryFolder.newFolder();
+ List<String> argsList = Lists
+ .newArrayList("--id", "--ref", "--consistency", "--store",
storePath,
+ "--dump", dump.getAbsolutePath());
+ testIncorrectParams(argsList, Lists.newArrayList("Missing required
option(s) [repoHome]"));
+ }
+
+ @Test
+ public void testTrackNoRepo() throws Exception {
+ setupDataStore.close();
+ File dump = temporaryFolder.newFolder();
+ List<String> argsList = Lists
+ .newArrayList("--id", "--ref", "--consistency", "--store",
storePath,
+ "--dump", dump.getAbsolutePath(), "--track");
+ testIncorrectParams(argsList, Lists.newArrayList("Missing required
option(s) [repoHome]"));
+ }
+
public static void testIncorrectParams(List<String> argList,
ArrayList<String> assertMsg) throws Exception {
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
System.setErr(new PrintStream(buffer, true, UTF_8.toString()));
