Author: amitj
Date: Wed Dec 7 09:23:45 2016
New Revision: 1773032
URL: http://svn.apache.org/viewvc?rev=1773032&view=rev
Log:
OAK-5232: Add option to datastorecheck command to add blob ids to tracker
offline
* Added --track option
* Test in BlobIdTrackerStoreTest
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java
jackrabbit/oak/trunk/oak-run/README.md
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java?rev=1773032&r1=1773031&r2=1773032&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java
Wed Dec 7 09:23:45 2016
@@ -29,9 +29,12 @@ import java.util.concurrent.ScheduledExe
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
+import com.google.common.collect.Iterators;
import com.google.common.io.Closer;
+import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.commons.FileIOUtils;
import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore;
@@ -155,6 +158,35 @@ public class BlobIdTrackerTest {
}
}
+ @Test
+ public void externalAddOffline() throws Exception {
+ // Close and open a new object to use the system property
+ closer.close();
+
+ //Add file offline
+ File offline = new File(new File(root, "blobids"),
"blob-offline123456.gen");
+ List<String> offlineLoad = range(0, 1000);
+ FileIOUtils.writeStrings(offlineLoad.iterator(), offline, false);
+
+ this.tracker = new BlobIdTracker(root.getAbsolutePath(), repoId, 100 *
60, dataStore);
+ this.scheduler = newSingleThreadScheduledExecutor();
+ closer.register(tracker);
+ closer.register(new ExecutorCloser(scheduler));
+
+ Set<String> initAdd = add(tracker, range(1001, 1005));
+ ScheduledFuture<?> scheduledFuture =
+ scheduler.schedule(tracker.new SnapshotJob(), 0,
TimeUnit.MILLISECONDS);
+ scheduledFuture.get();
+ initAdd.addAll(offlineLoad);
+
+ assertEquals(initAdd.size(),
+
Iterators.size(FileUtils.lineIterator(tracker.store.getBlobRecordsFile())));
+
+ Set<String> retrieved = retrieve(tracker);
+ assertEquals("Extra elements after add", initAdd, retrieved);
+
assertTrue(read(dataStore.getAllMetadataRecords(BLOBREFERENCES.getType())).isEmpty());
+ }
+
private static Set<String> read(List<DataRecord> recs)
throws IOException, DataStoreException {
Set<String> ids = newHashSet();
Modified: jackrabbit/oak/trunk/oak-run/README.md
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/README.md?rev=1773032&r1=1773031&r2=1773032&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/README.md (original)
+++ jackrabbit/oak/trunk/oak-run/README.md Wed Dec 7 09:23:45 2016
@@ -962,19 +962,21 @@ Use the following command:
$ java -jar oak-run-*.jar datastorecheck [--id] [--ref] [--consistency] \
[--store <path>|<mongo_uri>] \
[--s3ds <s3ds_config>|--fds <fds_config>] \
- [--dump <path>]
+ [--dump <path>] \
+ [--track <DataStore local tracking path>]
The following options are available:
--id - List all the ids in the data store
--ref - List all the blob references in the node store
- --consistency - Lists all the missing blobs by doind a consistency check
+ --consistency - List all the missing blobs by doing a consistency check
Atleast one of the above should be specified
--store - Path to the segment store of mongo uri (Required for
--ref & --consistency option above)
--dump - Path where to dump the files (Optional). Otherwise,
files will be dumped in the user tmp directory.
--s3ds - Path to the S3DataStore configuration file
--fds - Path to the FileDataStore configuration file ('path'
property is mandatory)
+ --track - Path of the local reposity home folder (Optional). This
will place a copy of the downloaded blob ids to be tracked.
Note:
For using S3DataStore the following additional jars have to be downloaded
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java?rev=1773032&r1=1773031&r2=1773032&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
Wed Dec 7 09:23:45 2016
@@ -43,6 +43,8 @@ import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
import joptsimple.OptionSpecBuilder;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
import
org.apache.jackrabbit.oak.commons.FileIOUtils.FileLineDifferenceIterator;
import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
@@ -93,6 +95,10 @@ public class DataStoreCheckCommand imple
.withRequiredArg().ofType(String.class);
OptionSpec segment = parser.accepts("segment", "Use oak-segment
instead of oak-segment-tar");
+ // Optional argument to specify tracking
+ ArgumentAcceptingOptionSpec<String> track =
parser.accepts("track", "Local repository home folder")
+ .withRequiredArg().ofType(String.class);
+
OptionSpec<?> help = parser.acceptsAll(asList("h", "?", "help"),
"show help").forHelp();
@@ -158,8 +164,17 @@ public class DataStoreCheckCommand imple
closer.register(register);
if (options.has(idOp) || options.has(consistencyOp)) {
- retrieveBlobIds(blobStore,
- register.createFile(idOp, dumpPath));
+ File dumpFile = register.createFile(idOp, dumpPath);
+ retrieveBlobIds(blobStore, dumpFile);
+
+ // If track path specified copy the file to the location
+ if (options.has(track)) {
+ String trackPath = options.valueOf(track);
+ File trackingFileParent = new
File(FilenameUtils.concat(trackPath, "blobids"));
+ File trackingFile = new File(trackingFileParent,
+ "blob-" + String.valueOf(System.currentTimeMillis()) +
".gen");
+ FileUtils.copyFile(dumpFile, trackingFile);
+ }
}
if (options.has(refOp) || options.has(consistencyOp)) {
