Author: mduerig
Date: Tue Jan 26 13:19:59 2016
New Revision: 1726790
URL: http://svn.apache.org/viewvc?rev=1726790&view=rev
Log:
OAK-3855: oak-run compact should check segment version
Bail out if compaction would upgrade the segment format unless --force
Modified:
jackrabbit/oak/trunk/oak-run/README.md
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/segment/FileStoreHelper.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersion.java
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersionTest.java
Modified: jackrabbit/oak/trunk/oak-run/README.md
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/README.md?rev=1726790&r1=1726789&r2=1726790&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/README.md (original)
+++ jackrabbit/oak/trunk/oak-run/README.md Tue Jan 26 13:19:59 2016
@@ -216,7 +216,14 @@ Compact
The 'compact' mode runs the segment compaction operation on the provided TarMK
repository. To start this mode, use:
- $ java -jar oak-run-*.jar compact /path/to/TarMK
+ $ java -jar oak-run-*.jar compact [path] <options>
+
+ [File] -- Path to segment store (required)
+
+ Option Description
+ ------ -----------
+ --force Force compaction and ignore non
+ matching segment version
Checkpoints
-----------
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/segment/FileStoreHelper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/segment/FileStoreHelper.java?rev=1726790&r1=1726789&r2=1726790&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/segment/FileStoreHelper.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/segment/FileStoreHelper.java
Tue Jan 26 13:19:59 2016
@@ -44,6 +44,16 @@ public final class FileStoreHelper {
private FileStoreHelper() {
}
+ /**
+ * Helper method to determine the segment version of the segment
+ * containing the current root node state.
+ * @param fileStore
+ * @return
+ */
+ public static SegmentVersion getSegmentVersion(FileStore fileStore) {
+ return
fileStore.getHead().getRecordId().getSegment().getSegmentVersion();
+ }
+
public static List<String> getTarFiles(FileStore store) {
List<String> files = newArrayList();
for (String p : store.getTarReaderIndex().keySet()) {
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java?rev=1726790&r1=1726789&r2=1726790&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
Tue Jan 26 13:19:59 2016
@@ -20,9 +20,11 @@ import static com.google.common.collect.
import static java.util.Arrays.asList;
import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
import static org.apache.jackrabbit.oak.checkpoint.Checkpoints.CP;
+import static
org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.getSegmentVersion;
import static org.apache.jackrabbit.oak.plugins.segment.RecordType.NODE;
import static
org.apache.jackrabbit.oak.plugins.segment.SegmentGraph.writeGCGraph;
import static
org.apache.jackrabbit.oak.plugins.segment.SegmentGraph.writeSegmentGraph;
+import static
org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.LATEST_VERSION;
import static
org.apache.jackrabbit.oak.plugins.segment.file.FileStore.newFileStore;
import static
org.apache.jackrabbit.oak.plugins.segment.file.tooling.ConsistencyChecker.checkConsistency;
import static org.slf4j.LoggerFactory.getLogger;
@@ -109,6 +111,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState;
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
import org.apache.jackrabbit.oak.plugins.segment.SegmentTracker;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentVersion;
import org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy;
import
org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy.CleanupType;
import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
@@ -496,90 +499,113 @@ public final class Main {
}
private static void compact(String[] args) throws IOException {
- if (args.length != 1) {
- System.err.println("usage: compact <path>");
- System.exit(1);
- } else if (!isValidFileStore(args[0])) {
- System.err.println("Invalid FileStore directory " + args[0]);
+ OptionParser parser = new OptionParser();
+ OptionSpec<File> directoryArg = parser.nonOptions(
+ "Path to segment store (required)").ofType(File.class);
+ OptionSpec<Void> forceFlag = parser.accepts(
+ "force", "Force compaction and ignore non matching segment
version");
+ OptionSet options = parser.parse(args);
+
+ File directory = directoryArg.value(options);
+ if (directory == null) {
+ System.err.println("Compact a file store. Usage: compact [path]
<options>");
+ parser.printHelpOn(System.err);
+ System.exit(-1);
+ }
+ if (!isValidFileStore(directory.getPath())) {
+ System.err.println("Invalid FileStore directory " + directory);
System.exit(1);
- } else {
- boolean persistCM = Boolean
- .getBoolean("tar.PersistCompactionMap");
- Stopwatch watch = Stopwatch.createStarted();
- File directory = new File(args[0]);
- System.out.println("Compacting " + directory);
- System.out.println(" before " +
Arrays.toString(directory.list()));
- long sizeBefore = FileUtils.sizeOfDirectory(directory);
- System.out.println(" size "
- + IOUtils.humanReadableByteCount(sizeBefore) + " (" +
sizeBefore
- + " bytes)");
+ }
- System.out.println(" -> compacting");
- FileStore store = openFileStore(directory);
- try {
- CompactionStrategy compactionStrategy = new CompactionStrategy(
- false, CompactionStrategy.CLONE_BINARIES_DEFAULT,
- CleanupType.CLEAN_ALL, 0,
- CompactionStrategy.MEMORY_THRESHOLD_DEFAULT) {
- @Override
- public boolean compacted(Callable<Boolean> setHead)
- throws Exception {
- // oak-run is doing compaction single-threaded
- // hence no guarding needed - go straight ahead
- // and call setHead
- return setHead.call();
- }
- };
- compactionStrategy.setOfflineCompaction(true);
- compactionStrategy.setPersistCompactionMap(persistCM);
- store.setCompactionStrategy(compactionStrategy);
- store.compact();
- } finally {
- store.close();
+ FileStore store = openFileStore(directory);
+ SegmentVersion segmentVersion = getSegmentVersion(store);
+ if (segmentVersion != LATEST_VERSION) {
+ if (options.has(forceFlag)) {
+ System.out.println("Segment version mismatch. " +
+ "Found " + segmentVersion + ", expected " + LATEST_VERSION
+ ". " +
+ "Upgrading the file store to segment version " +
LATEST_VERSION);
+ } else {
+ System.err.println("Segment version mismatch. " +
+ "Found " + segmentVersion + ", expected " + LATEST_VERSION
+ ". " +
+ "Specify --force to upgrade the file store to segment
version " + LATEST_VERSION);
+ System.exit(-1);
}
+ }
- System.out.println(" -> cleaning up");
- store = openFileStore(directory);
- try {
- for (File file : store.cleanup()) {
- if (!file.exists() || file.delete()) {
- System.out.println(" -> removed old file " +
file.getName());
- } else {
- System.out.println(" -> failed to remove old file "
+ file.getName());
- }
- }
+ boolean persistCM = Boolean.getBoolean("tar.PersistCompactionMap");
+ Stopwatch watch = Stopwatch.createStarted();
+ System.out.println("Compacting " + directory);
+ System.out.println(" before " + Arrays.toString(directory.list()));
+ long sizeBefore = FileUtils.sizeOfDirectory(directory);
+ System.out.println(" size "
+ + IOUtils.humanReadableByteCount(sizeBefore) + " (" +
sizeBefore
+ + " bytes)");
- String head;
- File journal = new File(directory, "journal.log");
- JournalReader journalReader = new JournalReader(journal);
- try {
- head = journalReader.iterator().next() + " root\n";
- } finally {
- journalReader.close();
+ System.out.println(" -> compacting");
+
+ try {
+ CompactionStrategy compactionStrategy = new CompactionStrategy(
+ false, CompactionStrategy.CLONE_BINARIES_DEFAULT,
+ CleanupType.CLEAN_ALL, 0,
+ CompactionStrategy.MEMORY_THRESHOLD_DEFAULT) {
+ @Override
+ public boolean compacted(Callable<Boolean> setHead)
+ throws Exception {
+ // oak-run is doing compaction single-threaded
+ // hence no guarding needed - go straight ahead
+ // and call setHead
+ return setHead.call();
}
+ };
+ compactionStrategy.setOfflineCompaction(true);
+ compactionStrategy.setPersistCompactionMap(persistCM);
+ store.setCompactionStrategy(compactionStrategy);
+ store.compact();
+ } finally {
+ store.close();
+ }
- RandomAccessFile journalFile = new RandomAccessFile(journal,
"rw");
- try {
- System.out.println(" -> writing new " +
journal.getName() + ": " + head);
- journalFile.setLength(0);
- journalFile.writeBytes(head);
- journalFile.getChannel().force(false);
- } finally {
- journalFile.close();
+ System.out.println(" -> cleaning up");
+ store = openFileStore(directory);
+ try {
+ for (File file : store.cleanup()) {
+ if (!file.exists() || file.delete()) {
+ System.out.println(" -> removed old file " +
file.getName());
+ } else {
+ System.out.println(" -> failed to remove old file " +
file.getName());
}
+ }
+
+ String head;
+ File journal = new File(directory, "journal.log");
+ JournalReader journalReader = new JournalReader(journal);
+ try {
+ head = journalReader.iterator().next() + " root\n";
} finally {
- store.close();
+ journalReader.close();
+ }
+
+ RandomAccessFile journalFile = new RandomAccessFile(journal, "rw");
+ try {
+ System.out.println(" -> writing new " + journal.getName() +
": " + head);
+ journalFile.setLength(0);
+ journalFile.writeBytes(head);
+ journalFile.getChannel().force(false);
+ } finally {
+ journalFile.close();
}
- watch.stop();
- System.out.println(" after "
- + Arrays.toString(directory.list()));
- long sizeAfter = FileUtils.sizeOfDirectory(directory);
- System.out.println(" size "
- + IOUtils.humanReadableByteCount(sizeAfter) + " (" +
sizeAfter
- + " bytes)");
- System.out.println(" duration " + watch.toString() + " ("
- + watch.elapsed(TimeUnit.SECONDS) + "s).");
+ } finally {
+ store.close();
}
+ watch.stop();
+ System.out.println(" after "
+ + Arrays.toString(directory.list()));
+ long sizeAfter = FileUtils.sizeOfDirectory(directory);
+ System.out.println(" size "
+ + IOUtils.humanReadableByteCount(sizeAfter) + " (" + sizeAfter
+ + " bytes)");
+ System.out.println(" duration " + watch.toString() + " ("
+ + watch.elapsed(TimeUnit.SECONDS) + "s).");
}
private static FileStore openFileStore(File directory) throws IOException {
Modified:
jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersion.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersion.java?rev=1726790&r1=1726789&r2=1726790&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersion.java
(original)
+++
jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersion.java
Tue Jan 26 13:19:59 2016
@@ -18,6 +18,13 @@
*/
package org.apache.jackrabbit.oak.plugins.segment;
+import static java.util.Collections.max;
+import static java.util.EnumSet.allOf;
+
+import java.util.Comparator;
+
+import com.google.common.primitives.UnsignedBytes;
+
/**
* Version of the segment storage format.
* <ul>
@@ -35,6 +42,17 @@ public enum SegmentVersion {
V_11((byte) 11);
+ /**
+ * Latest segment version
+ */
+ public static SegmentVersion LATEST_VERSION =
max(allOf(SegmentVersion.class),
+ new Comparator<SegmentVersion>() {
+ @Override
+ public int compare(SegmentVersion v1, SegmentVersion v2) {
+ return UnsignedBytes.compare(v1.version, v2.version);
+ }
+ });
+
private final byte version;
SegmentVersion(byte version) {
Modified:
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersionTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersionTest.java?rev=1726790&r1=1726789&r2=1726790&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersionTest.java
(original)
+++
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersionTest.java
Tue Jan 26 13:19:59 2016
@@ -23,6 +23,8 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.api.Type.LONGS;
import static org.apache.jackrabbit.oak.api.Type.STRING;
import static org.apache.jackrabbit.oak.api.Type.STRINGS;
+import static
org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.LATEST_VERSION;
+import static org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.V_11;
import static
org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy.CleanupType.CLEAN_NONE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -78,6 +80,11 @@ public class SegmentVersionTest {
}
@Test
+ public void latestVersion() {
+ assertEquals(V_11, LATEST_VERSION);
+ }
+
+ @Test
public void compareOldRevision() throws Exception {
FileStore fileStoreV10 = new FileStore(directory, 1) {
@SuppressWarnings("deprecation")
@@ -176,7 +183,7 @@ public class SegmentVersionTest {
FileStore fileStoreV11 = new FileStore(directory, 1);
try {
NodeState content11 = addTestContent(fileStoreV11, "content11");
- assertVersion(content11, SegmentVersion.V_11);
+ assertVersion(content11, V_11);
verifyContent(fileStoreV11, "content10");
verifyContent(fileStoreV11, "content11");
} finally {
@@ -210,7 +217,7 @@ public class SegmentVersionTest {
});
checkAllVersions(fileStoreV11.getHead(), SegmentVersion.V_10);
fileStoreV11.compact();
- checkAllVersions(fileStoreV11.getHead(), SegmentVersion.V_11);
+ checkAllVersions(fileStoreV11.getHead(), V_11);
} finally {
fileStoreV11.close();
}
