Author: frm
Date: Wed Nov 22 13:38:22 2017
New Revision: 1816056
URL: http://svn.apache.org/viewvc?rev=1816056&view=rev
Log:
OAK-6969 - Move the offline compaction logic in the tool backend
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/upgrade/UpgradeIT.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java?rev=1816056&r1=1816055&r2=1816056&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
Wed Nov 22 13:38:22 2017
@@ -17,63 +17,19 @@
package org.apache.jackrabbit.oak.run;
-import static com.google.common.collect.Sets.difference;
-import static com.google.common.collect.Sets.newHashSet;
-
import java.io.File;
-import java.util.Date;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
import com.google.common.base.StandardSystemProperty;
-import com.google.common.base.Stopwatch;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
-import org.apache.commons.io.FileUtils;
-import org.apache.jackrabbit.oak.commons.IOUtils;
import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.segment.tool.Compact;
class CompactCommand implements Command {
- private enum FileAccessMode {
-
- ARCH_DEPENDENT(null, "default access mode"),
- MEMORY_MAPPED(true, "memory mapped access mode"),
- REGULAR(false, "regular access mode"),
- REGULAR_ENFORCED(false, "enforced regular access mode");
-
- private final Boolean memoryMapped;
-
- private final String description;
-
- FileAccessMode(Boolean memoryMapped, String description) {
- this.memoryMapped = memoryMapped;
- this.description = description;
- }
-
- Boolean getMemoryMapped() {
- return memoryMapped;
- }
-
- @Override
- public String toString() {
- return description;
- }
-
- }
-
- private static FileAccessMode getFileAccessMode(Boolean arg, String os) {
- if (os != null && os.toLowerCase().contains("windows")) {
- return FileAccessMode.REGULAR_ENFORCED;
- }
- if (arg == null) {
- return FileAccessMode.ARCH_DEPENDENT;
- }
- if (arg) {
- return FileAccessMode.MEMORY_MAPPED;
- }
- return FileAccessMode.REGULAR;
+ private static boolean isTrue(Boolean value) {
+ return value != null && value;
}
@Override
@@ -95,79 +51,27 @@ class CompactCommand implements Command
"which is incompatible with older versions of Oak.")
.withOptionalArg()
.ofType(Boolean.class);
-
OptionSet options = parser.parse(args);
String path = directoryArg.value(options);
+
if (path == null) {
System.err.println("Compact a file store. Usage: compact [path]
<options>");
parser.printHelpOn(System.err);
System.exit(-1);
}
- File directory = new File(path);
-
- boolean success = false;
- Set<String> beforeLs = newHashSet();
- Set<String> afterLs = newHashSet();
- Stopwatch watch = Stopwatch.createStarted();
-
- FileAccessMode fileAccessMode = getFileAccessMode(
- mmapArg.value(options),
- StandardSystemProperty.OS_NAME.value()
- );
-
- System.out.println("Compacting " + directory + " with " +
fileAccessMode);
-
- boolean force = isTrue(forceArg.value(options));
-
- System.out.println(" before ");
- beforeLs.addAll(list(directory));
- long sizeBefore = FileUtils.sizeOfDirectory(directory);
- System.out.println(" size "
- + IOUtils.humanReadableByteCount(sizeBefore) + " (" +
sizeBefore
- + " bytes)");
- System.out.println(" -> compacting");
-
- try {
- SegmentTarUtils.compact(directory,
fileAccessMode.getMemoryMapped(), force);
- success = true;
- } catch (Exception e) {
- e.printStackTrace(System.err);
- } finally {
- watch.stop();
- if (success) {
- System.out.println(" after ");
- afterLs.addAll(list(directory));
- long sizeAfter = FileUtils.sizeOfDirectory(directory);
- System.out.println(" size "
- + IOUtils.humanReadableByteCount(sizeAfter) + " ("
- + sizeAfter + " bytes)");
- System.out.println(" removed files " + difference(beforeLs,
afterLs));
- System.out.println(" added files " + difference(afterLs,
beforeLs));
- System.out.println("Compaction succeeded in " +
watch.toString()
- + " (" + watch.elapsed(TimeUnit.SECONDS) + "s).");
- } else {
- System.out.println("Compaction failed in " + watch.toString()
- + " (" + watch.elapsed(TimeUnit.SECONDS) + "s).");
- System.exit(1);
- }
- }
- }
-
- private static boolean isTrue(Boolean value) {
- return value != null && value;
- }
+ int code = Compact.builder()
+ .withPath(new File(path))
+ .withForce(isTrue(forceArg.value(options)))
+ .withMmap(mmapArg.value(options))
+ .withOs(StandardSystemProperty.OS_NAME.value())
+ .withSegmentCacheSize(Integer.getInteger("cache", 256))
+ .withGCLogInterval(Long.getLong("compaction-progress-log", 150000))
+ .build()
+ .run();
- private static Set<String> list(File directory) {
- Set<String> files = newHashSet();
- for (File f : directory.listFiles()) {
- String d = new Date(f.lastModified()).toString();
- String n = f.getName();
- System.out.println(" " + d + ", " + n);
- files.add(n);
- }
- return files;
+ System.exit(code);
}
}
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java?rev=1816056&r1=1816055&r2=1816056&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
Wed Nov 22 13:38:22 2017
@@ -174,16 +174,6 @@ final class SegmentTarUtils {
.run();
}
- static void compact(@Nonnull File directory, @Nullable Boolean mmap,
boolean force) throws IOException, InvalidFileStoreVersionException {
- Compact.builder()
- .withPath(directory)
- .withMmap(mmap)
- .withForce(force)
- .withSegmentCacheSize(TAR_SEGMENT_CACHE_SIZE)
- .build()
- .run();
- }
-
static void diff(File store, File out, boolean listOnly, String interval,
boolean incremental, String path, boolean ignoreSNFEs) throws IOException {
if (listOnly) {
revisions(store, out);
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java?rev=1816056&r1=1816055&r2=1816056&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java
Wed Nov 22 13:38:22 2017
@@ -19,17 +19,27 @@ package org.apache.jackrabbit.oak.segmen
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.collect.Sets.difference;
+import static com.google.common.collect.Sets.newHashSet;
+import static java.util.Collections.emptySet;
+import static org.apache.commons.io.FileUtils.sizeOfDirectory;
+import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
import static
org.apache.jackrabbit.oak.segment.SegmentCache.DEFAULT_SEGMENT_CACHE_MB;
import static
org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.defaultGCOptions;
import static
org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
import java.io.File;
import java.io.IOException;
+import java.io.PrintStream;
import java.io.RandomAccessFile;
+import java.util.Date;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
-import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Supplier;
import org.apache.jackrabbit.oak.segment.SegmentCache;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
@@ -41,8 +51,6 @@ import org.apache.jackrabbit.oak.segment
*/
public class Compact {
- private final long logAt = Long.getLong("compaction-progress-log", 150000);
-
/**
* Create a builder for the {@link Compact} command.
*
@@ -59,11 +67,14 @@ public class Compact {
private File path;
- @CheckForNull
private Boolean mmap;
+ private String os;
+
private boolean force;
+ private long gcLogInterval = 150000;
+
private int segmentCacheSize = DEFAULT_SEGMENT_CACHE_MB;
private Builder() {
@@ -83,9 +94,11 @@ public class Compact {
/**
* Whether to use memory mapped access or file access.
- * @param mmap {@code true} for memory mapped access, {@code false}
for file access
- * {@code null} to determine the access mode from the
system architecture:
- * memory mapped on 64 bit systems, file access on 32
bit systems.
+ *
+ * @param mmap {@code true} for memory mapped access, {@code false} for
+ * file access {@code null} to determine the access mode
+ * from the system architecture: memory mapped on 64 bit
+ * systems, file access on 32 bit systems.
* @return this builder.
*/
public Builder withMmap(@Nullable Boolean mmap) {
@@ -94,8 +107,22 @@ public class Compact {
}
/**
- * Whether to fail if run on an older version of the store of force
upgrading its format.
- * @param force upgrade iff {@code true}
+ * Which operating system the code is running on.
+ *
+ * @param os The operating system as returned by the "os.name" standard
+ * system property.
+ * @return this builder.
+ */
+ public Builder withOs(String os) {
+ this.os = checkNotNull(os);
+ return this;
+ }
+
+ /**
+ * Whether to fail if run on an older version of the store of force
+ * upgrading its format.
+ *
+ * @param force upgrade iff {@code true}
* @return this builder.
*/
public Builder withForce(boolean force) {
@@ -104,19 +131,35 @@ public class Compact {
}
/**
- * The size of the segment cache in MB. The default of {@link
SegmentCache#DEFAULT_SEGMENT_CACHE_MB}
- * when this method is not invoked.
- * @param segmentCacheSize cache size in MB
+ * The size of the segment cache in MB. The default of {@link
+ * SegmentCache#DEFAULT_SEGMENT_CACHE_MB} when this method is not
+ * invoked.
+ *
+ * @param segmentCacheSize cache size in MB
* @return this builder
- * @throws IllegalArgumentException if {@code segmentCacheSize} is
not a positive integer.
+ * @throws IllegalArgumentException if {@code segmentCacheSize} is not
a
+ * positive integer.
*/
public Builder withSegmentCacheSize(int segmentCacheSize) {
- checkArgument(segmentCacheSize > 0, "segmentCacheSize must be
positive");
+ checkArgument(segmentCacheSize > 0, "segmentCacheSize must be
strictly positive");
this.segmentCacheSize = segmentCacheSize;
return this;
}
/**
+ * The number of nodes after which an update about the compaction
+ * process is logged. Set to a negative number to disable progress
+ * logging. If not specified, it defaults to 150,000 nodes.
+ *
+ * @param gcLogInterval The log interval.
+ * @return this builder.
+ */
+ public Builder withGCLogInterval(long gcLogInterval) {
+ this.gcLogInterval = gcLogInterval;
+ return this;
+ }
+
+ /**
* Create an executable version of the {@link Compact} command.
*
* @return an instance of {@link Runnable}.
@@ -128,53 +171,159 @@ public class Compact {
}
+ private enum FileAccessMode {
+
+ ARCH_DEPENDENT(null, "default access mode"),
+
+ MEMORY_MAPPED(true, "memory mapped access mode"),
+
+ REGULAR(false, "regular access mode"),
+
+ REGULAR_ENFORCED(false, "enforced regular access mode");
+
+ final Boolean memoryMapped;
+
+ final String description;
+
+ FileAccessMode(Boolean memoryMapped, String description) {
+ this.memoryMapped = memoryMapped;
+ this.description = description;
+ }
+
+ }
+
+ private static FileAccessMode newFileAccessMode(Boolean arg, String os) {
+ if (os != null && os.toLowerCase().contains("windows")) {
+ return FileAccessMode.REGULAR_ENFORCED;
+ }
+ if (arg == null) {
+ return FileAccessMode.ARCH_DEPENDENT;
+ }
+ if (arg) {
+ return FileAccessMode.MEMORY_MAPPED;
+ }
+ return FileAccessMode.REGULAR;
+ }
+
+ private static Set<File> listFiles(File directory) {
+ File[] files = directory.listFiles();
+ if (files == null) {
+ return emptySet();
+ }
+ return newHashSet(files);
+ }
+
+ private static void printFiles(PrintStream s, Set<File> files) {
+ for (File f : files) {
+ s.printf(" %s, %s\n", getLastModified(f), f.getName());
+ }
+ }
+
+ private static String getLastModified(File f) {
+ return new Date(f.lastModified()).toString();
+ }
+
+ private static Set<String> fileNames(Set<File> files) {
+ Set<String> names = newHashSet();
+ for (File f : files) {
+ names.add(f.getName());
+ }
+ return names;
+ }
+
+ private static Object printableSize(long size) {
+ return printable(() -> String.format("%s (%d bytes)",
humanReadableByteCount(size), size));
+ }
+
+ private static Object printableStopwatch(Stopwatch s) {
+ return printable(() -> String.format("%s (%ds)", s,
s.elapsed(TimeUnit.SECONDS)));
+ }
+
+ private static Object printable(Supplier<String> s) {
+ return new Object() {
+
+ @Override
+ public String toString() {
+ return s.get();
+ }
+
+ };
+ }
+
private final File path;
- @CheckForNull
- private final Boolean mmap;
+ private final File journal;
+
+ private final FileAccessMode fileAccessMode;
private final int segmentCacheSize;
private final boolean strictVersionCheck;
+ private final long gcLogInterval;
+
private Compact(Builder builder) {
this.path = builder.path;
- this.mmap = builder.mmap;
+ this.journal = new File(builder.path, "journal.log");
+ this.fileAccessMode = newFileAccessMode(builder.mmap, builder.os);
this.segmentCacheSize = builder.segmentCacheSize;
this.strictVersionCheck = !builder.force;
+ this.gcLogInterval = builder.gcLogInterval;
}
- public void run() throws IOException, InvalidFileStoreVersionException {
+ public int run() {
+ System.out.printf("Compacting %s with %s\n", path,
fileAccessMode.description);
+ System.out.printf(" before\n");
+ Set<File> beforeFiles = listFiles(path);
+ printFiles(System.out, beforeFiles);
+ System.out.printf(" size %s\n",
printableSize(sizeOfDirectory(path)));
+ System.out.printf(" -> compacting\n");
+
+ Stopwatch watch = Stopwatch.createStarted();
+
try (FileStore store = newFileStore()) {
store.compactFull();
- System.out.println(" -> cleaning up");
+ System.out.printf(" -> cleaning up\n");
store.cleanup();
- File journal = new File(path, "journal.log");
String head;
try (JournalReader journalReader = new JournalReader(journal)) {
- head = journalReader.next().getRevision() + " root " +
System.currentTimeMillis() + "\n";
+ head = String.format("%s root %s\n",
journalReader.next().getRevision(), System.currentTimeMillis());
}
-
try (RandomAccessFile journalFile = new RandomAccessFile(journal,
"rw")) {
- System.out.println(" -> writing new " + journal.getName() +
": " + head);
+ System.out.printf(" -> writing new %s: %s\n",
journal.getName(), head);
journalFile.setLength(0);
journalFile.writeBytes(head);
journalFile.getChannel().force(false);
}
+ } catch (Exception e) {
+ watch.stop();
+ e.printStackTrace(System.err);
+ System.out.printf("Compaction failed in %s.\n",
printableStopwatch(watch));
+ return 1;
}
+
+ watch.stop();
+ System.out.printf(" after\n");
+ Set<File> afterFiles = listFiles(path);
+ printFiles(System.out, afterFiles);
+ System.out.printf(" size %s\n",
printableSize(sizeOfDirectory(path)));
+ System.out.printf(" removed files %s\n",
fileNames(difference(beforeFiles, afterFiles)));
+ System.out.printf(" added files %s\n",
fileNames(difference(afterFiles, beforeFiles)));
+ System.out.printf("Compaction succeeded in %s.\n",
printableStopwatch(watch));
+ return 0;
}
private FileStore newFileStore() throws IOException,
InvalidFileStoreVersionException {
- FileStoreBuilder fileStoreBuilder =
fileStoreBuilder(path.getAbsoluteFile())
- .withStrictVersionCheck(strictVersionCheck)
- .withSegmentCacheSize(segmentCacheSize)
- .withGCOptions(defaultGCOptions()
- .setOffline()
- .setGCLogInterval(logAt));
-
- return mmap == null
- ? fileStoreBuilder.build()
- : fileStoreBuilder.withMemoryMapping(mmap).build();
+ FileStoreBuilder builder = fileStoreBuilder(path.getAbsoluteFile())
+ .withStrictVersionCheck(strictVersionCheck)
+ .withSegmentCacheSize(segmentCacheSize)
+ .withGCOptions(defaultGCOptions()
+ .setOffline()
+ .setGCLogInterval(gcLogInterval));
+ if (fileAccessMode.memoryMapped != null) {
+ builder.withMemoryMapping(fileAccessMode.memoryMapped);
+ }
+ return builder.build();
}
}
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/upgrade/UpgradeIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/upgrade/UpgradeIT.java?rev=1816056&r1=1816055&r2=1816056&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/upgrade/UpgradeIT.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/upgrade/UpgradeIT.java
Wed Nov 22 13:38:22 2017
@@ -113,16 +113,12 @@ public class UpgradeIT {
checkSegmentVersion(V_12);
checkStoreVersion(1);
- try {
- Compact.builder()
- .withPath(fileStoreHome.getRoot())
- .withMmap(true)
- .withForce(false)
- .build()
- .run();
- } catch (Exception e) {
- // The exception is not relevant to this test.
- }
+ Compact.builder()
+ .withPath(fileStoreHome.getRoot())
+ .withMmap(true)
+ .withForce(false)
+ .build()
+ .run();
// Not upgraded
checkStoreVersion(1);
