Author: frm Date: Wed Nov 22 13:38:22 2017 New Revision: 1816056 URL: http://svn.apache.org/viewvc?rev=1816056&view=rev Log: OAK-6969 - Move the offline compaction logic in the tool backend
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/upgrade/UpgradeIT.java Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java?rev=1816056&r1=1816055&r2=1816056&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java Wed Nov 22 13:38:22 2017 @@ -17,63 +17,19 @@ package org.apache.jackrabbit.oak.run; -import static com.google.common.collect.Sets.difference; -import static com.google.common.collect.Sets.newHashSet; - import java.io.File; -import java.util.Date; -import java.util.Set; -import java.util.concurrent.TimeUnit; import com.google.common.base.StandardSystemProperty; -import com.google.common.base.Stopwatch; import joptsimple.OptionParser; import joptsimple.OptionSet; import joptsimple.OptionSpec; -import org.apache.commons.io.FileUtils; -import org.apache.jackrabbit.oak.commons.IOUtils; import org.apache.jackrabbit.oak.run.commons.Command; +import org.apache.jackrabbit.oak.segment.tool.Compact; class CompactCommand implements Command { - private enum FileAccessMode { - - ARCH_DEPENDENT(null, "default access mode"), - MEMORY_MAPPED(true, "memory mapped access mode"), - REGULAR(false, "regular access mode"), - REGULAR_ENFORCED(false, "enforced regular access mode"); - - private final Boolean memoryMapped; - - private final String description; - - FileAccessMode(Boolean memoryMapped, String description) { - this.memoryMapped = memoryMapped; - this.description = description; - } - - Boolean getMemoryMapped() { - return memoryMapped; - } - - @Override - public String toString() { - return description; - } - - } - - private static FileAccessMode getFileAccessMode(Boolean arg, String os) { - if (os != null && os.toLowerCase().contains("windows")) { - return FileAccessMode.REGULAR_ENFORCED; - } - if (arg == null) { - return FileAccessMode.ARCH_DEPENDENT; - } - if (arg) { - return FileAccessMode.MEMORY_MAPPED; - } - return FileAccessMode.REGULAR; + private static boolean isTrue(Boolean value) { + return value != null && value; } @Override @@ -95,79 +51,27 @@ class CompactCommand implements Command "which is incompatible with older versions of Oak.") .withOptionalArg() .ofType(Boolean.class); - OptionSet options = parser.parse(args); String path = directoryArg.value(options); + if (path == null) { System.err.println("Compact a file store. Usage: compact [path] <options>"); parser.printHelpOn(System.err); System.exit(-1); } - File directory = new File(path); - - boolean success = false; - Set<String> beforeLs = newHashSet(); - Set<String> afterLs = newHashSet(); - Stopwatch watch = Stopwatch.createStarted(); - - FileAccessMode fileAccessMode = getFileAccessMode( - mmapArg.value(options), - StandardSystemProperty.OS_NAME.value() - ); - - System.out.println("Compacting " + directory + " with " + fileAccessMode); - - boolean force = isTrue(forceArg.value(options)); - - System.out.println(" before "); - beforeLs.addAll(list(directory)); - long sizeBefore = FileUtils.sizeOfDirectory(directory); - System.out.println(" size " - + IOUtils.humanReadableByteCount(sizeBefore) + " (" + sizeBefore - + " bytes)"); - System.out.println(" -> compacting"); - - try { - SegmentTarUtils.compact(directory, fileAccessMode.getMemoryMapped(), force); - success = true; - } catch (Exception e) { - e.printStackTrace(System.err); - } finally { - watch.stop(); - if (success) { - System.out.println(" after "); - afterLs.addAll(list(directory)); - long sizeAfter = FileUtils.sizeOfDirectory(directory); - System.out.println(" size " - + IOUtils.humanReadableByteCount(sizeAfter) + " (" - + sizeAfter + " bytes)"); - System.out.println(" removed files " + difference(beforeLs, afterLs)); - System.out.println(" added files " + difference(afterLs, beforeLs)); - System.out.println("Compaction succeeded in " + watch.toString() - + " (" + watch.elapsed(TimeUnit.SECONDS) + "s)."); - } else { - System.out.println("Compaction failed in " + watch.toString() - + " (" + watch.elapsed(TimeUnit.SECONDS) + "s)."); - System.exit(1); - } - } - } - - private static boolean isTrue(Boolean value) { - return value != null && value; - } + int code = Compact.builder() + .withPath(new File(path)) + .withForce(isTrue(forceArg.value(options))) + .withMmap(mmapArg.value(options)) + .withOs(StandardSystemProperty.OS_NAME.value()) + .withSegmentCacheSize(Integer.getInteger("cache", 256)) + .withGCLogInterval(Long.getLong("compaction-progress-log", 150000)) + .build() + .run(); - private static Set<String> list(File directory) { - Set<String> files = newHashSet(); - for (File f : directory.listFiles()) { - String d = new Date(f.lastModified()).toString(); - String n = f.getName(); - System.out.println(" " + d + ", " + n); - files.add(n); - } - return files; + System.exit(code); } } Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java?rev=1816056&r1=1816055&r2=1816056&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java Wed Nov 22 13:38:22 2017 @@ -174,16 +174,6 @@ final class SegmentTarUtils { .run(); } - static void compact(@Nonnull File directory, @Nullable Boolean mmap, boolean force) throws IOException, InvalidFileStoreVersionException { - Compact.builder() - .withPath(directory) - .withMmap(mmap) - .withForce(force) - .withSegmentCacheSize(TAR_SEGMENT_CACHE_SIZE) - .build() - .run(); - } - static void diff(File store, File out, boolean listOnly, String interval, boolean incremental, String path, boolean ignoreSNFEs) throws IOException { if (listOnly) { revisions(store, out); Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java?rev=1816056&r1=1816055&r2=1816056&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java Wed Nov 22 13:38:22 2017 @@ -19,17 +19,27 @@ package org.apache.jackrabbit.oak.segmen import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.Sets.difference; +import static com.google.common.collect.Sets.newHashSet; +import static java.util.Collections.emptySet; +import static org.apache.commons.io.FileUtils.sizeOfDirectory; +import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount; import static org.apache.jackrabbit.oak.segment.SegmentCache.DEFAULT_SEGMENT_CACHE_MB; import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.defaultGCOptions; import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder; import java.io.File; import java.io.IOException; +import java.io.PrintStream; import java.io.RandomAccessFile; +import java.util.Date; +import java.util.Set; +import java.util.concurrent.TimeUnit; -import javax.annotation.CheckForNull; import javax.annotation.Nullable; +import com.google.common.base.Stopwatch; +import com.google.common.base.Supplier; import org.apache.jackrabbit.oak.segment.SegmentCache; import org.apache.jackrabbit.oak.segment.file.FileStore; import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; @@ -41,8 +51,6 @@ import org.apache.jackrabbit.oak.segment */ public class Compact { - private final long logAt = Long.getLong("compaction-progress-log", 150000); - /** * Create a builder for the {@link Compact} command. * @@ -59,11 +67,14 @@ public class Compact { private File path; - @CheckForNull private Boolean mmap; + private String os; + private boolean force; + private long gcLogInterval = 150000; + private int segmentCacheSize = DEFAULT_SEGMENT_CACHE_MB; private Builder() { @@ -83,9 +94,11 @@ public class Compact { /** * Whether to use memory mapped access or file access. - * @param mmap {@code true} for memory mapped access, {@code false} for file access - * {@code null} to determine the access mode from the system architecture: - * memory mapped on 64 bit systems, file access on 32 bit systems. + * + * @param mmap {@code true} for memory mapped access, {@code false} for + * file access {@code null} to determine the access mode + * from the system architecture: memory mapped on 64 bit + * systems, file access on 32 bit systems. * @return this builder. */ public Builder withMmap(@Nullable Boolean mmap) { @@ -94,8 +107,22 @@ public class Compact { } /** - * Whether to fail if run on an older version of the store of force upgrading its format. - * @param force upgrade iff {@code true} + * Which operating system the code is running on. + * + * @param os The operating system as returned by the "os.name" standard + * system property. + * @return this builder. + */ + public Builder withOs(String os) { + this.os = checkNotNull(os); + return this; + } + + /** + * Whether to fail if run on an older version of the store of force + * upgrading its format. + * + * @param force upgrade iff {@code true} * @return this builder. */ public Builder withForce(boolean force) { @@ -104,19 +131,35 @@ public class Compact { } /** - * The size of the segment cache in MB. The default of {@link SegmentCache#DEFAULT_SEGMENT_CACHE_MB} - * when this method is not invoked. - * @param segmentCacheSize cache size in MB + * The size of the segment cache in MB. The default of {@link + * SegmentCache#DEFAULT_SEGMENT_CACHE_MB} when this method is not + * invoked. + * + * @param segmentCacheSize cache size in MB * @return this builder - * @throws IllegalArgumentException if {@code segmentCacheSize} is not a positive integer. + * @throws IllegalArgumentException if {@code segmentCacheSize} is not a + * positive integer. */ public Builder withSegmentCacheSize(int segmentCacheSize) { - checkArgument(segmentCacheSize > 0, "segmentCacheSize must be positive"); + checkArgument(segmentCacheSize > 0, "segmentCacheSize must be strictly positive"); this.segmentCacheSize = segmentCacheSize; return this; } /** + * The number of nodes after which an update about the compaction + * process is logged. Set to a negative number to disable progress + * logging. If not specified, it defaults to 150,000 nodes. + * + * @param gcLogInterval The log interval. + * @return this builder. + */ + public Builder withGCLogInterval(long gcLogInterval) { + this.gcLogInterval = gcLogInterval; + return this; + } + + /** * Create an executable version of the {@link Compact} command. * * @return an instance of {@link Runnable}. @@ -128,53 +171,159 @@ public class Compact { } + private enum FileAccessMode { + + ARCH_DEPENDENT(null, "default access mode"), + + MEMORY_MAPPED(true, "memory mapped access mode"), + + REGULAR(false, "regular access mode"), + + REGULAR_ENFORCED(false, "enforced regular access mode"); + + final Boolean memoryMapped; + + final String description; + + FileAccessMode(Boolean memoryMapped, String description) { + this.memoryMapped = memoryMapped; + this.description = description; + } + + } + + private static FileAccessMode newFileAccessMode(Boolean arg, String os) { + if (os != null && os.toLowerCase().contains("windows")) { + return FileAccessMode.REGULAR_ENFORCED; + } + if (arg == null) { + return FileAccessMode.ARCH_DEPENDENT; + } + if (arg) { + return FileAccessMode.MEMORY_MAPPED; + } + return FileAccessMode.REGULAR; + } + + private static Set<File> listFiles(File directory) { + File[] files = directory.listFiles(); + if (files == null) { + return emptySet(); + } + return newHashSet(files); + } + + private static void printFiles(PrintStream s, Set<File> files) { + for (File f : files) { + s.printf(" %s, %s\n", getLastModified(f), f.getName()); + } + } + + private static String getLastModified(File f) { + return new Date(f.lastModified()).toString(); + } + + private static Set<String> fileNames(Set<File> files) { + Set<String> names = newHashSet(); + for (File f : files) { + names.add(f.getName()); + } + return names; + } + + private static Object printableSize(long size) { + return printable(() -> String.format("%s (%d bytes)", humanReadableByteCount(size), size)); + } + + private static Object printableStopwatch(Stopwatch s) { + return printable(() -> String.format("%s (%ds)", s, s.elapsed(TimeUnit.SECONDS))); + } + + private static Object printable(Supplier<String> s) { + return new Object() { + + @Override + public String toString() { + return s.get(); + } + + }; + } + private final File path; - @CheckForNull - private final Boolean mmap; + private final File journal; + + private final FileAccessMode fileAccessMode; private final int segmentCacheSize; private final boolean strictVersionCheck; + private final long gcLogInterval; + private Compact(Builder builder) { this.path = builder.path; - this.mmap = builder.mmap; + this.journal = new File(builder.path, "journal.log"); + this.fileAccessMode = newFileAccessMode(builder.mmap, builder.os); this.segmentCacheSize = builder.segmentCacheSize; this.strictVersionCheck = !builder.force; + this.gcLogInterval = builder.gcLogInterval; } - public void run() throws IOException, InvalidFileStoreVersionException { + public int run() { + System.out.printf("Compacting %s with %s\n", path, fileAccessMode.description); + System.out.printf(" before\n"); + Set<File> beforeFiles = listFiles(path); + printFiles(System.out, beforeFiles); + System.out.printf(" size %s\n", printableSize(sizeOfDirectory(path))); + System.out.printf(" -> compacting\n"); + + Stopwatch watch = Stopwatch.createStarted(); + try (FileStore store = newFileStore()) { store.compactFull(); - System.out.println(" -> cleaning up"); + System.out.printf(" -> cleaning up\n"); store.cleanup(); - File journal = new File(path, "journal.log"); String head; try (JournalReader journalReader = new JournalReader(journal)) { - head = journalReader.next().getRevision() + " root " + System.currentTimeMillis() + "\n"; + head = String.format("%s root %s\n", journalReader.next().getRevision(), System.currentTimeMillis()); } - try (RandomAccessFile journalFile = new RandomAccessFile(journal, "rw")) { - System.out.println(" -> writing new " + journal.getName() + ": " + head); + System.out.printf(" -> writing new %s: %s\n", journal.getName(), head); journalFile.setLength(0); journalFile.writeBytes(head); journalFile.getChannel().force(false); } + } catch (Exception e) { + watch.stop(); + e.printStackTrace(System.err); + System.out.printf("Compaction failed in %s.\n", printableStopwatch(watch)); + return 1; } + + watch.stop(); + System.out.printf(" after\n"); + Set<File> afterFiles = listFiles(path); + printFiles(System.out, afterFiles); + System.out.printf(" size %s\n", printableSize(sizeOfDirectory(path))); + System.out.printf(" removed files %s\n", fileNames(difference(beforeFiles, afterFiles))); + System.out.printf(" added files %s\n", fileNames(difference(afterFiles, beforeFiles))); + System.out.printf("Compaction succeeded in %s.\n", printableStopwatch(watch)); + return 0; } private FileStore newFileStore() throws IOException, InvalidFileStoreVersionException { - FileStoreBuilder fileStoreBuilder = fileStoreBuilder(path.getAbsoluteFile()) - .withStrictVersionCheck(strictVersionCheck) - .withSegmentCacheSize(segmentCacheSize) - .withGCOptions(defaultGCOptions() - .setOffline() - .setGCLogInterval(logAt)); - - return mmap == null - ? fileStoreBuilder.build() - : fileStoreBuilder.withMemoryMapping(mmap).build(); + FileStoreBuilder builder = fileStoreBuilder(path.getAbsoluteFile()) + .withStrictVersionCheck(strictVersionCheck) + .withSegmentCacheSize(segmentCacheSize) + .withGCOptions(defaultGCOptions() + .setOffline() + .setGCLogInterval(gcLogInterval)); + if (fileAccessMode.memoryMapped != null) { + builder.withMemoryMapping(fileAccessMode.memoryMapped); + } + return builder.build(); } } Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/upgrade/UpgradeIT.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/upgrade/UpgradeIT.java?rev=1816056&r1=1816055&r2=1816056&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/upgrade/UpgradeIT.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/upgrade/UpgradeIT.java Wed Nov 22 13:38:22 2017 @@ -113,16 +113,12 @@ public class UpgradeIT { checkSegmentVersion(V_12); checkStoreVersion(1); - try { - Compact.builder() - .withPath(fileStoreHome.getRoot()) - .withMmap(true) - .withForce(false) - .build() - .run(); - } catch (Exception e) { - // The exception is not relevant to this test. - } + Compact.builder() + .withPath(fileStoreHome.getRoot()) + .withMmap(true) + .withForce(false) + .build() + .run(); // Not upgraded checkStoreVersion(1);