Author: alexparvulescu Date: Mon Jun 27 14:59:14 2016 New Revision: 1750356
URL: http://svn.apache.org/viewvc?rev=1750356&view=rev Log: OAK-4505 Offline compaction clearer output values - merged r1750076, r1750077 Modified: jackrabbit/oak/branches/1.4/ (props changed) jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java Propchange: jackrabbit/oak/branches/1.4/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon Jun 27 14:59:14 2016 @@ -1,3 +1,3 @@ /jackrabbit/oak/branches/1.0:1665962 -/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738775,1738795,1738833,1738950,1738957,1738963,1739894,1740116,1740625-1740626,1740971,1741032,1741339,1741343,1742520,1742888,1742916,1743097,1743172,1743343,1744265,1744959,1745038,1745197,1745368,1746086,1746117,1746342,1746345,1746696,1746981,1747341-1747342,1747492,1747512,1748505,1748553,1748870,1749275,1749350,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750287 +/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738775,1738795,1738833,1738950,1738957,1738963,1739894,1740116,1740625-1740626,1740971,1741032,1741339,1741343,1742520,1742888,1742916,1743097,1743172,1743343,1744265,1744959,1745038,1745197,1745368,1746086,1746117,1746342,1746345,1746696,1746981,1747341-1747342,1747492,1747512,1748505,1748553,1748870,1749275,1749350,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750076-1750077,1750287 /jackrabbit/trunk:1345480 Modified: jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java?rev=1750356&r1=1750355&r2=1750356&view=diff ============================================================================== --- jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java (original) +++ jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java Mon Jun 27 14:59:14 2016 @@ -19,13 +19,17 @@ package org.apache.jackrabbit.oak.run; import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.openFileStore; +import static com.google.common.collect.Sets.difference; +import static com.google.common.collect.Sets.newHashSet; + import java.io.File; import java.io.RandomAccessFile; import java.util.Arrays; +import java.util.Date; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; -import com.google.common.base.Stopwatch; import joptsimple.OptionParser; import joptsimple.OptionSet; import joptsimple.OptionSpec; @@ -35,6 +39,8 @@ import org.apache.jackrabbit.oak.plugins import org.apache.jackrabbit.oak.plugins.segment.file.FileStore; import org.apache.jackrabbit.oak.plugins.segment.file.JournalReader; +import com.google.common.base.Stopwatch; + class CompactCommand implements Command { @Override @@ -52,82 +58,114 @@ class CompactCommand implements Command parser.printHelpOn(System.err); System.exit(-1); } - Stopwatch watch = Stopwatch.createStarted(); - FileStore store = openFileStore(path, options.has(forceFlag)); + File directory = new File(path); - try { - boolean persistCM = Boolean.getBoolean("tar.PersistCompactionMap"); - System.out.println("Compacting " + directory); - System.out.println(" before " + Arrays.toString(directory.list())); - long sizeBefore = FileUtils.sizeOfDirectory(directory); - System.out.println(" size " - + IOUtils.humanReadableByteCount(sizeBefore) + " (" + sizeBefore - + " bytes)"); - - System.out.println(" -> compacting"); - - CompactionStrategy compactionStrategy = new CompactionStrategy( - false, CompactionStrategy.CLONE_BINARIES_DEFAULT, - CompactionStrategy.CleanupType.CLEAN_ALL, 0, - CompactionStrategy.MEMORY_THRESHOLD_DEFAULT) { - @Override - public boolean compacted(Callable<Boolean> setHead) - throws Exception { - // oak-run is doing compaction single-threaded - // hence no guarding needed - go straight ahead - // and call setHead - return setHead.call(); - } - }; - compactionStrategy.setOfflineCompaction(true); - compactionStrategy.setPersistCompactionMap(persistCM); - store.setCompactionStrategy(compactionStrategy); - store.compact(); - } finally { - store.close(); - } - System.out.println(" -> cleaning up"); - store = openFileStore(path, false); - try { - for (File file : store.cleanup()) { - if (!file.exists() || file.delete()) { - System.out.println(" -> removed old file " + file.getName()); - } else { - System.out.println(" -> failed to remove old file " + file.getName()); - } - } + boolean success = false; + Set<String> beforeLs = newHashSet(); + Set<String> afterLs = newHashSet(); + Stopwatch watch = Stopwatch.createStarted(); - String head; - File journal = new File(directory, "journal.log"); - JournalReader journalReader = new JournalReader(journal); + System.out.println("Compacting " + directory); + System.out.println(" before "); + beforeLs.addAll(list(directory)); + long sizeBefore = FileUtils.sizeOfDirectory(directory); + System.out.println(" size " + + IOUtils.humanReadableByteCount(sizeBefore) + " (" + sizeBefore + + " bytes)"); + System.out.println(" -> compacting"); + + try { + FileStore store = openFileStore(path, options.has(forceFlag)); + boolean persistCM = Boolean.getBoolean("tar.PersistCompactionMap"); try { - head = journalReader.iterator().next() + " root\n"; + CompactionStrategy compactionStrategy = new CompactionStrategy( + false, CompactionStrategy.CLONE_BINARIES_DEFAULT, + CompactionStrategy.CleanupType.CLEAN_ALL, 0, + CompactionStrategy.MEMORY_THRESHOLD_DEFAULT) { + @Override + public boolean compacted(Callable<Boolean> setHead) + throws Exception { + // oak-run is doing compaction single-threaded + // hence no guarding needed - go straight ahead + // and call setHead + return setHead.call(); + } + }; + compactionStrategy.setOfflineCompaction(true); + compactionStrategy.setPersistCompactionMap(persistCM); + store.setCompactionStrategy(compactionStrategy); + store.compact(); } finally { - journalReader.close(); + store.close(); } - RandomAccessFile journalFile = new RandomAccessFile(journal, "rw"); + System.out.println(" -> cleaning up"); + store = openFileStore(path, false); try { - System.out.println(" -> writing new " + journal.getName() + ": " + head); - journalFile.setLength(0); - journalFile.writeBytes(head); - journalFile.getChannel().force(false); + for (File file : store.cleanup()) { + if (!file.exists() || file.delete()) { + System.out.println(" -> removed old file " + file.getName()); + } else { + System.out.println(" -> failed to remove old file " + file.getName()); + } + } + + String head; + File journal = new File(directory, "journal.log"); + JournalReader journalReader = new JournalReader(journal); + try { + head = journalReader.iterator().next() + " root\n"; + } finally { + journalReader.close(); + } + + RandomAccessFile journalFile = new RandomAccessFile(journal, "rw"); + try { + System.out.println(" -> writing new " + journal.getName() + ": " + head); + journalFile.setLength(0); + journalFile.writeBytes(head); + journalFile.getChannel().force(false); + } finally { + journalFile.close(); + } } finally { - journalFile.close(); + store.close(); } + success = true; + } catch (Throwable e) { + System.out.println("Compaction failure stack trace:"); + e.printStackTrace(System.out); } finally { - store.close(); + watch.stop(); + if (success) { + System.out.println(" after "); + afterLs.addAll(list(directory)); + long sizeAfter = FileUtils.sizeOfDirectory(directory); + System.out.println(" size " + + IOUtils.humanReadableByteCount(sizeAfter) + " (" + + sizeAfter + " bytes)"); + System.out.println(" removed files " + difference(beforeLs, afterLs)); + System.out.println(" added files " + difference(afterLs, beforeLs)); + System.out.println("Compaction succeeded in " + watch.toString() + + " (" + watch.elapsed(TimeUnit.SECONDS) + "s)."); + } else { + System.out.println("Compaction failed in " + watch.toString() + + " (" + watch.elapsed(TimeUnit.SECONDS) + "s)."); + System.exit(1); + } } - watch.stop(); - System.out.println(" after " - + Arrays.toString(directory.list())); - long sizeAfter = FileUtils.sizeOfDirectory(directory); - System.out.println(" size " - + IOUtils.humanReadableByteCount(sizeAfter) + " (" + sizeAfter - + " bytes)"); - System.out.println(" duration " + watch.toString() + " (" - + watch.elapsed(TimeUnit.SECONDS) + "s)."); + } + + private static Set<String> list(File directory) { + Set<String> files = newHashSet(); + for (File f : directory.listFiles()) { + String d = new Date(f.lastModified()).toString(); + String n = f.getName(); + System.out.println(" " + d + ", " + n); + files.add(n); + } + return files; } }
