Author: alexparvulescu Date: Fri Jan 29 14:38:52 2016 New Revision: 1727595
URL: http://svn.apache.org/viewvc?rev=1727595&view=rev Log: OAK-2480 Incremental (FileStore)Backup copies the entire source instead of just the delta Modified: jackrabbit/oak/trunk/oak-run/README.md jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackup.java jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupRestore.java jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreRestore.java jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupTest.java Modified: jackrabbit/oak/trunk/oak-run/README.md URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/README.md?rev=1727595&r1=1727594&r2=1727595&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/README.md (original) +++ jackrabbit/oak/trunk/oak-run/README.md Fri Jan 29 14:38:52 2016 @@ -38,18 +38,25 @@ See the subsections below for more detai Backup ------ -The 'backup' mode creates a backup from an existing oak repository. To start this mode, use: +The 'backup' mode creates a backup from an existing oak repository. The most efficient +way to backup the TarMK repository is to use a file system copy of the repository folder. +The current backup implementation acts like a compaction to an enternal folder, on top of +copying the state, it will also try to compress it, so it will significantly slower than +what one might expect from a simple copy backup. Incremental backups (backup over an existing +backup will still need to perform a full content diff) and will attempt to compact the diff. +All optimisation flags used for offline compaction very much apply for this case as well. +The FileStore backup doesn't need access to the DataStore, but if one is usually configured with +the repository, it will need the following system property set to true in order to be able to +perform the diffing `-Doak.backup.UseFakeBlobStore=true`. To start this mode, use: - $ java -jar oak-run-*.jar backup \ - { /path/to/oak/repository | mongodb://host:port/database } /path/to/backup + $ java -jar oak-run-*.jar backup /path/to/oak/repository /path/to/backup Restore ------- The 'restore' mode imports a backup of an existing oak repository. To start this mode, use: - $ java -jar oak-run-*.jar restore \ - { /path/to/oak/repository | mongodb://host:port/database } /path/to/backup + $ java -jar oak-run-*.jar restore /path/to/oak/repository /path/to/backup Debug ----- Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java?rev=1727595&r1=1727594&r2=1727595&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java Fri Jan 29 14:38:52 2016 @@ -22,6 +22,7 @@ import static org.apache.commons.io.File import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.isValidFileStoreOrFail; import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.openFileStore; import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.openReadOnlyFileStore; +import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.newBasicReadOnlyBlobStore; import static org.apache.jackrabbit.oak.plugins.segment.RecordType.NODE; import static org.apache.jackrabbit.oak.plugins.segment.SegmentGraph.writeGCGraph; import static org.apache.jackrabbit.oak.plugins.segment.SegmentGraph.writeSegmentGraph; @@ -288,10 +289,18 @@ public final class Main { } private static void backup(String[] args) throws IOException { + boolean fakeBlobStore = FileStoreBackup.USE_FAKE_BLOBSTORE; Closer closer = Closer.create(); - String h = "backup { /path/to/oak/repository | mongodb://host:port/database } <path/to/backup>"; try { - NodeStore store = bootstrapNodeStore(args, closer, h); + FileStore fs; + if (fakeBlobStore) { + fs = openReadOnlyFileStore(new File(args[0]), + newBasicReadOnlyBlobStore()); + } else { + fs = openReadOnlyFileStore(new File(args[0])); + } + closer.register(asCloseable(fs)); + NodeStore store = SegmentNodeStore.newSegmentNodeStore(fs).create(); FileStoreBackup.backup(store, new File(args[1])); } catch (Throwable e) { throw closer.rethrow(e); @@ -301,16 +310,7 @@ public final class Main { } private static void restore(String[] args) throws IOException { - Closer closer = Closer.create(); - String h = "restore { /path/to/oak/repository | mongodb://host:port/database } <path/to/backup>"; - try { - NodeStore store = bootstrapNodeStore(args, closer, h); - FileStoreRestore.restore(new File(args[1]), store); - } catch (Throwable e) { - throw closer.rethrow(e); - } finally { - closer.close(); - } + FileStoreRestore.restore(new File(args[1]), new File(args[0])); } //TODO react to state changes of FailoverClient (triggered via JMX), once the state model of FailoverClient is complete. Modified: jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackup.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackup.java?rev=1727595&r1=1727594&r2=1727595&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackup.java (original) +++ jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackup.java Fri Jan 29 14:38:52 2016 @@ -18,77 +18,53 @@ */ package org.apache.jackrabbit.oak.plugins.backup; -import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; +import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.jackrabbit.oak.plugins.segment.file.FileStore.newFileStore; import java.io.File; import java.io.IOException; -import java.util.concurrent.TimeUnit; -import com.google.common.collect.ImmutableMap; import org.apache.jackrabbit.oak.plugins.segment.Compactor; -import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeBuilder; import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState; +import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore; import org.apache.jackrabbit.oak.plugins.segment.file.FileStore; +import org.apache.jackrabbit.oak.plugins.segment.file.tooling.BasicReadOnlyBlobStore; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Stopwatch; + public class FileStoreBackup { private static final Logger log = LoggerFactory .getLogger(FileStoreBackup.class); - private static final long DEFAULT_LIFETIME = TimeUnit.HOURS.toMillis(1); - - static int MAX_FILE_SIZE = 256; + public static boolean USE_FAKE_BLOBSTORE = Boolean.getBoolean("oak.backup.UseFakeBlobStore"); public static void backup(NodeStore store, File destination) throws IOException { - long s = System.currentTimeMillis(); - - // 1. create a new checkpoint with the current state - String checkpoint = store.checkpoint(DEFAULT_LIFETIME, ImmutableMap.of( - "creator", FileStoreBackup.class.getSimpleName(), - "thread", Thread.currentThread().getName())); - NodeState current = store.retrieve(checkpoint); - if (current == null) { - // unable to retrieve the checkpoint; use root state instead - current = store.getRoot(); + checkArgument(store instanceof SegmentNodeStore); + Stopwatch watch = Stopwatch.createStarted(); + NodeState current = ((SegmentNodeStore) store).getSuperRoot(); + FileStore.Builder builder = newFileStore(destination) + .withDefaultMemoryMapping(); + if (USE_FAKE_BLOBSTORE) { + builder.withBlobStore(new BasicReadOnlyBlobStore()); } - - // 2. init filestore - FileStore backup = new FileStore(destination, MAX_FILE_SIZE, false); + FileStore backup = builder.create(); try { SegmentNodeState state = backup.getHead(); - NodeState before = null; - String beforeCheckpoint = state.getString("checkpoint"); - if (beforeCheckpoint == null) { - // 3.1 no stored checkpoint, so do the initial full backup - before = EMPTY_NODE; - } else { - // 3.2 try to retrieve the previously backed up checkpoint - before = store.retrieve(beforeCheckpoint); - if (before == null) { - // the previous checkpoint is no longer available, - // so use the backed up state as the basis of the - // incremental backup diff - before = state.getChildNode("root"); - } - } - Compactor compactor = new Compactor(backup.getTracker()); - SegmentNodeState after = compactor.compact(before, current); - - // 4. commit the backup - SegmentNodeBuilder builder = state.builder(); - builder.setProperty("checkpoint", checkpoint); - builder.setChildNode("root", after); - backup.setHead(state, builder.getNodeState()); + compactor.setDeepCheckLargeBinaries(true); + compactor.setContentEqualityCheck(true); + SegmentNodeState after = compactor.compact(state, current, state); + backup.setHead(state, after); } finally { backup.close(); } - - log.debug("Backup finished in {} ms.", System.currentTimeMillis() - s); + watch.stop(); + log.info("Backup finished in {}.", watch); } } Modified: jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupRestore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupRestore.java?rev=1727595&r1=1727594&r2=1727595&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupRestore.java (original) +++ jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupRestore.java Fri Jan 29 14:38:52 2016 @@ -21,11 +21,10 @@ package org.apache.jackrabbit.oak.plugin import static com.google.common.base.Preconditions.checkNotNull; import static java.lang.System.nanoTime; -import static org.apache.jackrabbit.oak.management.ManagementOperation.Status.formatTime; import static org.apache.jackrabbit.oak.management.ManagementOperation.done; import static org.apache.jackrabbit.oak.management.ManagementOperation.newManagementOperation; +import static org.apache.jackrabbit.oak.management.ManagementOperation.Status.formatTime; import static org.apache.jackrabbit.oak.plugins.backup.FileStoreBackup.backup; -import static org.apache.jackrabbit.oak.plugins.backup.FileStoreRestore.restore; import java.io.File; import java.util.concurrent.Callable; @@ -37,14 +36,11 @@ import javax.management.openmbean.Compos import org.apache.jackrabbit.oak.api.jmx.FileStoreBackupRestoreMBean; import org.apache.jackrabbit.oak.management.ManagementOperation; import org.apache.jackrabbit.oak.spi.state.NodeStore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Default implementation of {@link FileStoreBackupRestoreMBean} based on a file. */ public class FileStoreBackupRestore implements FileStoreBackupRestoreMBean { - private static final Logger log = LoggerFactory.getLogger(FileStoreBackupRestore.class); public static final String BACKUP_OP_NAME = "Backup"; public static final String RESTORE_OP_NAME = "Restore"; @@ -97,9 +93,7 @@ public class FileStoreBackupRestore impl restoreOp = newManagementOperation("Restore", new Callable<String>() { @Override public String call() throws Exception { - long t0 = nanoTime(); - restore(file, store); - return "Restore completed in " + formatTime(nanoTime() - t0); + return "Restore not available as an online operation."; } }); executor.execute(restoreOp); Modified: jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreRestore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreRestore.java?rev=1727595&r1=1727594&r2=1727595&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreRestore.java (original) +++ jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreRestore.java Fri Jan 29 14:38:52 2016 @@ -18,22 +18,20 @@ */ package org.apache.jackrabbit.oak.plugins.backup; +import static org.apache.jackrabbit.oak.plugins.segment.file.FileStore.newFileStore; + import java.io.File; import java.io.IOException; -import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.plugins.segment.Compactor; -import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeBuilder; import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState; -import org.apache.jackrabbit.oak.plugins.segment.SegmentStore; import org.apache.jackrabbit.oak.plugins.segment.file.FileStore; -import org.apache.jackrabbit.oak.spi.commit.CommitInfo; -import org.apache.jackrabbit.oak.spi.commit.EmptyHook; -import org.apache.jackrabbit.oak.spi.state.NodeState; -import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.plugins.segment.file.FileStore.ReadOnlyStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Stopwatch; + public class FileStoreRestore { private static final Logger log = LoggerFactory @@ -43,44 +41,30 @@ public class FileStoreRestore { private static final String JOURNAL_FILE_NAME = "journal.log"; - public static void restore(File source, NodeStore store) - throws IOException, CommitFailedException { - // 1. verify that this is an actual filestore + public static void restore(File source, File destination) + throws IOException { if (!validFileStore(source)) { throw new IOException("Folder " + source + " is not a valid FileStore directory"); } - // 2. init filestore - FileStore restore = new FileStore(source, MAX_FILE_SIZE, false); + FileStore restore = new ReadOnlyStore(source); + Stopwatch watch = Stopwatch.createStarted(); + + FileStore store = newFileStore(destination).create(); + SegmentNodeState current = store.getHead(); try { - SegmentNodeState state = restore.getHead(); - restore(state.getChildNode("root"), store, restore); + Compactor compactor = new Compactor(store.getTracker()); + compactor.setDeepCheckLargeBinaries(true); + SegmentNodeState after = compactor.compact(current, + restore.getHead(), current); + store.setHead(current, after); } finally { restore.close(); + store.close(); } - } - - private static void restore(NodeState source, NodeStore store, - SegmentStore restore) throws CommitFailedException, IOException { - long s = System.currentTimeMillis(); - NodeState current = store.getRoot(); - RestoreCompactor compactor = new RestoreCompactor(restore); - SegmentNodeBuilder builder = compactor.process(current, source, current); - store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); - log.debug("Restore finished in {} ms.", System.currentTimeMillis() - s); - } - - private static class RestoreCompactor extends Compactor { - - public RestoreCompactor(SegmentStore store) { - super(store.getTracker()); - } - - @Override - protected SegmentNodeBuilder process(NodeState before, NodeState after, NodeState onto) throws IOException { - return super.process(before, after, onto); - } + watch.stop(); + log.info("Restore finished in {}.", watch); } private static boolean validFileStore(File source) { Modified: jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java?rev=1727595&r1=1727594&r2=1727595&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java (original) +++ jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java Fri Jan 29 14:38:52 2016 @@ -64,6 +64,8 @@ public class Compactor { r.getSegmentId().getLeastSignificantBits(), r.getOffset() }; } + private final SegmentTracker tracker; + private final SegmentWriter writer; private final PartialCompactionMap map; @@ -90,6 +92,21 @@ public class Compactor { private final boolean cloneBinaries; /** + * In the case of large inlined binaries, compaction will verify if all + * referenced segments exist in order to determine if a full clone is + * necessary, or just a shallow copy of the RecordId list is enough + * (Used in Backup scenario) + */ + private boolean deepCheckLargeBinaries; + + /** + * Flag to use content equality verification before actually compacting the + * state, on the childNodeChanged diff branch + * (Used in Backup scenario) + */ + private boolean contentEqualityCheck; + + /** * Allows the cancellation of the compaction process. If this {@code * Supplier} returns {@code true}, this compactor will cancel compaction and * return a partial {@code SegmentNodeState} containing the changes @@ -102,6 +119,7 @@ public class Compactor { } public Compactor(SegmentTracker tracker, Supplier<Boolean> cancel) { + this.tracker = tracker; this.writer = tracker.getWriter(); this.map = new InMemoryCompactionMap(tracker); this.cloneBinaries = false; @@ -113,6 +131,7 @@ public class Compactor { } public Compactor(SegmentTracker tracker, CompactionStrategy compactionStrategy, Supplier<Boolean> cancel) { + this.tracker = tracker; String wid = "c-" + (tracker.getCompactionMap().getGeneration() + 1); this.writer = tracker.createSegmentWriter(wid); if (compactionStrategy.getPersistCompactionMap()) { @@ -135,24 +154,6 @@ public class Compactor { /** * Compact the differences between a {@code before} and a {@code after} - * on top of the {@code before} state. - * <p> - * Equivalent to {@code compact(before, after, before)} - * - * @param before the before state - * @param after the after state - * @return the compacted state - */ - public SegmentNodeState compact(NodeState before, NodeState after) throws IOException { - progress.start(); - SegmentNodeState compacted = process(before, after, before).getNodeState(); - writer.flush(); - progress.stop(); - return compacted; - } - - /** - * Compact the differences between a {@code before} and a {@code after} * on top of an {@code onto} state. * @param before the before state * @param after the after state @@ -277,6 +278,10 @@ public class Compactor { } } + if (contentEqualityCheck && before.equals(after)) { + return true; + } + progress.onNode(); try { NodeBuilder child = builder.getChildNode(name); @@ -334,7 +339,7 @@ public class Compactor { // if the blob is inlined or external, just clone it if (sb.isExternal() || sb.length() < Segment.MEDIUM_LIMIT) { - SegmentBlob clone = sb.clone(writer, cloneBinaries); + SegmentBlob clone = sb.clone(writer, false); map.put(id, clone.getRecordId()); return clone; } @@ -351,8 +356,24 @@ public class Compactor { } } - // if not, clone the blob and keep track of the result - sb = sb.clone(writer, cloneBinaries); + boolean clone = cloneBinaries; + if (deepCheckLargeBinaries) { + clone = clone + || !tracker.getStore().containsSegment( + id.getSegmentId()); + if (!clone) { + for (SegmentId bid : SegmentBlob.getBulkSegmentIds(sb)) { + clone = clone + || !tracker.getStore().containsSegment(bid); + if (clone) { + break; + } + } + } + } + + // if not, clone the large blob and keep track of the result + sb = sb.clone(writer, clone); map.put(id, sb.getRecordId()); if (ids == null) { ids = newArrayList(); @@ -458,4 +479,12 @@ public class Compactor { } } + public void setDeepCheckLargeBinaries(boolean deepCheckLargeBinaries) { + this.deepCheckLargeBinaries = deepCheckLargeBinaries; + } + + public void setContentEqualityCheck(boolean contentEqualityCheck) { + this.contentEqualityCheck = contentEqualityCheck; + } + } Modified: jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java?rev=1727595&r1=1727594&r2=1727595&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java (original) +++ jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java Fri Jan 29 14:38:52 2016 @@ -210,6 +210,18 @@ public class SegmentNodeStore implements return head.get().getChildNode(ROOT); } + @Nonnull + public NodeState getSuperRoot() { + if (commitSemaphore.tryAcquire()) { + try { + refreshHead(); + } finally { + commitSemaphore.release(); + } + } + return head.get(); + } + @Override public NodeState merge( @Nonnull NodeBuilder builder, @Nonnull CommitHook commitHook, Modified: jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1727595&r1=1727594&r2=1727595&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original) +++ jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Fri Jan 29 14:38:52 2016 @@ -304,6 +304,16 @@ public class FileStore implements Segmen } /** + * Set memory mapping to the default value based on OS properties + * @return this instance + */ + @Nonnull + public Builder withDefaultMemoryMapping() { + this.memoryMapping = MEMORY_MAPPING_DEFAULT; + return this; + } + + /** * {@link GCMonitor} for monitoring this files store's gc process. * @param gcMonitor * @return this instance Modified: jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupTest.java?rev=1727595&r1=1727594&r2=1727595&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupTest.java (original) +++ jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupTest.java Fri Jan 29 14:38:52 2016 @@ -18,17 +18,20 @@ */ package org.apache.jackrabbit.oak.plugins.backup; +import static org.apache.commons.io.FileUtils.deleteQuietly; +import static org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore.newSegmentNodeStore; +import static org.apache.jackrabbit.oak.plugins.segment.file.FileStore.newFileStore; +import static org.junit.Assert.assertEquals; + import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; +import java.util.Random; import org.apache.jackrabbit.oak.Oak; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent; -import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore; import org.apache.jackrabbit.oak.plugins.segment.file.FileStore; import org.apache.jackrabbit.oak.spi.commit.CommitInfo; import org.apache.jackrabbit.oak.spi.commit.EmptyHook; @@ -39,10 +42,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import static org.apache.commons.io.FileUtils.deleteQuietly; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - public class FileStoreBackupTest { private File src; @@ -64,54 +63,61 @@ public class FileStoreBackupTest { @Test public void testBackup() throws Exception { - FileStore source = new FileStore(src, 8, false); + FileStore source = newFileStore(src).withMaxFileSize(8).create(); - NodeStore store = new SegmentNodeStore(source); + NodeStore store = newSegmentNodeStore(source).create(); init(store); // initial content FileStoreBackup.backup(store, destination); - compare(store, destination); + compare(source, destination); addTestContent(store); FileStoreBackup.backup(store, destination); - compare(store, destination); + compare(source, destination); source.close(); } @Test public void testRestore() throws Exception { - FileStore source = new FileStore(src, 8, false); + FileStore source = newFileStore(src).withMaxFileSize(8).create(); - NodeStore store = new SegmentNodeStore(source); + NodeStore store = newSegmentNodeStore(source).create(); init(store); - - // initial content FileStoreBackup.backup(store, destination); - addTestContent(store); + source.close(); - FileStoreRestore.restore(destination, store); - - compare(store, destination); + FileStoreRestore.restore(destination, src); + source = newFileStore(src).withMaxFileSize(8).create(); + compare(source, destination); source.close(); } private static void addTestContent(NodeStore store) - throws CommitFailedException { + throws CommitFailedException, IOException { NodeBuilder builder = store.getRoot().builder(); - builder.child("test-backup"); + NodeBuilder c = builder.child("test-backup").child("binaries"); + for (int i = 0; i < 2; i++) { + c.setProperty("bin" + i, createBlob(store, 64 * 1024)); + } builder.child("root"); // make sure we don't backup the super-root store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); } - private static void compare(NodeStore store, File destination) + private static Blob createBlob(NodeStore nodeStore, int size) throws IOException { + byte[] data = new byte[size]; + new Random().nextBytes(data); + return nodeStore.createBlob(new ByteArrayInputStream(data)); + } + + private static void compare(FileStore store, File destination) throws IOException { - FileStore backup = new FileStore(destination, 8, false); - assertEquals(store.getRoot(), new SegmentNodeStore(backup).getRoot()); + FileStore backup = newFileStore(destination).withMaxFileSize(8).create(); + assertEquals(store.getHead(), backup.getHead()); backup.close(); } @@ -119,45 +125,4 @@ public class FileStoreBackupTest { new Oak(store).with(new OpenSecurityProvider()) .with(new InitialContent()).createContentRepository(); } - - public void testSharedContent() throws Exception { - FileStore source = new FileStore(src, 256, false); - - NodeStore store = new SegmentNodeStore(source); - - // ~100k - Blob blob = store.createBlob(new ByteArrayInputStream(new byte[100000])); - - NodeBuilder builder = store.getRoot().builder(); - NodeBuilder c1 = builder.child("test-backup"); - c1.setProperty("blob", blob); - NodeBuilder c2 = builder.child("test-backup2"); - c2.setProperty("blob", blob); - store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); - - FileStoreBackup.backup(store, destination); - compare(store, destination); - source.close(); - - Map<String, Long> expected = new HashMap<String, Long>(); - for (File f : src.listFiles()) { - if (f.getName().endsWith(".tar")) { - expected.put(f.getName(), f.length()); - } - } - - for (File f : destination.listFiles()) { - if (!f.getName().endsWith(".tar")) { - continue; - } - assertTrue(f.getName() + " is missing from the backup", - expected.containsKey(f.getName())); - assertTrue( - f.getName() + " is expected to have size <= " - + expected.get(f.getName()) + " actually is " - + f.length(), - f.length() <= expected.get(f.getName())); - } - - } }
