Repository: aurora Updated Branches: refs/heads/master 12e92642f -> 95dcca56d
Revert "Moving db migration into LogStorage" This reverts commit ae051f3b92797d5c9f328c6c6d42d03ee4077938. Reviewed at https://reviews.apache.org/r/46499/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/95dcca56 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/95dcca56 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/95dcca56 Branch: refs/heads/master Commit: 95dcca56da934f1daaee6924af057eb60f95ad1e Parents: 12e9264 Author: Maxim Khutornenko <[email protected]> Authored: Thu Apr 21 14:10:41 2016 -0700 Committer: Maxim Khutornenko <[email protected]> Committed: Thu Apr 21 14:10:41 2016 -0700 ---------------------------------------------------------------------- .../storage/backup/TemporaryStorage.java | 5 +++- .../scheduler/storage/log/LogStorage.java | 27 +++----------------- .../storage/log/SnapshotStoreImpl.java | 12 ++++++++- .../scheduler/storage/log/LogStorageTest.java | 9 ++----- .../storage/log/SnapshotStoreImplIT.java | 4 ++- 5 files changed, 23 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/95dcca56/src/main/java/org/apache/aurora/scheduler/storage/backup/TemporaryStorage.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/backup/TemporaryStorage.java b/src/main/java/org/apache/aurora/scheduler/storage/backup/TemporaryStorage.java index 5c7d92f..d08873c 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/backup/TemporaryStorage.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/backup/TemporaryStorage.java @@ -76,7 +76,10 @@ interface TemporaryStorage { storage, // Safe to pass false here to default to the non-experimental task store // during restore from backup procedure. - false /** useDbSnapshotForTaskStore */); + false /** useDbSnapshotForTaskStore */, + // We can just pass an empty lambda for the MigrationManager as migration is a no-op + // when restoring from backup. + () -> { } /** migrationManager */); snapshotStore.applySnapshot(snapshot); return new TemporaryStorage() { http://git-wip-us.apache.org/repos/asf/aurora/blob/95dcca56/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorage.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorage.java b/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorage.java index 74c4688..f586186 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorage.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorage.java @@ -14,7 +14,6 @@ package org.apache.aurora.scheduler.storage.log; import java.io.IOException; -import java.sql.SQLException; import java.util.Date; import java.util.Map; import java.util.concurrent.ScheduledExecutorService; @@ -62,7 +61,6 @@ import org.apache.aurora.scheduler.storage.Storage; import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult; import org.apache.aurora.scheduler.storage.Storage.NonVolatileStorage; import org.apache.aurora.scheduler.storage.TaskStore; -import org.apache.aurora.scheduler.storage.db.MigrationManager; import org.apache.aurora.scheduler.storage.entities.IHostAttributes; import org.apache.aurora.scheduler.storage.entities.IJobConfiguration; import org.apache.aurora.scheduler.storage.entities.IJobInstanceUpdateEvent; @@ -192,7 +190,6 @@ public class LogStorage implements NonVolatileStorage, DistributedSnapshotStore private final AttributeStore.Mutable writeBehindAttributeStore; private final JobUpdateStore.Mutable writeBehindJobUpdateStore; private final ReentrantLock writeLock; - private final MigrationManager migrationManager; private StreamManager streamManager; private final WriteAheadStorage writeAheadStorage; @@ -225,8 +222,7 @@ public class LogStorage implements NonVolatileStorage, DistributedSnapshotStore @Volatile AttributeStore.Mutable attributeStore, @Volatile JobUpdateStore.Mutable jobUpdateStore, EventSink eventSink, - ReentrantLock writeLock, - MigrationManager migrationManager) { + ReentrantLock writeLock) { this(logManager, new ScheduledExecutorSchedulingService(shutdownRegistry, settings.getShutdownGracePeriod()), @@ -241,8 +237,7 @@ public class LogStorage implements NonVolatileStorage, DistributedSnapshotStore attributeStore, jobUpdateStore, eventSink, - writeLock, - migrationManager); + writeLock); } @VisibleForTesting @@ -260,8 +255,7 @@ public class LogStorage implements NonVolatileStorage, DistributedSnapshotStore AttributeStore.Mutable attributeStore, JobUpdateStore.Mutable jobUpdateStore, EventSink eventSink, - ReentrantLock writeLock, - MigrationManager migrationManager) { + ReentrantLock writeLock) { this.logManager = requireNonNull(logManager); this.schedulingService = requireNonNull(schedulingService); @@ -281,7 +275,6 @@ public class LogStorage implements NonVolatileStorage, DistributedSnapshotStore this.writeBehindAttributeStore = requireNonNull(attributeStore); this.writeBehindJobUpdateStore = requireNonNull(jobUpdateStore); this.writeLock = requireNonNull(writeLock); - this.migrationManager = requireNonNull(migrationManager); TransactionManager transactionManager = new TransactionManager() { @Override public boolean hasActiveTransaction() { @@ -422,11 +415,6 @@ public class LogStorage implements NonVolatileStorage, DistributedSnapshotStore // We replay these entries in the forwarded storage system's transactions but not ours - we // do not want to re-record these ops to the log. recover(); - - // Apply any schema/data migrations before 'activating' write-ahead (native log) writes. - migrate(); - - // Activate write-ahead writes now that the store is fully recovered and migrated. recovered = true; // Now that we're recovered we should let any mutations done in initializationLogic append @@ -451,15 +439,6 @@ public class LogStorage implements NonVolatileStorage, DistributedSnapshotStore } } - @Timed("scheduler_log_migrate") - void migrate() { - try { - migrationManager.migrate(); - } catch (SQLException e) { - throw new RecoveryFailedException(e); - } - } - private static final class RecoveryFailedException extends SchedulerException { RecoveryFailedException(Throwable cause) { super(cause); http://git-wip-us.apache.org/repos/asf/aurora/blob/95dcca56/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImpl.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImpl.java b/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImpl.java index 97b9e26..b6922e1 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImpl.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImpl.java @@ -56,6 +56,7 @@ import org.apache.aurora.scheduler.storage.Storage; import org.apache.aurora.scheduler.storage.Storage.MutableStoreProvider; import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult; import org.apache.aurora.scheduler.storage.Storage.Volatile; +import org.apache.aurora.scheduler.storage.db.MigrationManager; import org.apache.aurora.scheduler.storage.entities.IHostAttributes; import org.apache.aurora.scheduler.storage.entities.IJobConfiguration; import org.apache.aurora.scheduler.storage.entities.IJobInstanceUpdateEvent; @@ -147,6 +148,12 @@ public class SnapshotStoreImpl implements SnapshotStore<Snapshot> { } catch (SQLException e) { Throwables.propagate(e); } + + try { + migrationManager.migrate(); + } catch (SQLException e) { + Throwables.propagate(e); + } } } }, @@ -349,6 +356,7 @@ public class SnapshotStoreImpl implements SnapshotStore<Snapshot> { private final Clock clock; private final Storage storage; private final boolean useDbSnapshotForTaskStore; + private final MigrationManager migrationManager; /** * Identifies if experimental task store is in use. @@ -363,12 +371,14 @@ public class SnapshotStoreImpl implements SnapshotStore<Snapshot> { BuildInfo buildInfo, Clock clock, @Volatile Storage storage, - @ExperimentalTaskStore boolean useDbSnapshotForTaskStore) { + @ExperimentalTaskStore boolean useDbSnapshotForTaskStore, + MigrationManager migrationManager) { this.buildInfo = requireNonNull(buildInfo); this.clock = requireNonNull(clock); this.storage = requireNonNull(storage); this.useDbSnapshotForTaskStore = useDbSnapshotForTaskStore; + this.migrationManager = requireNonNull(migrationManager); } @Timed("snapshot_create") http://git-wip-us.apache.org/repos/asf/aurora/blob/95dcca56/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java b/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java index a2b1d22..bf9479d 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java @@ -92,7 +92,6 @@ import org.apache.aurora.scheduler.storage.Storage.MutableStoreProvider; import org.apache.aurora.scheduler.storage.Storage.MutateWork; import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult; import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult.Quiet; -import org.apache.aurora.scheduler.storage.db.MigrationManager; import org.apache.aurora.scheduler.storage.entities.IHostAttributes; import org.apache.aurora.scheduler.storage.entities.IJobConfiguration; import org.apache.aurora.scheduler.storage.entities.IJobInstanceUpdateEvent; @@ -143,7 +142,7 @@ public class LogStorageTest extends EasyMockTest { private EventSink eventSink; @Before - public void setUp() throws Exception { + public void setUp() { log = createMock(Log.class); deduplicator = createMock(SnapshotDeduplicator.class); @@ -161,9 +160,6 @@ public class LogStorageTest extends EasyMockTest { snapshotStore = createMock(new Clazz<SnapshotStore<Snapshot>>() { }); storageUtil = new StorageTestUtil(this); eventSink = createMock(EventSink.class); - MigrationManager migrationManager = createMock(MigrationManager.class); - migrationManager.migrate(); - expectLastCall().anyTimes(); logStorage = new LogStorage( logManager, @@ -179,8 +175,7 @@ public class LogStorageTest extends EasyMockTest { storageUtil.attributeStore, storageUtil.jobUpdateStore, eventSink, - new ReentrantLock(), - migrationManager); + new ReentrantLock()); stream = createMock(Stream.class); streamMatcher = LogOpMatcher.matcherFor(stream); http://git-wip-us.apache.org/repos/asf/aurora/blob/95dcca56/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImplIT.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImplIT.java b/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImplIT.java index d5918b9..ff9c1d0 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImplIT.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImplIT.java @@ -54,6 +54,7 @@ import org.apache.aurora.scheduler.base.TaskTestUtil; import org.apache.aurora.scheduler.resources.ResourceAggregates; import org.apache.aurora.scheduler.storage.SnapshotStore; import org.apache.aurora.scheduler.storage.Storage; +import org.apache.aurora.scheduler.storage.db.MigrationManager; import org.apache.aurora.scheduler.storage.entities.IHostAttributes; import org.apache.aurora.scheduler.storage.entities.IJobConfiguration; import org.apache.aurora.scheduler.storage.entities.IJobKey; @@ -97,7 +98,8 @@ public class SnapshotStoreImplIT { generateBuildInfo(), clock, storage, - dbTaskStore); + dbTaskStore, + createStorageInjector(testModuleWithWorkQueue()).getInstance(MigrationManager.class)); } private static Snapshot makeComparable(Snapshot snapshot) {
