Repository: aurora Updated Branches: refs/heads/master eae686023 -> 4b3d7bca9
Remove storage backfill and TaskStore mutateTasks. Reviewed at https://reviews.apache.org/r/42646/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/4b3d7bca Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/4b3d7bca Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/4b3d7bca Branch: refs/heads/master Commit: 4b3d7bca93adc7857820f997fc5d0b1831761e15 Parents: eae6860 Author: Bill Farner <[email protected]> Authored: Fri Jan 22 14:15:55 2016 -0800 Committer: Bill Farner <[email protected]> Committed: Fri Jan 22 14:15:55 2016 -0800 ---------------------------------------------------------------------- .../aurora/scheduler/SchedulerLifecycle.java | 6 +- .../aurora/scheduler/base/TaskTestUtil.java | 2 +- .../scheduler/storage/StorageBackfill.java | 84 -------------------- .../aurora/scheduler/storage/TaskStore.java | 13 --- .../scheduler/storage/db/DbTaskStore.java | 42 +++------- .../storage/log/WriteAheadStorage.java | 21 ----- .../scheduler/storage/mem/MemTaskStore.java | 36 ++------- .../aurora/scheduler/app/SchedulerIT.java | 8 +- .../storage/AbstractTaskStoreTest.java | 43 +++++----- .../scheduler/storage/StorageBackfillTest.java | 72 ----------------- .../storage/log/WriteAheadStorageTest.java | 30 +++---- 11 files changed, 51 insertions(+), 306 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/4b3d7bca/src/main/java/org/apache/aurora/scheduler/SchedulerLifecycle.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/SchedulerLifecycle.java b/src/main/java/org/apache/aurora/scheduler/SchedulerLifecycle.java index 0d3874e..5ba5e73 100644 --- a/src/main/java/org/apache/aurora/scheduler/SchedulerLifecycle.java +++ b/src/main/java/org/apache/aurora/scheduler/SchedulerLifecycle.java @@ -56,7 +56,6 @@ import org.apache.aurora.scheduler.events.PubsubEvent.DriverRegistered; import org.apache.aurora.scheduler.events.PubsubEvent.EventSubscriber; import org.apache.aurora.scheduler.mesos.Driver; import org.apache.aurora.scheduler.storage.Storage.NonVolatileStorage; -import org.apache.aurora.scheduler.storage.StorageBackfill; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -242,7 +241,10 @@ public class SchedulerLifecycle implements EventSubscriber { @Override public void execute(Transition<State> transition) { LOG.info("Elected as leading scheduler!"); - storage.start(StorageBackfill::backfill); + + storage.start(stores -> { + // If storage backfill operations are necessary, they can be done here. + }); driver.startAsync().awaitRunning(); http://git-wip-us.apache.org/repos/asf/aurora/blob/4b3d7bca/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java b/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java index 376f485..02ba1e3 100644 --- a/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java +++ b/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java @@ -97,7 +97,7 @@ public final class TaskTestUtil { public static IScheduledTask makeTask(String id, ITaskConfig config) { return IScheduledTask.build(new ScheduledTask() - .setStatus(ScheduleStatus.PENDING) + .setStatus(ScheduleStatus.ASSIGNED) .setTaskEvents(ImmutableList.of( new TaskEvent(100L, ScheduleStatus.PENDING) .setMessage("message") http://git-wip-us.apache.org/repos/asf/aurora/blob/4b3d7bca/src/main/java/org/apache/aurora/scheduler/storage/StorageBackfill.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/StorageBackfill.java b/src/main/java/org/apache/aurora/scheduler/storage/StorageBackfill.java deleted file mode 100644 index 609a624..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/StorageBackfill.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage; - -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.aurora.common.stats.Stats; -import org.apache.aurora.gen.JobConfiguration; -import org.apache.aurora.gen.JobKey; -import org.apache.aurora.gen.ScheduledTask; -import org.apache.aurora.gen.TaskConfig; -import org.apache.aurora.scheduler.base.JobKeys; -import org.apache.aurora.scheduler.base.Query; -import org.apache.aurora.scheduler.storage.Storage.MutableStoreProvider; -import org.apache.aurora.scheduler.storage.entities.IJobConfiguration; -import org.apache.aurora.scheduler.storage.entities.IJobKey; -import org.apache.aurora.scheduler.storage.entities.IScheduledTask; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -/** - * Utility class to contain and perform storage backfill operations. - */ -public final class StorageBackfill { - - private static final Logger LOG = LoggerFactory.getLogger(StorageBackfill.class); - - private static final AtomicLong BACKFILLED_TASK_CONFIG_KEYS = - Stats.exportLong("task_config_keys_backfilled"); - - private static final AtomicLong BACKFILLED_JOB_CONFIG_KEYS = - Stats.exportLong("job_store_task_config_keys_backfilled"); - - private StorageBackfill() { - // Utility class. - } - - private static void backfillJobDefaults(CronJobStore.Mutable jobStore) { - for (JobConfiguration job : IJobConfiguration.toBuildersList(jobStore.fetchJobs())) { - populateJobKey(job.getTaskConfig(), BACKFILLED_JOB_CONFIG_KEYS); - jobStore.saveAcceptedJob(IJobConfiguration.build(job)); - } - } - - private static void populateJobKey(TaskConfig config, AtomicLong counter) { - if (!config.isSetJob() || !JobKeys.isValid(IJobKey.build(config.getJob()))) { - config.setJob(new JobKey() - .setRole(config.getOwner().getRole()) - .setEnvironment(config.getEnvironment()) - .setName(config.getJobName())); - - counter.incrementAndGet(); - } - } - - /** - * Backfills the storage to make it match any assumptions that may have changed since - * the structs were first written. - * - * @param storeProvider Storage provider. - */ - public static void backfill(final MutableStoreProvider storeProvider) { - backfillJobDefaults(storeProvider.getCronJobStore()); - - // Backfilling job keys has to be done in a separate transaction to ensure follow up scoped - // Query calls work against upgraded MemTaskStore, which does not support deprecated fields. - LOG.info("Backfilling task config job keys."); - storeProvider.getUnsafeTaskStore().mutateTasks(Query.unscoped(), task -> { - ScheduledTask builder = task.newBuilder(); - populateJobKey(builder.getAssignedTask().getTask(), BACKFILLED_TASK_CONFIG_KEYS); - return IScheduledTask.build(builder); - }); - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/4b3d7bca/src/main/java/org/apache/aurora/scheduler/storage/TaskStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/TaskStore.java b/src/main/java/org/apache/aurora/scheduler/storage/TaskStore.java index 4e4f8d2..3ab2d54 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/TaskStore.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/TaskStore.java @@ -18,7 +18,6 @@ import java.util.Set; import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Predicate; -import com.google.common.collect.ImmutableSet; import org.apache.aurora.gen.TaskQuery; import org.apache.aurora.scheduler.base.Query; @@ -102,18 +101,6 @@ public interface TaskStore { Function<IScheduledTask, IScheduledTask> mutator); /** - * Offers temporary mutable access to tasks. If a task ID is not found, it will be silently - * skipped, and no corresponding task will be returned. - * - * @param query Query to match tasks against. - * @param mutator The mutate operation. - * @return Immutable copies <em>of only the tasks that were mutated</em>. - */ - ImmutableSet<IScheduledTask> mutateTasks( - Query.Builder query, - Function<IScheduledTask, IScheduledTask> mutator); - - /** * Rewrites a task's configuration in-place. * <p> * <b>WARNING</b>: this is a dangerous operation, and should not be used without exercising http://git-wip-us.apache.org/repos/asf/aurora/blob/4b3d7bca/src/main/java/org/apache/aurora/scheduler/storage/db/DbTaskStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/DbTaskStore.java b/src/main/java/org/apache/aurora/scheduler/storage/db/DbTaskStore.java index 43fda1d..b899d0f 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/DbTaskStore.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/db/DbTaskStore.java @@ -168,10 +168,16 @@ class DbTaskStore implements TaskStore.Mutable { } } - private Function<IScheduledTask, IScheduledTask> mutateAndSave( + @Timed("db_storage_mutate_task") + @Override + public Optional<IScheduledTask> mutateTask( + String taskId, Function<IScheduledTask, IScheduledTask> mutator) { - return original -> { + requireNonNull(taskId); + requireNonNull(mutator); + + return fetchTask(taskId).transform(original -> { IScheduledTask maybeMutated = mutator.apply(original); requireNonNull(maybeMutated); if (!original.equals(maybeMutated)) { @@ -181,37 +187,7 @@ class DbTaskStore implements TaskStore.Mutable { saveTasks(ImmutableSet.of(maybeMutated)); } return maybeMutated; - }; - } - - @Timed("db_storage_mutate_task") - @Override - public Optional<IScheduledTask> mutateTask( - String taskId, - Function<IScheduledTask, IScheduledTask> mutator) { - - requireNonNull(taskId); - requireNonNull(mutator); - - return fetchTask(taskId).transform(mutateAndSave(mutator)); - } - - @Timed("db_storage_mutate_tasks") - @Override - public ImmutableSet<IScheduledTask> mutateTasks( - Builder query, - Function<IScheduledTask, IScheduledTask> mutator) { - - requireNonNull(query); - requireNonNull(mutator); - - Function<IScheduledTask, IScheduledTask> mutateFunction = mutateAndSave(mutator); - Iterable<Optional<IScheduledTask>> mutations = matches(query) - .transform(original -> { - IScheduledTask mutateResult = mutateFunction.apply(original); - return original.equals(mutateResult) ? Optional.absent() : Optional.of(mutateResult); - }); - return ImmutableSet.copyOf(Optional.presentInstances(mutations)); + }); } @Timed("db_storage_unsafe_modify_in_place") http://git-wip-us.apache.org/repos/asf/aurora/blob/4b3d7bca/src/main/java/org/apache/aurora/scheduler/storage/log/WriteAheadStorage.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/WriteAheadStorage.java b/src/main/java/org/apache/aurora/scheduler/storage/log/WriteAheadStorage.java index 7283531..2f07afb 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/log/WriteAheadStorage.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/log/WriteAheadStorage.java @@ -13,14 +13,12 @@ */ package org.apache.aurora.scheduler.storage.log; -import java.util.Map; import java.util.Set; import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; import org.apache.aurora.gen.storage.Op; import org.apache.aurora.gen.storage.PruneJobUpdateHistory; @@ -38,8 +36,6 @@ import org.apache.aurora.gen.storage.SaveJobUpdateEvent; import org.apache.aurora.gen.storage.SaveLock; import org.apache.aurora.gen.storage.SaveQuota; import org.apache.aurora.gen.storage.SaveTasks; -import org.apache.aurora.scheduler.base.Query; -import org.apache.aurora.scheduler.base.Tasks; import org.apache.aurora.scheduler.events.EventSink; import org.apache.aurora.scheduler.events.PubsubEvent; import org.apache.aurora.scheduler.storage.AttributeStore; @@ -205,23 +201,6 @@ class WriteAheadStorage extends WriteAheadStorageForwarder implements } @Override - public ImmutableSet<IScheduledTask> mutateTasks( - Query.Builder query, - Function<IScheduledTask, IScheduledTask> mutator) { - - ImmutableSet<IScheduledTask> mutated = taskStore.mutateTasks(query, mutator); - - Map<String, IScheduledTask> tasksById = Tasks.mapById(mutated); - log.debug( - "Storing updated tasks to log: {}", - Maps.transformValues(tasksById, IScheduledTask::getStatus)); - - // TODO(William Farner): Avoid writing an op when mutated is empty. - write(Op.saveTasks(new SaveTasks(IScheduledTask.toBuildersSet(mutated)))); - return mutated; - } - - @Override public void saveQuota(final String role, final IResourceAggregate quota) { requireNonNull(role); requireNonNull(quota); http://git-wip-us.apache.org/repos/asf/aurora/blob/4b3d7bca/src/main/java/org/apache/aurora/scheduler/storage/mem/MemTaskStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemTaskStore.java b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemTaskStore.java index 8fd024a..231a556 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemTaskStore.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemTaskStore.java @@ -203,10 +203,13 @@ class MemTaskStore implements TaskStore.Mutable { } } - private Function<IScheduledTask, IScheduledTask> mutateAndSave( + @Timed("mem_storage_mutate_task") + @Override + public Optional<IScheduledTask> mutateTask( + String taskId, Function<IScheduledTask, IScheduledTask> mutator) { - return original -> { + return fetchTask(taskId).transform(original -> { IScheduledTask maybeMutated = mutator.apply(original); requireNonNull(maybeMutated); if (!original.equals(maybeMutated)) { @@ -219,34 +222,7 @@ class MemTaskStore implements TaskStore.Mutable { } } return maybeMutated; - }; - } - - @Timed("mem_storage_mutate_task") - @Override - public Optional<IScheduledTask> mutateTask( - String taskId, - Function<IScheduledTask, IScheduledTask> mutator) { - - return fetchTask(taskId).transform(mutateAndSave(mutator)); - } - - @Timed("mem_storage_mutate_tasks") - @Override - public ImmutableSet<IScheduledTask> mutateTasks( - Query.Builder query, - Function<IScheduledTask, IScheduledTask> mutator) { - - requireNonNull(query); - requireNonNull(mutator); - - Function<IScheduledTask, IScheduledTask> mutateFunction = mutateAndSave(mutator); - Iterable<Optional<IScheduledTask>> mutations = matches(query) - .transform(original -> { - IScheduledTask mutateResult = mutateFunction.apply(original); - return original.equals(mutateResult) ? Optional.absent() : Optional.of(mutateResult); - }); - return ImmutableSet.copyOf(Optional.presentInstances(mutations)); + }); } @Timed("mem_storage_unsafe_modify_in_place") http://git-wip-us.apache.org/repos/asf/aurora/blob/4b3d7bca/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java b/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java index 9fb8aad..83f4b7a 100644 --- a/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java +++ b/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java @@ -307,21 +307,17 @@ public class SchedulerIT extends BaseZooKeeperTest { expect(log.open()).andReturn(logStream); expect(logStream.readAll()).andReturn(recoveredEntries.iterator()).anyTimes(); - // An empty saveTasks is an artifact of the fact that mutateTasks always writes a log operation - // even if nothing is changed. - streamMatcher.expectTransaction(Op.saveTasks(new SaveTasks(ImmutableSet.of()))) - .andReturn(nextPosition()); streamMatcher.expectTransaction(Op.saveFrameworkId(new SaveFrameworkId(FRAMEWORK_ID))) .andReturn(nextPosition()); - final CountDownLatch driverStarted = new CountDownLatch(1); + CountDownLatch driverStarted = new CountDownLatch(1); expect(driver.start()).andAnswer(() -> { driverStarted.countDown(); return Status.DRIVER_RUNNING; }); // Try to be a good test suite citizen by releasing the blocked thread when the test case exits. - final CountDownLatch testCompleted = new CountDownLatch(1); + CountDownLatch testCompleted = new CountDownLatch(1); expect(driver.join()).andAnswer(() -> { testCompleted.await(); return Status.DRIVER_STOPPED; http://git-wip-us.apache.org/repos/asf/aurora/blob/4b3d7bca/src/test/java/org/apache/aurora/scheduler/storage/AbstractTaskStoreTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/AbstractTaskStoreTest.java b/src/test/java/org/apache/aurora/scheduler/storage/AbstractTaskStoreTest.java index 1ac41d1..dee34e2 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/AbstractTaskStoreTest.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/AbstractTaskStoreTest.java @@ -39,7 +39,6 @@ import org.apache.aurora.gen.HostAttributes; import org.apache.aurora.gen.MaintenanceMode; import org.apache.aurora.gen.MesosContainer; import org.apache.aurora.gen.Metadata; -import org.apache.aurora.gen.ScheduleStatus; import org.apache.aurora.gen.ScheduledTask; import org.apache.aurora.gen.TaskQuery; import org.apache.aurora.scheduler.base.JobKeys; @@ -57,6 +56,7 @@ import org.junit.Before; import org.junit.Ignore; import org.junit.Test; +import static org.apache.aurora.gen.ScheduleStatus.ASSIGNED; import static org.apache.aurora.gen.ScheduleStatus.RUNNING; import static org.apache.aurora.scheduler.base.TaskTestUtil.makeTask; import static org.junit.Assert.assertEquals; @@ -123,11 +123,6 @@ public abstract class AbstractTaskStoreTest extends TearDownTestCase { storeProvider -> storeProvider.getUnsafeTaskStore().mutateTask(taskId, mutation)); } - private ImmutableSet<IScheduledTask> mutateTasks(Query.Builder query, TaskMutation mutation) { - return storage.write( - storeProvider -> storeProvider.getUnsafeTaskStore().mutateTasks(query, mutation)); - } - private boolean unsafeModifyInPlace(String taskId, ITaskConfig taskConfiguration) { return storage.write(storeProvider -> storeProvider.getUnsafeTaskStore().unsafeModifyInPlace(taskId, taskConfiguration)); @@ -183,7 +178,7 @@ public abstract class AbstractTaskStoreTest extends TearDownTestCase { assertQueryResults(Query.envScoped("role-c", "env-c"), TASK_C); assertQueryResults(Query.envScoped("role-c", "devel")); assertQueryResults( - Query.unscoped().byStatus(ScheduleStatus.PENDING), + Query.unscoped().byStatus(ASSIGNED), TASK_A, TASK_B, TASK_C, TASK_D); assertQueryResults( Query.instanceScoped(JobKeys.from("role-a", "env-a", "job-a"), 2).active(), TASK_A); @@ -273,15 +268,17 @@ public abstract class AbstractTaskStoreTest extends TearDownTestCase { Query.statusScoped(RUNNING), IScheduledTask.build(TASK_A.newBuilder().setStatus(RUNNING))); - mutateTasks( - Query.unscoped(), - task -> IScheduledTask.build(task.newBuilder().setStatus(ScheduleStatus.ASSIGNED))); + assertEquals( + Optional.absent(), + mutateTask( + "nonexistent", + task -> IScheduledTask.build(task.newBuilder().setStatus(RUNNING)))); assertStoreContents( - IScheduledTask.build(TASK_A.newBuilder().setStatus(ScheduleStatus.ASSIGNED)), - IScheduledTask.build(TASK_B.newBuilder().setStatus(ScheduleStatus.ASSIGNED)), - IScheduledTask.build(TASK_C.newBuilder().setStatus(ScheduleStatus.ASSIGNED)), - IScheduledTask.build(TASK_D.newBuilder().setStatus(ScheduleStatus.ASSIGNED))); + IScheduledTask.build(TASK_A.newBuilder().setStatus(RUNNING)), + IScheduledTask.build(TASK_B.newBuilder().setStatus(ASSIGNED)), + IScheduledTask.build(TASK_C.newBuilder().setStatus(ASSIGNED)), + IScheduledTask.build(TASK_D.newBuilder().setStatus(ASSIGNED))); } @Test @@ -323,14 +320,14 @@ public abstract class AbstractTaskStoreTest extends TearDownTestCase { @Test public void testConsistentJobIndex() { - final IScheduledTask a = makeTask("a", JobKeys.from("jim", "test", "job")); - final IScheduledTask b = makeTask("b", JobKeys.from("jim", "test", "job")); - final IScheduledTask c = makeTask("c", JobKeys.from("jim", "test", "job2")); - final IScheduledTask d = makeTask("d", JobKeys.from("joe", "test", "job")); - final IScheduledTask e = makeTask("e", JobKeys.from("jim", "prod", "job")); - final Query.Builder jimsJob = Query.jobScoped(JobKeys.from("jim", "test", "job")); - final Query.Builder jimsJob2 = Query.jobScoped(JobKeys.from("jim", "test", "job2")); - final Query.Builder joesJob = Query.jobScoped(JobKeys.from("joe", "test", "job")); + IScheduledTask a = makeTask("a", JobKeys.from("jim", "test", "job")); + IScheduledTask b = makeTask("b", JobKeys.from("jim", "test", "job")); + IScheduledTask c = makeTask("c", JobKeys.from("jim", "test", "job2")); + IScheduledTask d = makeTask("d", JobKeys.from("joe", "test", "job")); + IScheduledTask e = makeTask("e", JobKeys.from("jim", "prod", "job")); + Query.Builder jimsJob = Query.jobScoped(JobKeys.from("jim", "test", "job")); + Query.Builder jimsJob2 = Query.jobScoped(JobKeys.from("jim", "test", "job2")); + Query.Builder joesJob = Query.jobScoped(JobKeys.from("joe", "test", "job")); saveTasks(a, b, c, d, e); assertQueryResults(jimsJob, a, b); @@ -342,7 +339,7 @@ public abstract class AbstractTaskStoreTest extends TearDownTestCase { assertQueryResults(jimsJob2, c); assertQueryResults(joesJob, d); - mutateTasks(jimsJob, task -> IScheduledTask.build(task.newBuilder().setStatus(RUNNING))); + mutateTask(Tasks.id(a), task -> IScheduledTask.build(task.newBuilder().setStatus(RUNNING))); IScheduledTask aRunning = IScheduledTask.build(a.newBuilder().setStatus(RUNNING)); assertQueryResults(jimsJob, aRunning); assertQueryResults(jimsJob2, c); http://git-wip-us.apache.org/repos/asf/aurora/blob/4b3d7bca/src/test/java/org/apache/aurora/scheduler/storage/StorageBackfillTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/StorageBackfillTest.java b/src/test/java/org/apache/aurora/scheduler/storage/StorageBackfillTest.java deleted file mode 100644 index 2570464..0000000 --- a/src/test/java/org/apache/aurora/scheduler/storage/StorageBackfillTest.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage; - -import java.util.Set; - -import com.google.common.collect.ImmutableSet; - -import org.apache.aurora.gen.Container; -import org.apache.aurora.gen.MesosContainer; -import org.apache.aurora.gen.ScheduledTask; -import org.apache.aurora.scheduler.base.Query; -import org.apache.aurora.scheduler.base.TaskTestUtil; -import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult; -import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult.Quiet; -import org.apache.aurora.scheduler.storage.db.DbUtil; -import org.apache.aurora.scheduler.storage.entities.IScheduledTask; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * TODO(wfarner): Data store constraints (not null, valid relations) have made this test of little - * value other than for coverage. Rethink. - */ -public class StorageBackfillTest { - // By default, only the mesos container is allowed by ConfigurationManager. - private static final IScheduledTask TASK = - setMesosContainer(TaskTestUtil.makeTask("task_id", TaskTestUtil.JOB)); - - private Storage storage; - - @Before - public void setUp() { - storage = DbUtil.createStorage(); - } - - @Test - public void testBackfillTask() { - final Set<IScheduledTask> backfilledTasks = ImmutableSet.of(TASK); - storage.write((NoResult.Quiet) - storeProvider -> storeProvider.getUnsafeTaskStore().saveTasks(backfilledTasks)); - - backfill(); - - assertEquals( - ImmutableSet.of(TASK), - Storage.Util.fetchTasks(storage, Query.unscoped())); - } - - private void backfill() { - storage.write((Quiet) StorageBackfill::backfill); - } - - private static IScheduledTask setMesosContainer(IScheduledTask task) { - ScheduledTask builder = task.newBuilder(); - builder.getAssignedTask().getTask().setContainer(Container.mesos(new MesosContainer())); - return IScheduledTask.build(builder); - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/4b3d7bca/src/test/java/org/apache/aurora/scheduler/storage/log/WriteAheadStorageTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/log/WriteAheadStorageTest.java b/src/test/java/org/apache/aurora/scheduler/storage/log/WriteAheadStorageTest.java index 3c4e2bd..13726cc 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/log/WriteAheadStorageTest.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/log/WriteAheadStorageTest.java @@ -16,21 +16,20 @@ package org.apache.aurora.scheduler.storage.log; import java.util.Set; import com.google.common.base.Function; +import com.google.common.base.Optional; import com.google.common.collect.ImmutableSet; import org.apache.aurora.common.testing.easymock.EasyMockTest; -import org.apache.aurora.gen.AssignedTask; import org.apache.aurora.gen.Attribute; import org.apache.aurora.gen.HostAttributes; import org.apache.aurora.gen.JobUpdateKey; import org.apache.aurora.gen.MaintenanceMode; -import org.apache.aurora.gen.ScheduledTask; import org.apache.aurora.gen.storage.Op; import org.apache.aurora.gen.storage.PruneJobUpdateHistory; import org.apache.aurora.gen.storage.SaveHostAttributes; import org.apache.aurora.gen.storage.SaveTasks; import org.apache.aurora.scheduler.base.JobKeys; -import org.apache.aurora.scheduler.base.Query; +import org.apache.aurora.scheduler.base.TaskTestUtil; import org.apache.aurora.scheduler.events.EventSink; import org.apache.aurora.scheduler.events.PubsubEvent; import org.apache.aurora.scheduler.storage.AttributeStore; @@ -43,10 +42,9 @@ import org.apache.aurora.scheduler.storage.TaskStore; import org.apache.aurora.scheduler.storage.entities.IHostAttributes; import org.apache.aurora.scheduler.storage.entities.IJobUpdateKey; import org.apache.aurora.scheduler.storage.entities.IScheduledTask; -import org.easymock.EasyMock; import org.junit.Before; import org.junit.Test; -import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static org.easymock.EasyMock.expect; import static org.junit.Assert.assertEquals; @@ -59,7 +57,6 @@ public class WriteAheadStorageTest extends EasyMockTest { private TaskStore.Mutable taskStore; private AttributeStore.Mutable attributeStore; private JobUpdateStore.Mutable jobUpdateStore; - private Logger log; private EventSink eventSink; private WriteAheadStorage storage; @@ -69,7 +66,6 @@ public class WriteAheadStorageTest extends EasyMockTest { taskStore = createMock(TaskStore.Mutable.class); attributeStore = createMock(AttributeStore.Mutable.class); jobUpdateStore = createMock(JobUpdateStore.Mutable.class); - log = createMock(Logger.class); eventSink = createMock(EventSink.class); storage = new WriteAheadStorage( @@ -81,7 +77,7 @@ public class WriteAheadStorageTest extends EasyMockTest { createMock(QuotaStore.Mutable.class), attributeStore, jobUpdateStore, - log, + LoggerFactory.getLogger(WriteAheadStorageTest.class), eventSink); } @@ -115,25 +111,17 @@ public class WriteAheadStorageTest extends EasyMockTest { @Test public void testMutate() { - Query.Builder query = Query.taskScoped("a"); + String taskId = "a"; Function<IScheduledTask, IScheduledTask> mutator = createMock(new Clazz<Function<IScheduledTask, IScheduledTask>>() { }); - ImmutableSet<IScheduledTask> mutated = ImmutableSet.of(IScheduledTask.build( - new ScheduledTask().setAssignedTask(new AssignedTask().setTaskId("a")))); + Optional<IScheduledTask> mutated = Optional.of(TaskTestUtil.makeTask(taskId, TaskTestUtil.JOB)); - expect(taskStore.mutateTasks(query, mutator)).andReturn(mutated); - log.debug(EasyMock.anyString(), EasyMock.<Object>anyObject()); - expectOp(Op.saveTasks(new SaveTasks(IScheduledTask.toBuildersSet(mutated)))); - - // With increased logging. - expect(taskStore.mutateTasks(query, mutator)).andReturn(mutated); - expectOp(Op.saveTasks(new SaveTasks(IScheduledTask.toBuildersSet(mutated)))); - log.debug(EasyMock.anyString(), EasyMock.<Object>anyObject()); + expect(taskStore.mutateTask(taskId, mutator)).andReturn(mutated); + expectOp(Op.saveTasks(new SaveTasks(ImmutableSet.of(mutated.get().newBuilder())))); control.replay(); - assertEquals(mutated, storage.mutateTasks(query, mutator)); - assertEquals(mutated, storage.mutateTasks(query, mutator)); + assertEquals(mutated, storage.mutateTask(taskId, mutator)); } @Test
