Repository: aurora Updated Branches: refs/heads/master 80139da46 -> 284f40f5e
http://git-wip-us.apache.org/repos/asf/aurora/blob/284f40f5/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java b/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java index 9fab7ae..1510817 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java @@ -15,16 +15,13 @@ package org.apache.aurora.scheduler.storage; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; import com.google.inject.Injector; import org.apache.aurora.gen.InstanceTaskConfig; @@ -46,48 +43,37 @@ import org.apache.aurora.gen.TaskConfig; import org.apache.aurora.scheduler.base.JobKeys; import org.apache.aurora.scheduler.base.TaskTestUtil; import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult; -import org.apache.aurora.scheduler.storage.Storage.StorageException; import org.apache.aurora.scheduler.storage.entities.IJobInstanceUpdateEvent; import org.apache.aurora.scheduler.storage.entities.IJobKey; -import org.apache.aurora.scheduler.storage.entities.IJobUpdate; import org.apache.aurora.scheduler.storage.entities.IJobUpdateDetails; import org.apache.aurora.scheduler.storage.entities.IJobUpdateEvent; import org.apache.aurora.scheduler.storage.entities.IJobUpdateInstructions; import org.apache.aurora.scheduler.storage.entities.IJobUpdateKey; import org.apache.aurora.scheduler.storage.entities.IJobUpdateQuery; -import org.apache.aurora.scheduler.storage.entities.IJobUpdateState; import org.apache.aurora.scheduler.storage.entities.IJobUpdateSummary; import org.apache.aurora.scheduler.storage.testing.StorageEntityUtil; -import org.apache.aurora.scheduler.testing.FakeStatsProvider; import org.junit.After; import org.junit.Before; import org.junit.Test; -import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_ROLLBACK_FAILED; import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_ROLLED_BACK; import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_ROLLING_BACK; import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_UPDATED; import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_UPDATING; import static org.apache.aurora.gen.JobUpdateStatus.ABORTED; import static org.apache.aurora.gen.JobUpdateStatus.ERROR; -import static org.apache.aurora.gen.JobUpdateStatus.FAILED; import static org.apache.aurora.gen.JobUpdateStatus.ROLLED_BACK; import static org.apache.aurora.gen.JobUpdateStatus.ROLLING_BACK; import static org.apache.aurora.gen.JobUpdateStatus.ROLLING_FORWARD; -import static org.apache.aurora.gen.JobUpdateStatus.ROLL_BACK_PAUSED; import static org.apache.aurora.gen.JobUpdateStatus.ROLL_FORWARD_PAUSED; import static org.apache.aurora.gen.Resource.diskMb; import static org.apache.aurora.gen.Resource.numCpus; import static org.apache.aurora.gen.Resource.ramMb; -import static org.apache.aurora.scheduler.storage.Util.jobUpdateActionStatName; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; public abstract class AbstractJobUpdateStoreTest { private static final IJobKey JOB = JobKeys.from("testRole", "testEnv", "job"); - private static final IJobUpdateKey UPDATE1 = - IJobUpdateKey.build(new JobUpdateKey(JOB.newBuilder(), "update1")); private static final long CREATED_MS = 111L; private static final IJobUpdateEvent FIRST_EVENT = makeJobUpdateEvent(ROLLING_FORWARD, CREATED_MS); @@ -95,13 +81,11 @@ public abstract class AbstractJobUpdateStoreTest { ImmutableSet.of(new Metadata("k1", "v1"), new Metadata("k2", "v2"), new Metadata("k3", "v3")); private Storage storage; - private FakeStatsProvider stats; @Before public void setUp() throws Exception { Injector injector = createStorageInjector(); storage = injector.getInstance(Storage.class); - stats = injector.getInstance(FakeStatsProvider.class); } protected abstract Injector createStorageInjector(); @@ -111,9 +95,18 @@ public abstract class AbstractJobUpdateStoreTest { truncateUpdates(); } - private static IJobUpdate makeFullyPopulatedUpdate(IJobUpdateKey key) { - JobUpdate builder = makeJobUpdate(key).newBuilder(); - JobUpdateInstructions instructions = builder.getInstructions(); + private static IJobUpdateDetails makeFullyPopulatedUpdate(IJobUpdateKey key) { + JobUpdateDetails builder = makeJobUpdate(key).newBuilder() + .setUpdateEvents(ImmutableList.of(new JobUpdateEvent() + .setStatus(ROLLING_FORWARD) + .setTimestampMs(1) + .setUser("user") + .setMessage("message"))) + .setInstanceEvents(ImmutableList.of(new JobInstanceUpdateEvent() + .setTimestampMs(2) + .setInstanceId(1) + .setAction(INSTANCE_UPDATING))); + JobUpdateInstructions instructions = builder.getUpdate().getInstructions(); Stream.of( instructions.getInitialState().stream() .map(InstanceTaskConfig::getInstances) @@ -130,7 +123,7 @@ public abstract class AbstractJobUpdateStoreTest { range.setLast(1); } }); - return IJobUpdate.build(builder); + return IJobUpdateDetails.build(builder); } @Test @@ -138,8 +131,8 @@ public abstract class AbstractJobUpdateStoreTest { IJobUpdateKey updateId1 = makeKey(JobKeys.from("role", "env", "name1"), "u1"); IJobUpdateKey updateId2 = makeKey(JobKeys.from("role", "env", "name2"), "u2"); - IJobUpdate update1 = makeFullyPopulatedUpdate(updateId1); - IJobUpdate update2 = makeJobUpdate(updateId2); + IJobUpdateDetails update1 = makeFullyPopulatedUpdate(updateId1); + IJobUpdateDetails update2 = makeJobUpdate(updateId2); assertEquals(Optional.absent(), getUpdate(updateId1)); assertEquals(Optional.absent(), getUpdate(updateId2)); @@ -151,19 +144,10 @@ public abstract class AbstractJobUpdateStoreTest { StorageEntityUtil.getField(Range.class, "first"), StorageEntityUtil.getField(Range.class, "last")); saveUpdate(update1); - assertUpdate(update1); + assertEquals(withUpdateState(update1, ROLLING_FORWARD, 1, 2), getUpdate(key(update1)).get()); saveUpdate(update2); assertUpdate(update2); - - // Colliding update keys should be forbidden. - IJobUpdate update3 = makeJobUpdate(updateId2); - try { - saveUpdate(update3); - fail("Update ID collision should not be allowed"); - } catch (StorageException e) { - // Expected. - } } @Test @@ -171,17 +155,14 @@ public abstract class AbstractJobUpdateStoreTest { // AURORA-1494 regression test validating max resources values are allowed. IJobUpdateKey updateId = makeKey(JobKeys.from("role", "env", "name1"), "u1"); - JobUpdate builder = makeFullyPopulatedUpdate(updateId).newBuilder(); - builder.getInstructions().getDesiredState().getTask().setResources( + JobUpdateDetails builder = makeFullyPopulatedUpdate(updateId).newBuilder(); + builder.getUpdate().getInstructions().getDesiredState().getTask().setResources( ImmutableSet.of( numCpus(Double.MAX_VALUE), ramMb(Long.MAX_VALUE), diskMb(Long.MAX_VALUE))); - IJobUpdate update = IJobUpdate.build(builder); - - assertEquals(Optional.absent(), getUpdate(updateId)); - + IJobUpdateDetails update = makeFullyPopulatedUpdate(updateId); StorageEntityUtil.assertFullyPopulated( update, StorageEntityUtil.getField(JobUpdateSummary.class, "state"), @@ -189,86 +170,86 @@ public abstract class AbstractJobUpdateStoreTest { StorageEntityUtil.getField(Range.class, "first"), StorageEntityUtil.getField(Range.class, "last")); saveUpdate(update); - assertUpdate(update); + assertEquals(withUpdateState(update, ROLLING_FORWARD, 1, 2), getUpdate(key(update)).get()); } @Test public void testSaveNullInitialState() { - JobUpdate builder = makeJobUpdate(makeKey("u1")).newBuilder(); - builder.getInstructions().unsetInitialState(); + JobUpdateDetails builder = makeJobUpdate(makeKey("u1")).newBuilder(); + builder.getUpdate().getInstructions().unsetInitialState(); // Save with null initial state instances. - saveUpdate(IJobUpdate.build(builder)); + saveUpdate(IJobUpdateDetails.build(builder)); - builder.getInstructions().setInitialState(ImmutableSet.of()); - assertUpdate(IJobUpdate.build(builder)); + builder.getUpdate().getInstructions().setInitialState(ImmutableSet.of()); + assertUpdate(IJobUpdateDetails.build(builder)); } @Test public void testSaveNullDesiredState() { - JobUpdate builder = makeJobUpdate(makeKey("u1")).newBuilder(); - builder.getInstructions().unsetDesiredState(); + JobUpdateDetails builder = makeJobUpdate(makeKey("u1")).newBuilder(); + builder.getUpdate().getInstructions().unsetDesiredState(); // Save with null desired state instances. - saveUpdate(IJobUpdate.build(builder)); + saveUpdate(IJobUpdateDetails.build(builder)); - assertUpdate(IJobUpdate.build(builder)); + assertUpdate(IJobUpdateDetails.build(builder)); } @Test(expected = IllegalArgumentException.class) public void testSaveBothInitialAndDesiredMissingThrows() { - JobUpdate builder = makeJobUpdate(makeKey("u1")).newBuilder(); - builder.getInstructions().unsetInitialState(); - builder.getInstructions().unsetDesiredState(); + JobUpdateDetails builder = makeJobUpdate(makeKey("u1")).newBuilder(); + builder.getUpdate().getInstructions().unsetInitialState(); + builder.getUpdate().getInstructions().unsetDesiredState(); - saveUpdate(IJobUpdate.build(builder)); + saveUpdate(IJobUpdateDetails.build(builder)); } @Test(expected = NullPointerException.class) public void testSaveNullInitialStateTaskThrows() { - JobUpdate builder = makeJobUpdate(makeKey("u1")).newBuilder(); - builder.getInstructions().getInitialState().add( + JobUpdateDetails builder = makeJobUpdate(makeKey("u1")).newBuilder(); + builder.getUpdate().getInstructions().getInitialState().add( new InstanceTaskConfig(null, ImmutableSet.of())); - saveUpdate(IJobUpdate.build(builder)); + saveUpdate(IJobUpdateDetails.build(builder)); } @Test(expected = IllegalArgumentException.class) public void testSaveEmptyInitialStateRangesThrows() { - JobUpdate builder = makeJobUpdate(makeKey("u1")).newBuilder(); - builder.getInstructions().getInitialState().add( + JobUpdateDetails builder = makeJobUpdate(makeKey("u1")).newBuilder(); + builder.getUpdate().getInstructions().getInitialState().add( new InstanceTaskConfig( TaskTestUtil.makeConfig(TaskTestUtil.JOB).newBuilder(), ImmutableSet.of())); - saveUpdate(IJobUpdate.build(builder)); + saveUpdate(IJobUpdateDetails.build(builder)); } @Test(expected = NullPointerException.class) public void testSaveNullDesiredStateTaskThrows() { - JobUpdate builder = makeJobUpdate(makeKey("u1")).newBuilder(); - builder.getInstructions().getDesiredState().setTask(null); + JobUpdateDetails builder = makeJobUpdate(makeKey("u1")).newBuilder(); + builder.getUpdate().getInstructions().getDesiredState().setTask(null); - saveUpdate(IJobUpdate.build(builder)); + saveUpdate(IJobUpdateDetails.build(builder)); } @Test(expected = IllegalArgumentException.class) public void testSaveEmptyDesiredStateRangesThrows() { - JobUpdate builder = makeJobUpdate(makeKey("u1")).newBuilder(); - builder.getInstructions().getDesiredState().setInstances(ImmutableSet.of()); + JobUpdateDetails builder = makeJobUpdate(makeKey("u1")).newBuilder(); + builder.getUpdate().getInstructions().getDesiredState().setInstances(ImmutableSet.of()); - saveUpdate(IJobUpdate.build(builder)); + saveUpdate(IJobUpdateDetails.build(builder)); } @Test public void testSaveJobUpdateEmptyInstanceOverrides() { IJobUpdateKey updateId = makeKey("u1"); - IJobUpdate update = makeJobUpdate(updateId); - JobUpdate builder = update.newBuilder(); - builder.getInstructions().getSettings().setUpdateOnlyTheseInstances(ImmutableSet.of()); + JobUpdateDetails builder = makeJobUpdate(updateId).newBuilder(); + builder.getUpdate().getInstructions().getSettings() + .setUpdateOnlyTheseInstances(ImmutableSet.of()); - IJobUpdate expected = IJobUpdate.build(builder); + IJobUpdateDetails expected = IJobUpdateDetails.build(builder); // Save with empty overrides. saveUpdate(expected); @@ -279,126 +260,42 @@ public abstract class AbstractJobUpdateStoreTest { public void testSaveJobUpdateNullInstanceOverrides() { IJobUpdateKey updateId = makeKey("u1"); - IJobUpdate update = makeJobUpdate(updateId); - JobUpdate builder = update.newBuilder(); - builder.getInstructions().getSettings().setUpdateOnlyTheseInstances(ImmutableSet.of()); + JobUpdateDetails builder = makeJobUpdate(updateId).newBuilder(); + builder.getUpdate().getInstructions().getSettings() + .setUpdateOnlyTheseInstances(ImmutableSet.of()); - IJobUpdate expected = IJobUpdate.build(builder); + IJobUpdateDetails expected = IJobUpdateDetails.build(builder); // Save with null overrides. - builder.getInstructions().getSettings().setUpdateOnlyTheseInstances(null); - saveUpdate(IJobUpdate.build(builder)); + builder.getUpdate().getInstructions().getSettings().setUpdateOnlyTheseInstances(null); + saveUpdate(IJobUpdateDetails.build(builder)); assertUpdate(expected); } - @Test(expected = StorageException.class) - public void testSaveJobUpdateTwiceThrows() { - IJobUpdateKey updateId = makeKey("u1"); - IJobUpdate update = makeJobUpdate(updateId); - - saveUpdate(update); - saveUpdate(update); - } - - @Test - public void testSaveJobEvents() { - IJobUpdateKey updateId = makeKey("u3"); - IJobUpdate update = makeJobUpdate(updateId); - IJobUpdateEvent event1 = makeJobUpdateEvent(ROLLING_FORWARD, 124L); - IJobUpdateEvent event2 = makeJobUpdateEvent(ROLL_FORWARD_PAUSED, 125L); - - saveUpdate(update); - assertUpdate(update); - assertEquals(ImmutableList.of(FIRST_EVENT), getUpdateDetails(updateId).get().getUpdateEvents()); - - saveJobEvent(event1, updateId); - assertEquals( - populateExpected(update, ROLLING_FORWARD, CREATED_MS, 124L), - getUpdateDetails(updateId).get().getUpdate()); - assertEquals(event1, getUpdateDetails(updateId).get().getUpdateEvents().get(1)); - - saveJobEvent(event2, updateId); - assertEquals( - populateExpected(update, ROLL_FORWARD_PAUSED, CREATED_MS, 125L), - getUpdateDetails(updateId).get().getUpdate()); - assertEquals(event1, getUpdateDetails(updateId).get().getUpdateEvents().get(1)); - assertEquals(event2, getUpdateDetails(updateId).get().getUpdateEvents().get(2)); - assertStats(ImmutableMap.of(ROLL_FORWARD_PAUSED, 1, ROLLING_FORWARD, 2)); - } - - private <T extends Number> void assertStats(Map<JobUpdateStatus, T> expected) { - for (Map.Entry<JobUpdateStatus, T> entry : expected.entrySet()) { - assertEquals( - entry.getValue().longValue(), - stats.getLongValue(Util.jobUpdateStatusStatName(entry.getKey()))); - } - } - - @Test - public void testSaveInstanceEvents() { - IJobUpdateKey updateId = makeKey("u3"); - IJobUpdate update = makeJobUpdate(updateId); - IJobInstanceUpdateEvent event1 = makeJobInstanceEvent(0, 125L, INSTANCE_UPDATED); - IJobInstanceUpdateEvent event2 = makeJobInstanceEvent(1, 126L, INSTANCE_ROLLING_BACK); - - saveUpdate(update); - assertUpdate(update); - assertEquals(0, getUpdateDetails(updateId).get().getInstanceEvents().size()); - - saveJobInstanceEvent(event1, updateId); - assertEquals( - populateExpected(update, ROLLING_FORWARD, CREATED_MS, 125L), - getUpdateDetails(updateId).get().getUpdate()); - assertEquals( - event1, - Iterables.getOnlyElement(getUpdateDetails(updateId).get().getInstanceEvents())); - assertEquals(1L, stats.getLongValue(jobUpdateActionStatName(INSTANCE_UPDATED))); - - saveJobInstanceEvent(event2, updateId); - assertEquals( - populateExpected(update, ROLLING_FORWARD, CREATED_MS, 126L), - getUpdateDetails(updateId).get().getUpdate()); - assertEquals(event1, getUpdateDetails(updateId).get().getInstanceEvents().get(0)); - assertEquals(event2, getUpdateDetails(updateId).get().getInstanceEvents().get(1)); - assertEquals(1L, stats.getLongValue(jobUpdateActionStatName(INSTANCE_ROLLING_BACK))); - } - - @Test(expected = StorageException.class) - public void testSaveJobEventWithoutUpdateFails() { - saveJobEvent(makeJobUpdateEvent(ROLLING_FORWARD, 123L), makeKey("u2")); - } - - @Test(expected = StorageException.class) - public void testSaveInstanceEventWithoutUpdateFails() { - saveJobInstanceEvent(makeJobInstanceEvent(0, 125L, INSTANCE_UPDATED), makeKey("u1")); - } - @Test public void testSaveJobUpdateStateIgnored() { IJobUpdateKey updateId = makeKey("u1"); - IJobUpdate update = populateExpected(makeJobUpdate(updateId), ABORTED, 567L, 567L); + IJobUpdateDetails update = withUpdateState(makeJobUpdate(updateId), ABORTED, 567L, 567L); saveUpdate(update); // Assert state fields were ignored. - assertUpdate(update); + assertUpdate(withDefaultUpdateState(update)); } @Test public void testMultipleJobDetails() { IJobUpdateKey updateId1 = makeKey(JobKeys.from("role", "env", "name1"), "u1"); IJobUpdateKey updateId2 = makeKey(JobKeys.from("role", "env", "name2"), "u2"); - IJobUpdateDetails details1 = makeJobDetails(makeJobUpdate(updateId1)); - IJobUpdateDetails details2 = makeJobDetails(makeJobUpdate(updateId2)); - - assertEquals(ImmutableList.of(), getInstanceEvents(updateId2, 3)); + IJobUpdateDetails update1 = makeJobUpdate(updateId1); + IJobUpdateDetails update2 = makeJobUpdate(updateId2); - saveUpdate(details1.getUpdate()); - saveUpdate(details2.getUpdate()); + saveUpdate(update1); + saveUpdate(update2); - details1 = updateJobDetails(populateExpected(details1.getUpdate()), FIRST_EVENT); - details2 = updateJobDetails(populateExpected(details2.getUpdate()), FIRST_EVENT); - assertEquals(Optional.of(details1), getUpdateDetails(updateId1)); - assertEquals(Optional.of(details2), getUpdateDetails(updateId2)); + update1 = updateJobDetails(withDefaultUpdateState(update1), FIRST_EVENT); + update2 = updateJobDetails(withDefaultUpdateState(update2), FIRST_EVENT); + assertEquals(Optional.of(update1), getUpdateDetails(updateId1)); + assertEquals(Optional.of(update2), getUpdateDetails(updateId2)); IJobUpdateEvent jEvent11 = makeJobUpdateEvent(ROLLING_FORWARD, 456L); IJobUpdateEvent jEvent12 = makeJobUpdateEvent(ERROR, 457L); @@ -413,168 +310,66 @@ public abstract class AbstractJobUpdateStoreTest { saveJobEvent(jEvent11, updateId1); saveJobEvent(jEvent12, updateId1); saveJobInstanceEvent(iEvent11, updateId1); - assertEquals(1L, stats.getLongValue(jobUpdateActionStatName(INSTANCE_UPDATED))); saveJobInstanceEvent(iEvent12, updateId1); - assertEquals(1L, stats.getLongValue(jobUpdateActionStatName(INSTANCE_UPDATING))); saveJobEvent(jEvent21, updateId2); saveJobEvent(jEvent22, updateId2); assertEquals(ImmutableList.of(), getInstanceEvents(updateId2, 3)); saveJobInstanceEvent(iEvent21, updateId2); - assertEquals(2L, stats.getLongValue(jobUpdateActionStatName(INSTANCE_UPDATING))); assertEquals(ImmutableList.of(iEvent21), getInstanceEvents(updateId2, 3)); saveJobInstanceEvent(iEvent22, updateId2); assertEquals(ImmutableList.of(iEvent21, iEvent22), getInstanceEvents(updateId2, 3)); - assertEquals(2L, stats.getLongValue(jobUpdateActionStatName(INSTANCE_UPDATING))); - details1 = updateJobDetails( - populateExpected(details1.getUpdate(), ERROR, CREATED_MS, 457L), + update1 = updateJobDetails( + withUpdateState(update1, ERROR, CREATED_MS, 457L), ImmutableList.of(FIRST_EVENT, jEvent11, jEvent12), ImmutableList.of(iEvent11, iEvent12)); - details2 = updateJobDetails( - populateExpected(details2.getUpdate(), ABORTED, CREATED_MS, 568L), + update2 = updateJobDetails( + withUpdateState(update2, ABORTED, CREATED_MS, 568L), ImmutableList.of(FIRST_EVENT, jEvent21, jEvent22), ImmutableList.of(iEvent21, iEvent22)); - assertEquals(Optional.of(details1), getUpdateDetails(updateId1)); - assertEquals(Optional.of(details2), getUpdateDetails(updateId2)); - - assertEquals(ImmutableSet.of(details1, details2), getAllUpdateDetails()); + assertEquals( + ImmutableList.of(update2, update1), + fetchUpdates(JobUpdateStore.MATCH_ALL.newBuilder())); assertEquals( ImmutableList.of(getUpdateDetails(updateId2).get(), getUpdateDetails(updateId1).get()), - queryDetails(new JobUpdateQuery().setRole("role"))); + fetchUpdates(new JobUpdateQuery().setRole("role"))); } @Test public void testTruncateJobUpdates() { - IJobUpdateKey updateId = makeKey("u5"); - IJobUpdate update = makeJobUpdate(updateId); - IJobInstanceUpdateEvent instanceEvent = IJobInstanceUpdateEvent.build( - new JobInstanceUpdateEvent(0, 125L, INSTANCE_ROLLBACK_FAILED)); - - saveUpdate(update); - saveJobEvent(makeJobUpdateEvent(ROLLING_FORWARD, 123L), updateId); - saveJobInstanceEvent(instanceEvent, updateId); - assertEquals(1L, stats.getLongValue(jobUpdateActionStatName(INSTANCE_ROLLBACK_FAILED))); - assertEquals( - populateExpected(update, ROLLING_FORWARD, CREATED_MS, 125L), - getUpdate(updateId).get()); - assertEquals(2, getUpdateDetails(updateId).get().getUpdateEvents().size()); - assertEquals(1, getUpdateDetails(updateId).get().getInstanceEvents().size()); - + saveUpdate(makeJobUpdate(makeKey("u1"))); + saveUpdate(makeJobUpdate(makeKey("u2"))); + assertEquals(2, fetchUpdates(new JobUpdateQuery()).size()); + truncateUpdates(); + assertEquals(0, fetchUpdates(new JobUpdateQuery()).size()); truncateUpdates(); - assertEquals(Optional.absent(), getUpdateDetails(updateId)); + assertEquals(0, fetchUpdates(new JobUpdateQuery()).size()); } @Test - public void testPruneHistory() { - IJobUpdateKey updateId1 = makeKey("u11"); - IJobUpdateKey updateId2 = makeKey("u12"); - IJobUpdateKey updateId3 = makeKey("u13"); - IJobUpdateKey updateId4 = makeKey("u14"); - IJobKey job2 = JobKeys.from("testRole2", "testEnv2", "job2"); - IJobUpdateKey updateId5 = makeKey(job2, "u15"); - IJobUpdateKey updateId6 = makeKey(job2, "u16"); - IJobUpdateKey updateId7 = makeKey(job2, "u17"); - - IJobUpdate update1 = makeJobUpdate(updateId1); - IJobUpdate update2 = makeJobUpdate(updateId2); - IJobUpdate update3 = makeJobUpdate(updateId3); - IJobUpdate update4 = makeJobUpdate(updateId4); - IJobUpdate update5 = makeJobUpdate(updateId5); - IJobUpdate update6 = makeJobUpdate(updateId6); - IJobUpdate update7 = makeJobUpdate(updateId7); - - IJobUpdateEvent updateEvent1 = makeJobUpdateEvent(ROLLING_BACK, 123L); - IJobUpdateEvent updateEvent2 = makeJobUpdateEvent(ABORTED, 124L); - IJobUpdateEvent updateEvent3 = makeJobUpdateEvent(ROLLED_BACK, 125L); - IJobUpdateEvent updateEvent4 = makeJobUpdateEvent(FAILED, 126L); - IJobUpdateEvent updateEvent5 = makeJobUpdateEvent(ERROR, 123L); - IJobUpdateEvent updateEvent6 = makeJobUpdateEvent(FAILED, 125L); - IJobUpdateEvent updateEvent7 = makeJobUpdateEvent(ROLLING_FORWARD, 126L); - - update1 = populateExpected(saveUpdateNoEvent(update1), ROLLING_BACK, 123L, 123L); - update2 = populateExpected(saveUpdateNoEvent(update2), ABORTED, 124L, 124L); - update3 = populateExpected(saveUpdateNoEvent(update3), ROLLED_BACK, 125L, 125L); - update4 = populateExpected(saveUpdateNoEvent(update4), FAILED, 126L, 126L); - update5 = populateExpected(saveUpdateNoEvent(update5), ERROR, 123L, 123L); - update6 = populateExpected(saveUpdateNoEvent(update6), FAILED, 125L, 125L); - update7 = populateExpected(saveUpdateNoEvent(update7), ROLLING_FORWARD, 126L, 126L); - - saveJobEvent(updateEvent1, updateId1); - saveJobEvent(updateEvent2, updateId2); - saveJobEvent(updateEvent3, updateId3); - saveJobEvent(updateEvent4, updateId4); - saveJobEvent(updateEvent5, updateId5); - saveJobEvent(updateEvent6, updateId6); - saveJobEvent(updateEvent7, updateId7); - - assertEquals(update1, getUpdate(updateId1).get()); - assertEquals(update2, getUpdate(updateId2).get()); - assertEquals(update3, getUpdate(updateId3).get()); - assertEquals(update4, getUpdate(updateId4).get()); - assertEquals(update5, getUpdate(updateId5).get()); - assertEquals(update6, getUpdate(updateId6).get()); - assertEquals(update7, getUpdate(updateId7).get()); - - long pruningThreshold = 120L; - - // No updates pruned. - assertEquals(ImmutableSet.of(), pruneHistory(3, pruningThreshold)); - assertEquals(Optional.of(update7), getUpdate(updateId7)); // active update - assertEquals(Optional.of(update6), getUpdate(updateId6)); - assertEquals(Optional.of(update5), getUpdate(updateId5)); - - assertEquals(Optional.of(update4), getUpdate(updateId4)); - assertEquals(Optional.of(update3), getUpdate(updateId3)); - assertEquals(Optional.of(update2), getUpdate(updateId2)); - assertEquals(Optional.of(update1), getUpdate(updateId1)); // active update - - assertEquals(ImmutableSet.of(updateId2), pruneHistory(2, pruningThreshold)); - // No updates pruned. - assertEquals(Optional.of(update7), getUpdate(updateId7)); // active update - assertEquals(Optional.of(update6), getUpdate(updateId6)); - assertEquals(Optional.of(update5), getUpdate(updateId5)); - - // 1 update pruned. - assertEquals(Optional.of(update4), getUpdate(updateId4)); - assertEquals(Optional.of(update3), getUpdate(updateId3)); - assertEquals(Optional.absent(), getUpdate(updateId2)); - assertEquals(Optional.of(update1), getUpdate(updateId1)); // active update - - assertEquals(ImmutableSet.of(updateId5, updateId3), pruneHistory(1, pruningThreshold)); - // 1 update pruned. - assertEquals(Optional.of(update7), getUpdate(updateId7)); // active update - assertEquals(Optional.of(update6), getUpdate(updateId6)); - assertEquals(Optional.absent(), getUpdate(updateId5)); - - // 2 updates pruned. - assertEquals(Optional.of(update4), getUpdate(updateId4)); - assertEquals(Optional.absent(), getUpdate(updateId3)); - assertEquals(Optional.of(update1), getUpdate(updateId1)); // active update - - // The oldest update is pruned. - assertEquals(ImmutableSet.of(updateId6), pruneHistory(1, 126L)); - assertEquals(Optional.of(update7), getUpdate(updateId7)); // active update - assertEquals(Optional.absent(), getUpdate(updateId6)); + public void testRemoveUpdates() { + IJobUpdateDetails u1 = makeJobUpdate(makeKey("u1")); + IJobUpdateDetails u2 = makeJobUpdate(makeKey("u2")); + IJobUpdateDetails u3 = makeJobUpdate(makeKey("u3")); + saveUpdate(u1); + saveUpdate(u2); + saveUpdate(u3); - assertEquals(Optional.of(update4), getUpdate(updateId4)); - assertEquals(Optional.of(update1), getUpdate(updateId1)); // active update + storage.write((NoResult.Quiet) store -> + store.getJobUpdateStore().removeJobUpdates(ImmutableSet.of(key(u1), key(u3)))); + assertQueryMatches(new JobUpdateQuery(), withDefaultUpdateState(u2)); - // Nothing survives the 0 per job count. - assertEquals(ImmutableSet.of(updateId4), pruneHistory(0, pruningThreshold)); - assertEquals(Optional.of(update7), getUpdate(updateId7)); // active update + // Noop - update already deleted + storage.write((NoResult.Quiet) store -> + store.getJobUpdateStore().removeJobUpdates(ImmutableSet.of(key(u1)))); + assertQueryMatches(new JobUpdateQuery(), withDefaultUpdateState(u2)); - assertEquals(Optional.absent(), getUpdate(updateId4)); - assertEquals(Optional.of(update1), getUpdate(updateId1)); // active update - } - - @Test(expected = StorageException.class) - public void testSaveTwoUpdatesForOneJob() { - IJobUpdate update = makeJobUpdate(makeKey("updateId")); - saveUpdate(update); - saveUpdate(update); + storage.write((NoResult.Quiet) store -> + store.getJobUpdateStore().removeJobUpdates(ImmutableSet.of(key(u2)))); + assertQueryMatches(new JobUpdateQuery()); } @Test @@ -583,114 +378,31 @@ public abstract class AbstractJobUpdateStoreTest { ImmutableSet<Metadata> duplicatedMetadata = ImmutableSet.of(new Metadata("k1", "v1"), new Metadata("k1", "v2")); - JobUpdate builder = makeJobUpdate(updateId).newBuilder(); - builder.getSummary().setMetadata(duplicatedMetadata); - IJobUpdate update = IJobUpdate.build(builder); + JobUpdateDetails builder = makeJobUpdate(updateId).newBuilder(); + builder.getUpdate().getSummary().setMetadata(duplicatedMetadata); assertEquals(Optional.absent(), getUpdate(updateId)); + IJobUpdateDetails update = IJobUpdateDetails.build(builder); saveUpdate(update); assertUpdate(update); } @Test - public void testGetSummaries() { - String role1 = "role1"; - IJobKey job1 = JobKeys.from(role1, "env", "name1"); - IJobKey job2 = JobKeys.from(role1, "env", "name2"); - IJobKey job3 = JobKeys.from(role1, "env", "name3"); - IJobKey job4 = JobKeys.from(role1, "env", "name4"); - IJobKey job5 = JobKeys.from("role", "env", "name5"); - IJobUpdateSummary s1 = saveSummary(makeKey(job1, "u1"), 1230L, ROLLED_BACK, "user"); - IJobUpdateSummary s2 = saveSummary(makeKey(job2, "u2"), 1231L, ABORTED, "user"); - IJobUpdateSummary s3 = saveSummary(makeKey(job3, "u3"), 1239L, ERROR, "user2"); - IJobUpdateSummary s4 = saveSummary(makeKey(job4, "u4"), 1234L, ROLL_BACK_PAUSED, "user3"); - IJobUpdateSummary s5 = saveSummary(makeKey(job5, "u5"), 1235L, ROLLING_FORWARD, "user4"); - - // Test empty query returns all. - assertEquals(ImmutableList.of(s3, s5, s4, s2, s1), getSummaries(new JobUpdateQuery())); - - // Test query by updateId. - assertEquals( - ImmutableList.of(s1), - getSummaries(new JobUpdateQuery().setKey(new JobUpdateKey(job1.newBuilder(), "u1")))); - - // Test query by role. - assertEquals( - ImmutableList.of(s3, s4, s2, s1), - getSummaries(new JobUpdateQuery().setRole(role1))); - - // Test query by job key. - assertEquals( - ImmutableList.of(s5), - getSummaries(new JobUpdateQuery().setJobKey(job5.newBuilder()))); - - // Test querying by update key. - assertEquals( - ImmutableList.of(s5), - getSummaries( - new JobUpdateQuery().setKey(new JobUpdateKey(job5.newBuilder(), s5.getKey().getId())))); - - // Test querying by incorrect update keys. - assertEquals( - ImmutableList.of(), - getSummaries( - new JobUpdateQuery().setKey(new JobUpdateKey(job5.newBuilder(), s4.getKey().getId())))); - assertEquals( - ImmutableList.of(), - getSummaries( - new JobUpdateQuery().setKey(new JobUpdateKey(job4.newBuilder(), s5.getKey().getId())))); - - // Test query by user. - assertEquals(ImmutableList.of(s2, s1), getSummaries(new JobUpdateQuery().setUser("user"))); - - // Test query by one status. - assertEquals(ImmutableList.of(s3), getSummaries(new JobUpdateQuery().setUpdateStatuses( - ImmutableSet.of(ERROR)))); - - // Test query by multiple statuses. - assertEquals(ImmutableList.of(s3, s2, s1), getSummaries(new JobUpdateQuery().setUpdateStatuses( - ImmutableSet.of(ERROR, ABORTED, ROLLED_BACK)))); - - // Test query by empty statuses. - assertEquals( - ImmutableList.of(s3, s5, s4, s2, s1), - getSummaries(new JobUpdateQuery().setUpdateStatuses(ImmutableSet.of()))); - - // Test paging. - assertEquals( - ImmutableList.of(s3, s5), - getSummaries(new JobUpdateQuery().setLimit(2).setOffset(0))); - assertEquals( - ImmutableList.of(s4, s2), - getSummaries(new JobUpdateQuery().setLimit(2).setOffset(2))); - assertEquals( - ImmutableList.of(s1), - getSummaries(new JobUpdateQuery().setLimit(2).setOffset(4))); - - // Test no match. - assertEquals( - ImmutableList.of(), - getSummaries(new JobUpdateQuery().setRole("no_match"))); - } - - @Test public void testQueryDetails() { IJobKey jobKey1 = JobKeys.from("role1", "env", "name1"); IJobUpdateKey updateId1 = makeKey(jobKey1, "u1"); IJobKey jobKey2 = JobKeys.from("role2", "env", "name2"); IJobUpdateKey updateId2 = makeKey(jobKey2, "u2"); - IJobUpdate update1 = makeJobUpdate(updateId1); - IJobUpdate update2 = makeJobUpdate(updateId2); - - assertEquals(ImmutableList.of(), getInstanceEvents(updateId2, 3)); + IJobUpdateDetails update1 = makeJobUpdate(updateId1); + IJobUpdateDetails update2 = makeJobUpdate(updateId2); saveUpdate(update1); saveUpdate(update2); - updateJobDetails(populateExpected(update1), FIRST_EVENT); - updateJobDetails(populateExpected(update2), FIRST_EVENT); + updateJobDetails(withDefaultUpdateState(update1), FIRST_EVENT); + updateJobDetails(withDefaultUpdateState(update2), FIRST_EVENT); IJobUpdateEvent jEvent11 = makeJobUpdateEvent(ROLLING_BACK, 450L); IJobUpdateEvent jEvent12 = makeJobUpdateEvent(ROLLED_BACK, 500L); @@ -713,57 +425,60 @@ public abstract class AbstractJobUpdateStoreTest { saveJobInstanceEvent(iEvent21, updateId2); saveJobInstanceEvent(iEvent22, updateId2); - IJobUpdateDetails details1 = getUpdateDetails(updateId1).get(); - IJobUpdateDetails details2 = getUpdateDetails(updateId2).get(); + // Fetch the updates for checking query results. This avoids the need to manually populate + // any fields filled by the storage, which is out of scope for this test case. + update1 = getUpdateDetails(updateId1).get(); + update2 = getUpdateDetails(updateId2).get(); - // Test empty query returns all. - assertEquals(ImmutableList.of(details2, details1), queryDetails(new JobUpdateQuery())); + // Empty query returns all. + assertQueryMatches(new JobUpdateQuery(), update2, update1); - // Test query by update ID. - assertEquals( - ImmutableList.of(details1), - queryDetails(new JobUpdateQuery().setKey(updateId1.newBuilder()))); + // Query by update ID. + assertQueryMatches(new JobUpdateQuery().setKey(updateId1.newBuilder()), update1); - // Test query by role. - assertEquals( - ImmutableList.of(details2), - queryDetails(new JobUpdateQuery().setRole(jobKey2.getRole()))); + // Query by role. + assertQueryMatches(new JobUpdateQuery().setRole(jobKey2.getRole()), update2); - // Test query by job key. - assertEquals( - ImmutableList.of(details2), - queryDetails(new JobUpdateQuery().setJobKey(jobKey2.newBuilder()))); + // Query by job key. + assertQueryMatches(new JobUpdateQuery().setJobKey(jobKey2.newBuilder()), update2); - // Test query by status. - assertEquals( - ImmutableList.of(details2), - queryDetails(new JobUpdateQuery().setUpdateStatuses(ImmutableSet.of(ABORTED)))); + // Query by status. + assertQueryMatches(new JobUpdateQuery().setUpdateStatuses(ImmutableSet.of(ABORTED)), update2); - // Test no match. - assertEquals( - ImmutableList.of(), - queryDetails(new JobUpdateQuery().setRole("no match"))); - } + // No match. + assertQueryMatches(new JobUpdateQuery().setRole("no match")); - @Test - public void testSaveEventsOutOfChronologicalOrder() { - IJobUpdate update1 = makeJobUpdate(UPDATE1); - saveUpdate(update1); + // Querying by incorrect update keys. + assertQueryMatches(new JobUpdateQuery().setJobKey(JobKeys.from("a", "b", "c").newBuilder())); + + // Query by multiple statuses. + assertQueryMatches( + new JobUpdateQuery().setUpdateStatuses(ImmutableSet.of(ABORTED, ROLLING_FORWARD)), + update2); + + // Query by empty statuses. + assertQueryMatches(new JobUpdateQuery().setUpdateStatuses(ImmutableSet.of()), update2, update1); - IJobUpdateEvent event2 = makeJobUpdateEvent(ROLLING_FORWARD, 124); - IJobUpdateEvent event1 = makeJobUpdateEvent(ROLL_FORWARD_PAUSED, 122); - saveJobEvent(event2, UPDATE1); - saveJobEvent(event1, UPDATE1); + // Query by user. + assertQueryMatches(new JobUpdateQuery().setUser("user"), update2, update1); - IJobInstanceUpdateEvent instanceEvent2 = makeJobInstanceEvent(0, 125, INSTANCE_UPDATED); - IJobInstanceUpdateEvent instanceEvent1 = makeJobInstanceEvent(0, 124, INSTANCE_UPDATING); + // Test paging. + assertQueryMatches(new JobUpdateQuery().setLimit(1).setOffset(0), update2); + assertQueryMatches(new JobUpdateQuery().setLimit(2).setOffset(0), update2, update1); + assertQueryMatches(new JobUpdateQuery().setLimit(3).setOffset(0), update2, update1); + assertQueryMatches(new JobUpdateQuery().setLimit(1).setOffset(1), update1); + assertQueryMatches(new JobUpdateQuery().setLimit(2).setOffset(4)); + } - saveJobInstanceEvent(instanceEvent2, UPDATE1); - saveJobInstanceEvent(instanceEvent1, UPDATE1); + private static IJobUpdateKey key(IJobUpdateDetails update) { + return update.getUpdate().getSummary().getKey(); + } - IJobUpdateState state = getUpdateDetails(UPDATE1).get().getUpdate().getSummary().getState(); - assertEquals(ROLLING_FORWARD, state.getStatus()); - assertEquals(125, state.getLastModifiedTimestampMs()); + private void assertQueryMatches(JobUpdateQuery query, IJobUpdateDetails... matches) { + assertEquals( + ImmutableList.copyOf(matches), + storage.read(store -> + store.getJobUpdateStore().fetchJobUpdates(IJobUpdateQuery.build(query)))); } private static IJobUpdateKey makeKey(String id) { @@ -774,63 +489,43 @@ public abstract class AbstractJobUpdateStoreTest { return IJobUpdateKey.build(new JobUpdateKey(job.newBuilder(), id)); } - private void assertUpdate(IJobUpdate expected) { - IJobUpdateKey key = expected.getSummary().getKey(); - assertEquals(populateExpected(expected), getUpdate(key).get()); - assertEquals(getUpdate(key).get(), getUpdateDetails(key).get().getUpdate()); - assertEquals(getUpdateInstructions(key).get(), expected.getInstructions()); + private void assertUpdate(IJobUpdateDetails expected) { + assertEquals(withDefaultUpdateState(expected), getUpdate(key(expected)).get()); } - private Optional<IJobUpdate> getUpdate(IJobUpdateKey key) { + private Optional<IJobUpdateDetails> getUpdate(IJobUpdateKey key) { return storage.read(storeProvider -> storeProvider.getJobUpdateStore().fetchJobUpdate(key)); } private List<IJobInstanceUpdateEvent> getInstanceEvents(IJobUpdateKey key, int id) { - return storage.read( - storeProvider -> storeProvider.getJobUpdateStore().fetchInstanceEvents(key, id)); - } - - private Optional<IJobUpdateInstructions> getUpdateInstructions(IJobUpdateKey key) { - return storage.read( - storeProvider -> storeProvider.getJobUpdateStore().fetchJobUpdateInstructions(key)); + IJobUpdateDetails update = + storage.read(store -> store.getJobUpdateStore().fetchJobUpdate(key).get()); + return update.getInstanceEvents().stream() + .filter(e -> e.getInstanceId() == id) + .collect(Collectors.toList()); } private Optional<IJobUpdateDetails> getUpdateDetails(IJobUpdateKey key) { - return storage.read( - storeProvider -> storeProvider.getJobUpdateStore().fetchJobUpdateDetails(key)); - } - - private Set<IJobUpdateDetails> getAllUpdateDetails() { - return storage.read( - storeProvider -> storeProvider.getJobUpdateStore().fetchAllJobUpdateDetails()); - } - - private List<IJobUpdateDetails> queryDetails(JobUpdateQuery query) { - return storage.read(storeProvider -> storeProvider.getJobUpdateStore().fetchJobUpdateDetails( - IJobUpdateQuery.build(query))); + return storage.read(storeProvider -> storeProvider.getJobUpdateStore().fetchJobUpdate(key)); } - private List<IJobUpdateSummary> getSummaries(JobUpdateQuery query) { - return storage.read(storeProvider -> storeProvider.getJobUpdateStore().fetchJobUpdateSummaries( - IJobUpdateQuery.build(query))); + private List<IJobUpdateDetails> fetchUpdates(JobUpdateQuery query) { + return storage.read(storeProvider -> + storeProvider.getJobUpdateStore().fetchJobUpdates(IJobUpdateQuery.build(query))); } - private IJobUpdate saveUpdate(IJobUpdate update) { + private void saveUpdate(IJobUpdateDetails update) { storage.write((NoResult.Quiet) storeProvider -> { - storeProvider.getJobUpdateStore().saveJobUpdate(update); - storeProvider.getJobUpdateStore().saveJobUpdateEvent( - update.getSummary().getKey(), - FIRST_EVENT); + JobUpdateStore.Mutable store = storeProvider.getJobUpdateStore(); + store.saveJobUpdate(update.getUpdate()); + IJobUpdateKey key = update.getUpdate().getSummary().getKey(); + update.getUpdateEvents().forEach(event -> { + store.saveJobUpdateEvent(key, event); + }); + update.getInstanceEvents().forEach(event -> { + store.saveJobInstanceUpdateEvent(key, event); + }); }); - - return update; - } - - private IJobUpdate saveUpdateNoEvent(IJobUpdate update) { - storage.write((NoResult.Quiet) storeProvider -> - storeProvider.getJobUpdateStore().saveJobUpdate(update)); - - return update; } private void saveJobEvent(IJobUpdateEvent event, IJobUpdateKey key) { @@ -845,20 +540,15 @@ public abstract class AbstractJobUpdateStoreTest { private void truncateUpdates() { storage.write((NoResult.Quiet) - storeProvider -> storeProvider.getJobUpdateStore().deleteAllUpdatesAndEvents()); - } - - private Set<IJobUpdateKey> pruneHistory(int retainCount, long pruningThresholdMs) { - return storage.write(storeProvider -> - storeProvider.getJobUpdateStore().pruneHistory(retainCount, pruningThresholdMs)); + storeProvider -> storeProvider.getJobUpdateStore().deleteAllUpdates()); } - private IJobUpdate populateExpected(IJobUpdate update) { - return populateExpected(update, ROLLING_FORWARD, CREATED_MS, CREATED_MS); + private IJobUpdateDetails withDefaultUpdateState(IJobUpdateDetails update) { + return withUpdateState(update, ROLLING_FORWARD, CREATED_MS, CREATED_MS); } - private IJobUpdate populateExpected( - IJobUpdate update, + private IJobUpdateDetails withUpdateState( + IJobUpdateDetails update, JobUpdateStatus status, long createdMs, long lastMs) { @@ -867,9 +557,9 @@ public abstract class AbstractJobUpdateStoreTest { .setCreatedTimestampMs(createdMs) .setLastModifiedTimestampMs(lastMs) .setStatus(status); - JobUpdate builder = update.newBuilder(); - builder.getSummary().setState(state); - return IJobUpdate.build(builder); + JobUpdateDetails builder = update.newBuilder(); + builder.getUpdate().getSummary().setState(state); + return IJobUpdateDetails.build(builder); } private static IJobUpdateEvent makeJobUpdateEvent(JobUpdateStatus status, long timestampMs) { @@ -888,14 +578,7 @@ public abstract class AbstractJobUpdateStoreTest { new JobInstanceUpdateEvent(instanceId, timestampMs, action)); } - private IJobUpdateDetails makeJobDetails(IJobUpdate update) { - return updateJobDetails( - update, - ImmutableList.of(FIRST_EVENT), - ImmutableList.of()); - } - - private IJobUpdateDetails updateJobDetails(IJobUpdate update, IJobUpdateEvent event) { + private IJobUpdateDetails updateJobDetails(IJobUpdateDetails update, IJobUpdateEvent event) { return updateJobDetails( update, ImmutableList.of(event), @@ -903,12 +586,11 @@ public abstract class AbstractJobUpdateStoreTest { } private IJobUpdateDetails updateJobDetails( - IJobUpdate update, + IJobUpdateDetails update, List<IJobUpdateEvent> jobEvents, List<IJobInstanceUpdateEvent> instanceEvents) { - return IJobUpdateDetails.build(new JobUpdateDetails() - .setUpdate(update.newBuilder()) + return IJobUpdateDetails.build(update.newBuilder() .setUpdateEvents(IJobUpdateEvent.toBuildersList(jobEvents)) .setInstanceEvents(IJobInstanceUpdateEvent.toBuildersList(instanceEvents))); } @@ -920,35 +602,12 @@ public abstract class AbstractJobUpdateStoreTest { .setMetadata(METADATA)); } - private IJobUpdateSummary saveSummary( - IJobUpdateKey key, - Long modifiedTimestampMs, - JobUpdateStatus status, - String user) { - - IJobUpdateSummary summary = IJobUpdateSummary.build(new JobUpdateSummary() - .setKey(key.newBuilder()) - .setUser(user) - .setMetadata(METADATA)); - - IJobUpdate update = makeJobUpdate(summary); - saveUpdate(update); - saveJobEvent(makeJobUpdateEvent(status, modifiedTimestampMs), key); - return populateExpected(update, status, CREATED_MS, modifiedTimestampMs).getSummary(); - } - - private IJobUpdate makeJobUpdate(IJobUpdateSummary summary) { - return IJobUpdate.build(makeJobUpdate().newBuilder().setSummary(summary.newBuilder())); - } - - private static IJobUpdate makeJobUpdate(IJobUpdateKey key) { - return IJobUpdate.build(makeJobUpdate().newBuilder() - .setSummary(makeSummary(key, "user").newBuilder())); - } - - private static IJobUpdate makeJobUpdate() { - return IJobUpdate.build(new JobUpdate() - .setInstructions(makeJobUpdateInstructions().newBuilder())); + private static IJobUpdateDetails makeJobUpdate(IJobUpdateKey key) { + return IJobUpdateDetails.build(new JobUpdateDetails() + .setUpdateEvents(ImmutableList.of(FIRST_EVENT.newBuilder())) + .setUpdate(new JobUpdate() + .setInstructions(makeJobUpdateInstructions().newBuilder()) + .setSummary(makeSummary(key, "user").newBuilder()))); } private static IJobUpdateInstructions makeJobUpdateInstructions() { http://git-wip-us.apache.org/repos/asf/aurora/blob/284f40f5/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java b/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java index c208210..3c056c9 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java @@ -59,6 +59,7 @@ import org.apache.aurora.gen.storage.LogEntry; import org.apache.aurora.gen.storage.Op; import org.apache.aurora.gen.storage.PruneJobUpdateHistory; import org.apache.aurora.gen.storage.RemoveJob; +import org.apache.aurora.gen.storage.RemoveJobUpdates; import org.apache.aurora.gen.storage.RemoveQuota; import org.apache.aurora.gen.storage.RemoveTasks; import org.apache.aurora.gen.storage.SaveCronJob; @@ -126,7 +127,7 @@ public class LogStorageTest extends EasyMockTest { private static final IJobKey JOB_KEY = JobKeys.from("role", "env", "name"); private static final IJobUpdateKey UPDATE_ID = IJobUpdateKey.build(new JobUpdateKey(JOB_KEY.newBuilder(), "testUpdateId")); - private static final long NOW = 42L; + private static final long NOW = 42; private LogStorage logStorage; private Log log; @@ -350,8 +351,11 @@ public class LogStorageTest extends EasyMockTest { IJobInstanceUpdateEvent.build(saveInstanceEvent.getEvent())); builder.add(createTransaction(Op.pruneJobUpdateHistory(new PruneJobUpdateHistory(5, 10L)))); - expect(storageUtil.jobUpdateStore.pruneHistory(5, 10L)) - .andReturn(ImmutableSet.of(IJobUpdateKey.build(UPDATE_ID.newBuilder()))); + // No expectation - this op is ignored. + + builder.add(createTransaction(Op.removeJobUpdate( + new RemoveJobUpdates().setKeys(ImmutableSet.of(UPDATE_ID.newBuilder()))))); + storageUtil.jobUpdateStore.removeJobUpdates(ImmutableSet.of(UPDATE_ID)); // NOOP LogEntry builder.add(LogEntry.noop(true)); @@ -856,28 +860,24 @@ public class LogStorageTest extends EasyMockTest { } @Test - public void testPruneHistory() throws Exception { - PruneJobUpdateHistory pruneHistory = new PruneJobUpdateHistory() - .setHistoryPruneThresholdMs(1L) - .setPerJobRetainCount(1); + public void testRemoveJobUpdates() throws Exception { + IJobUpdateKey key = IJobUpdateKey.build(new JobUpdateKey() + .setJob(JOB_KEY.newBuilder()) + .setId("update-id")); new AbstractMutationFixture() { @Override protected void setupExpectations() throws Exception { storageUtil.expectWrite(); - expect(storageUtil.jobUpdateStore.pruneHistory( - pruneHistory.getPerJobRetainCount(), - pruneHistory.getHistoryPruneThresholdMs())) - .andReturn(ImmutableSet.of(UPDATE_ID)); + storageUtil.jobUpdateStore.removeJobUpdates(ImmutableSet.of(key)); - streamMatcher.expectTransaction(Op.pruneJobUpdateHistory(pruneHistory)).andReturn(position); + // No log transaction is generated since this version is currently in 'read-only' + // compatibility mode for this operation type. } @Override protected void performMutations(MutableStoreProvider storeProvider) { - storeProvider.getJobUpdateStore().pruneHistory( - pruneHistory.getPerJobRetainCount(), - pruneHistory.getHistoryPruneThresholdMs()); + storeProvider.getJobUpdateStore().removeJobUpdates(ImmutableSet.of(key)); } }.run(); } http://git-wip-us.apache.org/repos/asf/aurora/blob/284f40f5/src/test/java/org/apache/aurora/scheduler/storage/log/WriteAheadStorageTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/log/WriteAheadStorageTest.java b/src/test/java/org/apache/aurora/scheduler/storage/log/WriteAheadStorageTest.java index d5e5c11..8a99b36 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/log/WriteAheadStorageTest.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/log/WriteAheadStorageTest.java @@ -25,10 +25,8 @@ import org.apache.aurora.gen.HostAttributes; import org.apache.aurora.gen.JobUpdateKey; import org.apache.aurora.gen.MaintenanceMode; import org.apache.aurora.gen.storage.Op; -import org.apache.aurora.gen.storage.PruneJobUpdateHistory; import org.apache.aurora.gen.storage.SaveHostAttributes; import org.apache.aurora.gen.storage.SaveTasks; -import org.apache.aurora.scheduler.base.JobKeys; import org.apache.aurora.scheduler.base.TaskTestUtil; import org.apache.aurora.scheduler.events.EventSink; import org.apache.aurora.scheduler.events.PubsubEvent; @@ -85,26 +83,16 @@ public class WriteAheadStorageTest extends EasyMockTest { } @Test - public void testPruneHistory() { - Set<IJobUpdateKey> pruned = ImmutableSet.of( - IJobUpdateKey.build(new JobUpdateKey(JobKeys.from("role", "env", "job").newBuilder(), "a")), - IJobUpdateKey.build( - new JobUpdateKey(JobKeys.from("role", "env", "job").newBuilder(), "b"))); - expect(jobUpdateStore.pruneHistory(1, 1)).andReturn(pruned); - expectOp(Op.pruneJobUpdateHistory(new PruneJobUpdateHistory(1, 1))); + public void testRemoveUpdates() { + Set<IJobUpdateKey> removed = ImmutableSet.of( + IJobUpdateKey.build(new JobUpdateKey(TaskTestUtil.JOB.newBuilder(), "a")), + IJobUpdateKey.build(new JobUpdateKey(TaskTestUtil.JOB.newBuilder(), "b"))); + jobUpdateStore.removeJobUpdates(removed); + // No operation is written since this Op is in read-only compatibility mode. control.replay(); - storage.pruneHistory(1, 1); - } - - @Test - public void testNoopPruneHistory() { - expect(jobUpdateStore.pruneHistory(1, 1)).andReturn(ImmutableSet.of()); - - control.replay(); - - storage.pruneHistory(1, 1); + storage.removeJobUpdates(removed); } @Test @@ -172,6 +160,6 @@ public class WriteAheadStorageTest extends EasyMockTest { @Test(expected = UnsupportedOperationException.class) public void testDeleteAllUpdatesAndEvents() { control.replay(); - storage.deleteAllUpdatesAndEvents(); + storage.deleteAllUpdates(); } } http://git-wip-us.apache.org/repos/asf/aurora/blob/284f40f5/src/test/java/org/apache/aurora/scheduler/thrift/ReadOnlySchedulerImplTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/ReadOnlySchedulerImplTest.java b/src/test/java/org/apache/aurora/scheduler/thrift/ReadOnlySchedulerImplTest.java index 39456cf..312e5e0 100644 --- a/src/test/java/org/apache/aurora/scheduler/thrift/ReadOnlySchedulerImplTest.java +++ b/src/test/java/org/apache/aurora/scheduler/thrift/ReadOnlySchedulerImplTest.java @@ -16,8 +16,8 @@ package org.apache.aurora.scheduler.thrift; import java.util.Date; import java.util.List; import java.util.Set; - import java.util.stream.Collectors; + import javax.annotation.Nullable; import com.google.common.base.Function; @@ -83,7 +83,6 @@ import org.apache.aurora.scheduler.storage.entities.IJobConfiguration; import org.apache.aurora.scheduler.storage.entities.IJobKey; import org.apache.aurora.scheduler.storage.entities.IJobUpdateDetails; import org.apache.aurora.scheduler.storage.entities.IJobUpdateQuery; -import org.apache.aurora.scheduler.storage.entities.IJobUpdateSummary; import org.apache.aurora.scheduler.storage.entities.IRange; import org.apache.aurora.scheduler.storage.entities.IResponse; import org.apache.aurora.scheduler.storage.entities.IScheduledTask; @@ -590,22 +589,22 @@ public class ReadOnlySchedulerImplTest extends EasyMockTest { @Test public void testGetJobUpdateSummaries() throws Exception { JobUpdateQuery query = new JobUpdateQuery().setRole(ROLE); - List<JobUpdateSummary> summaries = createJobUpdateSummaries(5); - expect(storageUtil.jobUpdateStore.fetchJobUpdateSummaries(IJobUpdateQuery.build(query))) - .andReturn(IJobUpdateSummary.listFromBuilders(summaries)); + Set<JobUpdateDetails> details = createJobUpdateDetails(5); + expect(storageUtil.jobUpdateStore.fetchJobUpdates(IJobUpdateQuery.build(query))) + .andReturn(IJobUpdateDetails.listFromBuilders(details)); control.replay(); Response response = assertOkResponse(thrift.getJobUpdateSummaries(query)); assertEquals( - summaries, + details.stream().map(u -> u.getUpdate().getSummary()).collect(Collectors.toList()), response.getResult().getGetJobUpdateSummariesResult().getUpdateSummaries()); } @Test public void testGetJobUpdateDetails() throws Exception { JobUpdateDetails details = createJobUpdateDetails(); - expect(storageUtil.jobUpdateStore.fetchJobUpdateDetails(UPDATE_KEY)) + expect(storageUtil.jobUpdateStore.fetchJobUpdate(UPDATE_KEY)) .andReturn(Optional.of(IJobUpdateDetails.build(details))); control.replay(); @@ -622,7 +621,7 @@ public class ReadOnlySchedulerImplTest extends EasyMockTest { public void testGetJobUpdateDetailsQuery() throws Exception { JobUpdateQuery query = new JobUpdateQuery().setRole(ROLE); List<IJobUpdateDetails> details = IJobUpdateDetails.listFromBuilders(createJobUpdateDetails(5)); - expect(storageUtil.jobUpdateStore.fetchJobUpdateDetails(IJobUpdateQuery.build(query))) + expect(storageUtil.jobUpdateStore.fetchJobUpdates(IJobUpdateQuery.build(query))) .andReturn(details); control.replay(); @@ -644,16 +643,16 @@ public class ReadOnlySchedulerImplTest extends EasyMockTest { return builder.build(); } - private static List<JobUpdateDetails> createJobUpdateDetails(int count) { + private static Set<JobUpdateDetails> createJobUpdateDetails(int count) { List<JobUpdateSummary> summaries = createJobUpdateSummaries(count); return summaries.stream() .map(jobUpdateSummary -> new JobUpdateDetails().setUpdate(new JobUpdate().setSummary(jobUpdateSummary))) - .collect(Collectors.toList()); + .collect(Collectors.toSet()); } private static JobUpdateDetails createJobUpdateDetails() { - return createJobUpdateDetails(1).get(0); + return createJobUpdateDetails(1).stream().findFirst().get(); } @Test @@ -739,7 +738,7 @@ public class ReadOnlySchedulerImplTest extends EasyMockTest { @Test public void testGetJobUpdateDetailsInvalidId() throws Exception { - expect(storageUtil.jobUpdateStore.fetchJobUpdateDetails(UPDATE_KEY)) + expect(storageUtil.jobUpdateStore.fetchJobUpdate(UPDATE_KEY)) .andReturn(Optional.absent()); control.replay(); http://git-wip-us.apache.org/repos/asf/aurora/blob/284f40f5/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java b/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java index 661ce58..92f2582 100644 --- a/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java +++ b/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java @@ -249,12 +249,12 @@ public class JobUpdaterIT extends EasyMockTest { private IJobUpdateDetails getDetails() { return storage.read( - storeProvider -> storeProvider.getJobUpdateStore().fetchJobUpdateDetails(UPDATE_ID).get()); + storeProvider -> storeProvider.getJobUpdateStore().fetchJobUpdate(UPDATE_ID).get()); } private IJobUpdateDetails getDetails(IJobUpdateKey key) { return storage.read( - storeProvider -> storeProvider.getJobUpdateStore().fetchJobUpdateDetails(key).get()); + storeProvider -> storeProvider.getJobUpdateStore().fetchJobUpdate(key).get()); } private void assertLatestUpdateMessage(String expected) { @@ -709,7 +709,7 @@ public class JobUpdaterIT extends EasyMockTest { assertState(ROLL_FORWARD_AWAITING_PULSE, actions.build()); storage.write((NoResult.Quiet) storeProvider -> { - storeProvider.getJobUpdateStore().deleteAllUpdatesAndEvents(); + storeProvider.getJobUpdateStore().deleteAllUpdates(); }); // The pulse still returns OK but the error is handled. @@ -1099,7 +1099,7 @@ public class JobUpdaterIT extends EasyMockTest { storage.write((NoResult.Quiet) storeProvider -> { JobUpdateStore.Mutable store = storeProvider.getJobUpdateStore(); - store.deleteAllUpdatesAndEvents(); + store.deleteAllUpdates(); JobUpdate builder = update.newBuilder(); builder.getInstructions().getSettings().setUpdateGroupSize(0);
