Repository: aurora Updated Branches: refs/heads/master 165a617f8 -> fbd29a176
Revert "Improving job update query performance." This reverts commit fee5943a95c4f08e148dc5f1366486a8c23d5773. We discovered a bug when deploying this commit that caused corruption of the update store. Reviewed at https://reviews.apache.org/r/42922/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/fbd29a17 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/fbd29a17 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/fbd29a17 Branch: refs/heads/master Commit: fbd29a176c38ce9bac643794daa217056138c448 Parents: 165a617 Author: Joshua Cohen <jco...@apache.org> Authored: Thu Jan 28 16:19:14 2016 -0600 Committer: Joshua Cohen <jco...@apache.org> Committed: Thu Jan 28 16:19:14 2016 -0600 ---------------------------------------------------------------------- .../org/apache/aurora/benchmark/JobUpdates.java | 31 +------ .../aurora/benchmark/UpdateStoreBenchmarks.java | 92 +++++--------------- .../storage/db/JobUpdateDetailsMapper.xml | 57 ++++++------ 3 files changed, 52 insertions(+), 128 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/fbd29a17/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java ---------------------------------------------------------------------- diff --git a/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java b/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java index 50044e1..0c9dbae 100644 --- a/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java +++ b/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java @@ -47,7 +47,6 @@ final class JobUpdates { private static final String USER = "user"; private int numEvents = 1; private int numInstanceEvents = 5000; - private int numInstanceOverrides = 1; Builder setNumEvents(int newCount) { numEvents = newCount; @@ -59,11 +58,6 @@ final class JobUpdates { return this; } - Builder setNumInstanceOverrides(int newCount) { - numInstanceOverrides = newCount; - return this; - } - Set<IJobUpdateDetails> build(int count) { ImmutableSet.Builder<IJobUpdateDetails> result = ImmutableSet.builder(); for (int i = 0; i < count; i++) { @@ -84,9 +78,10 @@ final class JobUpdates { .setMaxPerInstanceFailures(1) .setMinWaitInInstanceRunningMs(1) .setRollbackOnFailure(true) - .setWaitForBatchCompletion(false) - .setUpdateOnlyTheseInstances(ranges(numInstanceOverrides))) - .setInitialState(instanceConfigs(task, numInstanceOverrides)) + .setWaitForBatchCompletion(false)) + .setInitialState(ImmutableSet.of(new InstanceTaskConfig() + .setTask(task) + .setInstances(ImmutableSet.of(new Range(0, 10))))) .setDesiredState(new InstanceTaskConfig() .setTask(task) .setInstances(ImmutableSet.of(new Range(0, numInstanceEvents))))); @@ -112,24 +107,6 @@ final class JobUpdates { return result.build(); } - private static Set<InstanceTaskConfig> instanceConfigs(TaskConfig task, int count) { - ImmutableSet.Builder<InstanceTaskConfig> result = ImmutableSet.builder(); - for (int i = 0; i < count; i++) { - result.add(new InstanceTaskConfig(task, ImmutableSet.of(new Range(i, i)))); - } - - return result.build(); - } - - private static Set<Range> ranges(int count) { - ImmutableSet.Builder<Range> result = ImmutableSet.builder(); - for (int i = 0; i < count; i++) { - result.add(new Range(i, i)); - } - - return result.build(); - } - private static String string(int numChars) { char[] chars = new char[numChars]; Arrays.fill(chars, 'a'); http://git-wip-us.apache.org/repos/asf/aurora/blob/fbd29a17/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java ---------------------------------------------------------------------- diff --git a/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java b/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java index 92849d9..1d8986b 100644 --- a/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java +++ b/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java @@ -48,7 +48,6 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; public class UpdateStoreBenchmarks { - private static final String USER = "user"; @BenchmarkMode(Mode.Throughput) @OutputTimeUnit(TimeUnit.SECONDS) @@ -57,6 +56,7 @@ public class UpdateStoreBenchmarks { @Fork(1) @State(Scope.Thread) public static class JobDetailsBenchmark { + private static final String USER = "user"; private Storage storage; private Set<IJobUpdateKey> keys; @@ -71,10 +71,29 @@ public class UpdateStoreBenchmarks { @Setup(Level.Iteration) public void setUpIteration() { storage.write((NoResult.Quiet) storeProvider -> { + JobUpdateStore.Mutable updateStore = storeProvider.getJobUpdateStore(); Set<IJobUpdateDetails> updates = new JobUpdates.Builder().setNumInstanceEvents(instances).build(1); - keys = saveToStore(updates, storeProvider); + ImmutableSet.Builder<IJobUpdateKey> keyBuilder = ImmutableSet.builder(); + for (IJobUpdateDetails details : updates) { + IJobUpdateKey key = details.getUpdate().getSummary().getKey(); + keyBuilder.add(key); + String lockToken = UUID.randomUUID().toString(); + storeProvider.getLockStore().saveLock( + ILock.build(new Lock(LockKey.job(key.getJob().newBuilder()), lockToken, USER, 0L))); + + updateStore.saveJobUpdate(details.getUpdate(), Optional.of(lockToken)); + + for (IJobUpdateEvent updateEvent : details.getUpdateEvents()) { + updateStore.saveJobUpdateEvent(key, updateEvent); + } + + for (IJobInstanceUpdateEvent instanceEvent : details.getInstanceEvents()) { + updateStore.saveJobInstanceUpdateEvent(key, instanceEvent); + } + } + keys = keyBuilder.build(); }); } @@ -92,73 +111,4 @@ public class UpdateStoreBenchmarks { Iterables.getOnlyElement(keys)).get()); } } - - @BenchmarkMode(Mode.Throughput) - @OutputTimeUnit(TimeUnit.SECONDS) - @Warmup(iterations = 1, time = 10, timeUnit = TimeUnit.SECONDS) - @Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS) - @Fork(1) - @State(Scope.Thread) - public static class JobInstructionsBenchmark { - private Storage storage; - private Set<IJobUpdateKey> keys; - - @Param({"1", "10", "100", "1000"}) - private int instanceOverrides; - - @Setup(Level.Trial) - public void setUp() { - storage = DbUtil.createStorage(); - } - - @Setup(Level.Iteration) - public void setUpIteration() { - storage.write((NoResult.Quiet) storeProvider -> { - Set<IJobUpdateDetails> updates = - new JobUpdates.Builder().setNumInstanceOverrides(instanceOverrides).build(1); - - keys = saveToStore(updates, storeProvider); - }); - } - - @TearDown(Level.Iteration) - public void tearDownIteration() { - storage.write((NoResult.Quiet) storeProvider -> { - storeProvider.getJobUpdateStore().deleteAllUpdatesAndEvents(); - storeProvider.getLockStore().deleteLocks(); - }); - } - - @Benchmark - public IJobUpdateDetails run() throws TException { - return storage.read(store -> store.getJobUpdateStore().fetchJobUpdateDetails( - Iterables.getOnlyElement(keys)).get()); - } - } - - private static Set<IJobUpdateKey> saveToStore( - Set<IJobUpdateDetails> updates, - Storage.MutableStoreProvider storeProvider) { - - JobUpdateStore.Mutable updateStore = storeProvider.getJobUpdateStore(); - ImmutableSet.Builder<IJobUpdateKey> keyBuilder = ImmutableSet.builder(); - for (IJobUpdateDetails details : updates) { - IJobUpdateKey key = details.getUpdate().getSummary().getKey(); - keyBuilder.add(key); - String lockToken = UUID.randomUUID().toString(); - storeProvider.getLockStore().saveLock( - ILock.build(new Lock(LockKey.job(key.getJob().newBuilder()), lockToken, USER, 0L))); - - updateStore.saveJobUpdate(details.getUpdate(), Optional.of(lockToken)); - - for (IJobUpdateEvent updateEvent : details.getUpdateEvents()) { - updateStore.saveJobUpdateEvent(key, updateEvent); - } - - for (IJobInstanceUpdateEvent instanceEvent : details.getInstanceEvents()) { - updateStore.saveJobInstanceUpdateEvent(key, instanceEvent); - } - } - return keyBuilder.build(); - } } http://git-wip-us.apache.org/repos/asf/aurora/blob/fbd29a17/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml ---------------------------------------------------------------------- diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml b/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml index 8438f72..fba7d4f 100644 --- a/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml +++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml @@ -170,9 +170,9 @@ <resultMap id="jobUpdateSettingsMap" type="org.apache.aurora.gen.JobUpdateSettings"> <id column="id" /> <collection property="updateOnlyTheseInstances" - select="selectInstanceOverrides" - column="id" - foreignColumn="update_row_id" /> + resultMap="rangeMap" + columnPrefix="r_" + notNullColumn="id" /> </resultMap> <resultMap @@ -182,9 +182,9 @@ <association property="desiredState" resultMap="instanceConfigMap" columnPrefix="ditc_" /> <association property="settings" resultMap="jobUpdateSettingsMap" columnPrefix="juse_"/> <collection property="initialState" - select="selectInstanceConfigs" - column="id" - foreignColumn="update_row_id" /> + resultMap="instanceConfigMap" + columnPrefix="iitc_" + notNullColumn="id" /> </resultMap> <resultMap id="jobUpdateMap" type="org.apache.aurora.scheduler.storage.db.views.DbJobUpdate"> @@ -350,12 +350,23 @@ cn.task_config_row_id AS jui_ditc_task_config_row_id, di.id AS jui_ditc_r_id, di.first AS jui_ditc_r_first, - di.last AS jui_ditc_r_last + di.last AS jui_ditc_r_last, + co.id AS jui_iitc_id, + co.task_config_row_id AS jui_iitc_task_config_row_id, + ci.id AS jui_iitc_r_id, + ci.first AS jui_iitc_r_first, + ci.last AS jui_iitc_r_last, + io.id AS jui_juse_r_id, + io.first AS jui_juse_r_first, + io.last AS jui_juse_r_last </sql> <sql id="job_update_outer_joins"> LEFT OUTER JOIN job_update_configs AS cn ON cn.update_row_id = u.id AND cn.is_new = TRUE + LEFT OUTER JOIN job_update_configs AS co ON co.update_row_id = u.id AND co.is_new = FALSE + LEFT OUTER JOIN job_update_configs_to_instances AS ci ON ci.config_id = co.id LEFT OUTER JOIN job_updates_to_desired_instances AS di ON di.update_row_id = u.id + LEFT OUTER JOIN job_updates_to_instance_overrides AS io ON io.update_row_id = u.id </sql> <sql id="lock_outer_join"> @@ -391,7 +402,15 @@ cn.task_config_row_id AS ditc_task_config_row_id, di.id AS ditc_r_id, di.first AS ditc_r_first, - di.last AS ditc_r_last + di.last AS ditc_r_last, + co.id AS iitc_id, + co.task_config_row_id AS iitc_task_config_row_id, + ci.id AS iitc_r_id, + ci.first AS iitc_r_first, + ci.last AS iitc_r_last, + io.id AS juse_r_id, + io.first AS juse_r_first, + io.last AS juse_r_last FROM job_updates AS u <include refid="job_key_inner_join" /> <include refid="job_update_outer_joins" /> @@ -431,28 +450,6 @@ WHERE <include refid="filter_by_update_key"/> </select> - <select id="selectInstanceConfigs" resultMap="instanceConfigMap"> - SELECT - co.id AS id, - co.task_config_row_id AS task_config_row_id, - ci.id AS r_id, - ci.first AS r_first, - ci.last AS r_last - FROM job_update_configs as co - INNER JOIN job_update_configs_to_instances AS ci ON ci.config_id = co.id - WHERE co.update_row_id = #{id} - AND co.is_new = FALSE - </select> - - <select id="selectInstanceOverrides" resultMap="rangeMap"> - SELECT - o.id, - o.first, - o.last - FROM job_updates_to_instance_overrides as o - WHERE update_row_id = #{id} - </select> - <select id="selectInstanceUpdateEvents" resultMap="jobInstanceUpdateMap"> SELECT e.id,