Repository: aurora Updated Branches: refs/heads/master 9e77a6f37 -> 7ccd5325b
Remove job update `maxWaitToInstanceRunningMs` field. This field in the thrift api `JobUpdateSettings` struct and its sibling in `UpdateConfig.restart_threshold` on the client side were deprecated in Aurora 0.11.0. Testing Done: Locally green: ``` ./build-support/jenkins/build.sh ./src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh ``` Bugs closed: AURORA-1254 Reviewed at https://reviews.apache.org/r/42804/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/7ccd5325 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/7ccd5325 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/7ccd5325 Branch: refs/heads/master Commit: 7ccd5325b5215c1f0fd11d4d681d681e4187437f Parents: 9e77a6f Author: John Sirois <[email protected]> Authored: Tue Jan 26 11:30:09 2016 -0700 Committer: John Sirois <[email protected]> Committed: Tue Jan 26 11:30:09 2016 -0700 ---------------------------------------------------------------------- NEWS | 3 +++ api/src/main/thrift/org/apache/aurora/gen/api.thrift | 6 ------ docs/configuration-reference.md | 1 - src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java | 1 - .../aurora/scheduler/thrift/SchedulerThriftInterface.java | 6 ------ src/main/python/apache/aurora/client/api/updater_util.py | 3 --- src/main/python/apache/aurora/config/schema/base.py | 1 - .../aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml | 4 ---- .../org/apache/aurora/scheduler/storage/db/schema.sql | 1 - .../aurora/scheduler/storage/db/DbJobUpdateStoreTest.java | 1 - .../scheduler/thrift/SchedulerThriftInterfaceTest.java | 1 - .../org/apache/aurora/scheduler/updater/JobUpdaterIT.java | 9 ++------- .../aurora/scheduler/updater/UpdateFactoryImplTest.java | 1 - src/test/python/apache/aurora/client/api/test_api.py | 1 - .../python/apache/aurora/client/api/test_updater_util.py | 6 +++--- .../python/apache/aurora/client/cli/test_config_noun.py | 1 - src/test/python/apache/aurora/client/cli/util.py | 2 -- 17 files changed, 8 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/7ccd5325/NEWS ---------------------------------------------------------------------- diff --git a/NEWS b/NEWS index f2798f6..3f40aba 100644 --- a/NEWS +++ b/NEWS @@ -32,6 +32,9 @@ - Removed scheduler flag `-extra_modules`. - Deprecated `TaskQuery` argument in `killTasks` thrift RPC to disallow killing tasks across multiple roles. The new safer approach is using `JobKey` with `instances` instead. +- Removed the deprecated 'JobUpdateSettings.maxWaitToInstanceRunningMs' thrift api field ( + UpdateConfig.restart_threshold in client-side configuration). This aspect of job restarts is now + controlled exclusively via the client with `aurora job restart --restart-threshold=[seconds]`. 0.11.0 ------ http://git-wip-us.apache.org/repos/asf/aurora/blob/7ccd5325/api/src/main/thrift/org/apache/aurora/gen/api.thrift ---------------------------------------------------------------------- diff --git a/api/src/main/thrift/org/apache/aurora/gen/api.thrift b/api/src/main/thrift/org/apache/aurora/gen/api.thrift index a93df21..f0e330c 100644 --- a/api/src/main/thrift/org/apache/aurora/gen/api.thrift +++ b/api/src/main/thrift/org/apache/aurora/gen/api.thrift @@ -659,12 +659,6 @@ struct JobUpdateSettings { /** Max number of FAILED instances to tolerate before terminating the update. */ 3: i32 maxFailedInstances - /** - * Max time to wait until an instance reaches RUNNING state. - * Note: Deprecated in 0.11.0. - */ - 4: i32 maxWaitToInstanceRunningMs - /** Min time to watch a RUNNING instance. */ 5: i32 minWaitInInstanceRunningMs http://git-wip-us.apache.org/repos/asf/aurora/blob/7ccd5325/docs/configuration-reference.md ---------------------------------------------------------------------- diff --git a/docs/configuration-reference.md b/docs/configuration-reference.md index e0eb908..c04f3b7 100644 --- a/docs/configuration-reference.md +++ b/docs/configuration-reference.md @@ -415,7 +415,6 @@ Parameters for controlling the rate and policy of rolling updates. | object | type | description | ---------------------------- | :------: | ------------ | ```batch_size``` | Integer | Maximum number of shards to be updated in one iteration (Default: 1) -| ```restart_threshold``` | Integer | Maximum number of seconds before a shard must move into the ```RUNNING``` state before considered a failure (Default: 60) | ```watch_secs``` | Integer | Minimum number of seconds a shard must remain in ```RUNNING``` state before considered a success (Default: 45) | ```max_per_shard_failures``` | Integer | Maximum number of restarts per shard during update. Increments total failure count when this limit is exceeded. (Default: 0) | ```max_total_failures``` | Integer | Maximum number of shard failures to be tolerated in total during an update. Cannot be greater than or equal to the total number of tasks in a job. (Default: 0) http://git-wip-us.apache.org/repos/asf/aurora/blob/7ccd5325/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java ---------------------------------------------------------------------- diff --git a/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java b/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java index 493f75d..0c9dbae 100644 --- a/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java +++ b/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java @@ -76,7 +76,6 @@ final class JobUpdates { .setUpdateGroupSize(100) .setMaxFailedInstances(1) .setMaxPerInstanceFailures(1) - .setMaxWaitToInstanceRunningMs(1) .setMinWaitInInstanceRunningMs(1) .setRollbackOnFailure(true) .setWaitForBatchCompletion(false)) http://git-wip-us.apache.org/repos/asf/aurora/blob/7ccd5325/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java b/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java index 662cae1..c53c49e 100644 --- a/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java +++ b/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java @@ -979,12 +979,6 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin { .setInstructions(instructions)); Response response = empty(); - if (update.getInstructions().getSettings().getMaxWaitToInstanceRunningMs() > 0) { - addMessage( - response, - "The maxWaitToInstanceRunningMs (restart_threshold) field is deprecated."); - } - try { validateTaskLimits( request.getTaskConfig(), http://git-wip-us.apache.org/repos/asf/aurora/blob/7ccd5325/src/main/python/apache/aurora/client/api/updater_util.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/client/api/updater_util.py b/src/main/python/apache/aurora/client/api/updater_util.py index a3389f3..c649316 100644 --- a/src/main/python/apache/aurora/client/api/updater_util.py +++ b/src/main/python/apache/aurora/client/api/updater_util.py @@ -26,7 +26,6 @@ class UpdaterConfig(object): def __init__(self, batch_size, - restart_threshold, watch_secs, max_per_shard_failures, max_total_failures, @@ -41,8 +40,6 @@ class UpdaterConfig(object): if pulse_interval_secs is not None and pulse_interval_secs < self.MIN_PULSE_INTERVAL_SECONDS: raise ValueError('Pulse interval seconds must be at least %s seconds.' % self.MIN_PULSE_INTERVAL_SECONDS) - if restart_threshold: - log.warn('restart_threshold has been deprecated and will be removed in a future release') self.batch_size = batch_size self.watch_secs = watch_secs http://git-wip-us.apache.org/repos/asf/aurora/blob/7ccd5325/src/main/python/apache/aurora/config/schema/base.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/config/schema/base.py b/src/main/python/apache/aurora/config/schema/base.py index 7bc903c..0e4dfc2 100644 --- a/src/main/python/apache/aurora/config/schema/base.py +++ b/src/main/python/apache/aurora/config/schema/base.py @@ -27,7 +27,6 @@ class MesosContext(Struct): class UpdateConfig(Struct): batch_size = Default(Integer, 1) - restart_threshold = Default(Integer, 60) watch_secs = Default(Integer, 45) max_per_shard_failures = Default(Integer, 0) max_total_failures = Default(Integer, 0) http://git-wip-us.apache.org/repos/asf/aurora/blob/7ccd5325/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml ---------------------------------------------------------------------- diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml b/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml index 69e11b6..fba7d4f 100644 --- a/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml +++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml @@ -49,7 +49,6 @@ update_group_size, max_per_instance_failures, max_failed_instances, - max_wait_to_instance_running_ms, min_wait_in_instance_running_ms, rollback_on_failure, wait_for_batch_completion, @@ -67,7 +66,6 @@ #{instructions.settings.updateGroupSize}, #{instructions.settings.maxPerInstanceFailures}, #{instructions.settings.maxFailedInstances}, - #{instructions.settings.maxWaitToInstanceRunningMs}, #{instructions.settings.minWaitInInstanceRunningMs}, #{instructions.settings.rollbackOnFailure}, #{instructions.settings.waitForBatchCompletion}, @@ -343,7 +341,6 @@ u.update_group_size AS jui_juse_update_group_size, u.max_per_instance_failures AS jui_juse_max_per_instance_failures, u.max_failed_instances AS jui_juse_max_failed_instances, - u.max_wait_to_instance_running_ms AS jui_juse_max_wait_to_instance_running_ms, u.min_wait_in_instance_running_ms AS jui_juse_min_wait_in_instance_running_ms, u.rollback_on_failure AS jui_juse_rollback_on_failure, u.wait_for_batch_completion AS jui_juse_wait_for_batch_completion, @@ -396,7 +393,6 @@ u.update_group_size AS juse_update_group_size, u.max_per_instance_failures AS juse_max_per_instance_failures, u.max_failed_instances AS juse_max_failed_instances, - u.max_wait_to_instance_running_ms AS juse_max_wait_to_instance_running_ms, u.min_wait_in_instance_running_ms AS juse_min_wait_in_instance_running_ms, u.rollback_on_failure AS juse_rollback_on_failure, u.wait_for_batch_completion AS juse_wait_for_batch_completion, http://git-wip-us.apache.org/repos/asf/aurora/blob/7ccd5325/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql ---------------------------------------------------------------------- diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql b/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql index 803960f..7fbdcbf 100644 --- a/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql +++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql @@ -250,7 +250,6 @@ CREATE TABLE job_updates( update_group_size INT NOT NULL, max_per_instance_failures INT NOT NULL, max_failed_instances INT NOT NULL, - max_wait_to_instance_running_ms INT NOT NULL, min_wait_in_instance_running_ms INT NOT NULL, rollback_on_failure BOOLEAN NOT NULL, wait_for_batch_completion BOOLEAN NOT NULL, http://git-wip-us.apache.org/repos/asf/aurora/blob/7ccd5325/src/test/java/org/apache/aurora/scheduler/storage/db/DbJobUpdateStoreTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/db/DbJobUpdateStoreTest.java b/src/test/java/org/apache/aurora/scheduler/storage/db/DbJobUpdateStoreTest.java index d035513..54defc2 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/db/DbJobUpdateStoreTest.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/db/DbJobUpdateStoreTest.java @@ -1039,7 +1039,6 @@ public class DbJobUpdateStoreTest { .setUpdateGroupSize(1) .setMaxPerInstanceFailures(1) .setMaxFailedInstances(1) - .setMaxWaitToInstanceRunningMs(100) .setMinWaitInInstanceRunningMs(200) .setRollbackOnFailure(true) .setWaitForBatchCompletion(true) http://git-wip-us.apache.org/repos/asf/aurora/blob/7ccd5325/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java b/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java index b9ce2dd..84356d6 100644 --- a/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java +++ b/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java @@ -2080,7 +2080,6 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest { .setUpdateGroupSize(10) .setMaxFailedInstances(2) .setMaxPerInstanceFailures(1) - .setMaxWaitToInstanceRunningMs(30000) .setMinWaitInInstanceRunningMs(15000) .setRollbackOnFailure(true); } http://git-wip-us.apache.org/repos/asf/aurora/blob/7ccd5325/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java b/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java index 2024b2c..b39e388 100644 --- a/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java +++ b/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java @@ -133,7 +133,6 @@ public class JobUpdaterIT extends EasyMockTest { private static final IJobKey JOB = JobKeys.from("role", "env", "job1"); private static final IJobUpdateKey UPDATE_ID = IJobUpdateKey.build(new JobUpdateKey(JOB.newBuilder(), "update_id")); - private static final Amount<Long, Time> RUNNING_TIMEOUT = Amount.of(1000L, Time.MILLISECONDS); private static final Amount<Long, Time> WATCH_TIMEOUT = Amount.of(2000L, Time.MILLISECONDS); private static final Amount<Long, Time> FLAPPING_THRESHOLD = Amount.of(1L, Time.MILLISECONDS); private static final Amount<Long, Time> ONE_DAY = Amount.of(1L, Time.DAYS); @@ -695,10 +694,9 @@ public class JobUpdaterIT extends EasyMockTest { .putAll(1, INSTANCE_UPDATING); assertState(ROLLING_FORWARD, actions.build()); changeState(JOB, 1, FINISHED, ASSIGNED, STARTING, RUNNING); - clock.advance(Amount.of(RUNNING_TIMEOUT.getValue() / 2, Time.MILLISECONDS)); + clock.advance(Amount.of(WATCH_TIMEOUT.getValue() / 2, Time.MILLISECONDS)); changeState(JOB, 0, FINISHED, ASSIGNED, STARTING, RUNNING); - clock.advance( - Amount.of(WATCH_TIMEOUT.getValue() - (RUNNING_TIMEOUT.getValue() / 2), Time.MILLISECONDS)); + clock.advance(Amount.of(WATCH_TIMEOUT.getValue() / 2, Time.MILLISECONDS)); // Instance 1 finished first, but update does not yet proceed until 0 finishes. actions.putAll(1, INSTANCE_UPDATED); @@ -1181,8 +1179,6 @@ public class JobUpdaterIT extends EasyMockTest { actions.putAll(0, INSTANCE_UPDATED) .putAll(1, INSTANCE_UPDATING); assertState(ROLLING_FORWARD, actions.build()); - clock.advance(RUNNING_TIMEOUT); - assertState(ROLLING_FORWARD, actions.build()); updater.abort(update.getSummary().getKey(), AUDIT); assertState(ABORTED, actions.build()); @@ -1344,7 +1340,6 @@ public class JobUpdaterIT extends EasyMockTest { .setSettings(new JobUpdateSettings() .setUpdateGroupSize(1) .setRollbackOnFailure(true) - .setMaxWaitToInstanceRunningMs(RUNNING_TIMEOUT.as(Time.MILLISECONDS).intValue()) .setMinWaitInInstanceRunningMs(WATCH_TIMEOUT.as(Time.MILLISECONDS).intValue()) .setUpdateOnlyTheseInstances(ImmutableSet.of()))); http://git-wip-us.apache.org/repos/asf/aurora/blob/7ccd5325/src/test/java/org/apache/aurora/scheduler/updater/UpdateFactoryImplTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/updater/UpdateFactoryImplTest.java b/src/test/java/org/apache/aurora/scheduler/updater/UpdateFactoryImplTest.java index c31e082..611f6b8 100644 --- a/src/test/java/org/apache/aurora/scheduler/updater/UpdateFactoryImplTest.java +++ b/src/test/java/org/apache/aurora/scheduler/updater/UpdateFactoryImplTest.java @@ -41,7 +41,6 @@ public class UpdateFactoryImplTest { .setSettings(new JobUpdateSettings() .setMaxFailedInstances(1) .setMaxPerInstanceFailures(1) - .setMaxWaitToInstanceRunningMs(100) .setMinWaitInInstanceRunningMs(100) .setUpdateGroupSize(2) .setUpdateOnlyTheseInstances(ImmutableSet.of()))); http://git-wip-us.apache.org/repos/asf/aurora/blob/7ccd5325/src/test/python/apache/aurora/client/api/test_api.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/client/api/test_api.py b/src/test/python/apache/aurora/client/api/test_api.py index 59f34a5..1f041f4 100644 --- a/src/test/python/apache/aurora/client/api/test_api.py +++ b/src/test/python/apache/aurora/client/api/test_api.py @@ -44,7 +44,6 @@ class TestJobUpdateApis(unittest.TestCase): UPDATE_CONFIG = { 'batch_size': 1, - 'restart_threshold': 50, 'watch_secs': 50, 'max_per_shard_failures': 2, 'max_total_failures': 1, http://git-wip-us.apache.org/repos/asf/aurora/blob/7ccd5325/src/test/python/apache/aurora/client/api/test_updater_util.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/client/api/test_updater_util.py b/src/test/python/apache/aurora/client/api/test_updater_util.py index 11b95c6..48926c3 100644 --- a/src/test/python/apache/aurora/client/api/test_updater_util.py +++ b/src/test/python/apache/aurora/client/api/test_updater_util.py @@ -46,15 +46,15 @@ class TestRangeConversion(unittest.TestCase): assert UpdaterConfig.instances_to_ranges([]) is None, "Result must be None." def test_pulse_interval_secs(self): - config = UpdaterConfig(1, 1, 1, 1, 1, pulse_interval_secs=60) + config = UpdaterConfig(1, 1, 1, 1, pulse_interval_secs=60) assert 60000 == config.to_thrift_update_settings().blockIfNoPulsesAfterMs def test_pulse_interval_unset(self): - config = UpdaterConfig(1, 1, 1, 1, 1) + config = UpdaterConfig(1, 1, 1, 1) assert config.to_thrift_update_settings().blockIfNoPulsesAfterMs is None def test_pulse_interval_too_low(self): threshold = UpdaterConfig.MIN_PULSE_INTERVAL_SECONDS with raises(ValueError) as e: - UpdaterConfig(1, 1, 1, 1, 1, pulse_interval_secs=threshold - 1) + UpdaterConfig(1, 1, 1, 1, pulse_interval_secs=threshold - 1) assert 'Pulse interval seconds must be at least %s seconds.' % threshold in e.value.message http://git-wip-us.apache.org/repos/asf/aurora/blob/7ccd5325/src/test/python/apache/aurora/client/cli/test_config_noun.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/client/cli/test_config_noun.py b/src/test/python/apache/aurora/client/cli/test_config_noun.py index d82a69d..a64a58b 100644 --- a/src/test/python/apache/aurora/client/cli/test_config_noun.py +++ b/src/test/python/apache/aurora/client/cli/test_config_noun.py @@ -61,7 +61,6 @@ class TestClientCreateCommand(AuroraClientCommandTest): instances = 20, update_config = UpdateConfig( batch_size = 5, - restart_threshold = 30, watch_secs = 10, max_per_shard_failures = 2, ), http://git-wip-us.apache.org/repos/asf/aurora/blob/7ccd5325/src/test/python/apache/aurora/client/cli/util.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/client/cli/util.py b/src/test/python/apache/aurora/client/cli/util.py index c7aa5cc..7ae5886 100644 --- a/src/test/python/apache/aurora/client/cli/util.py +++ b/src/test/python/apache/aurora/client/cli/util.py @@ -265,7 +265,6 @@ HELLO_WORLD = Job( %(inner)s update_config = UpdateConfig( batch_size = 1, - restart_threshold = 60, watch_secs = 45, max_per_shard_failures = 2, ), @@ -300,7 +299,6 @@ jobs = [HELLO_WORLD] instances = '{{instances_binding}}', update_config = UpdateConfig( batch_size = "{{TEST_BATCH}}", - restart_threshold = 60, watch_secs = 45, max_per_shard_failures = 2, ),
