Repository: aurora Updated Branches: refs/heads/master f58cbf934 -> 506d0cdbc
Revocable: schema changes. Bugs closed: AURORA-1414 Reviewed at https://reviews.apache.org/r/37719/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/506d0cdb Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/506d0cdb Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/506d0cdb Branch: refs/heads/master Commit: 506d0cdbcb558fe43cab60ced8b613cfddc54b7f Parents: f58cbf9 Author: Maxim Khutornenko <[email protected]> Authored: Tue Aug 25 15:55:20 2015 -0700 Committer: Maxim Khutornenko <[email protected]> Committed: Tue Aug 25 15:55:20 2015 -0700 ---------------------------------------------------------------------- .../thrift/org/apache/aurora/gen/api.thrift | 2 ++ .../apache/aurora/scheduler/TierManager.java | 4 +--- .../aurora/scheduler/base/TaskTestUtil.java | 1 + .../configuration/ConfigurationManager.java | 4 ++++ .../storage/db/views/DbTaskConfig.java | 2 ++ src/main/python/apache/aurora/client/config.py | 25 ++++++++++++++++---- .../python/apache/aurora/config/schema/base.py | 1 + src/main/python/apache/aurora/config/thrift.py | 1 + .../scheduler/storage/db/TaskConfigMapper.xml | 3 +++ .../aurora/scheduler/storage/db/schema.sql | 3 ++- .../aurora/scheduler/TierManagerTest.java | 2 +- .../configuration/ConfigurationManagerTest.java | 15 ++++++++++-- .../apache/aurora/client/cli/test_create.py | 5 +--- .../python/apache/aurora/client/test_config.py | 25 ++++++++++++++++---- .../python/apache/aurora/config/test_thrift.py | 7 ++++++ 15 files changed, 80 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/api/src/main/thrift/org/apache/aurora/gen/api.thrift ---------------------------------------------------------------------- diff --git a/api/src/main/thrift/org/apache/aurora/gen/api.thrift b/api/src/main/thrift/org/apache/aurora/gen/api.thrift index f792be0..21137bb 100644 --- a/api/src/main/thrift/org/apache/aurora/gen/api.thrift +++ b/api/src/main/thrift/org/apache/aurora/gen/api.thrift @@ -251,6 +251,8 @@ struct TaskConfig { 13: i32 maxTaskFailures /** Whether this is a production task, which can preempt. */ 18: optional bool production + /** Task tier type. */ + 30: optional string tier 20: set<Constraint> constraints /** a list of named ports this task requests */ http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/java/org/apache/aurora/scheduler/TierManager.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/TierManager.java b/src/main/java/org/apache/aurora/scheduler/TierManager.java index ebfad97..652afec 100644 --- a/src/main/java/org/apache/aurora/scheduler/TierManager.java +++ b/src/main/java/org/apache/aurora/scheduler/TierManager.java @@ -17,8 +17,7 @@ import org.apache.aurora.scheduler.storage.entities.ITaskConfig; /** * Translates job tier configuration into a set of task traits/attributes. - * TODO(maxim): Implement external configuration support defined here: - * https://docs.google.com/document/d/1gexe2uM_9gjsV62cMmX0VjH85Uokko21vEoENY2jjF0 + * TODO(maxim): Implement external configuration support: AURORA-1443. */ public interface TierManager { @@ -34,7 +33,6 @@ public interface TierManager { @Override public TierInfo getTier(ITaskConfig taskConfig) { - // TODO(maxim): Implement when schema changes are defined. return new TierInfo(false); } } http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java b/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java index 1903315..d7f3c60 100644 --- a/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java +++ b/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java @@ -63,6 +63,7 @@ public final class TaskTestUtil { .setPriority(1) .setMaxTaskFailures(-1) .setProduction(true) + .setTier("tier-" + job.getEnvironment()) .setConstraints(ImmutableSet.of( new Constraint( "valueConstraint", http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java b/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java index d103d19..05e8b71 100644 --- a/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java +++ b/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java @@ -287,6 +287,10 @@ public final class ConfigurationManager { "Environment contains illegal characters: " + config.getEnvironment()); } + if (config.isSetTier() && !isGoodIdentifier(config.getTier())) { + throw new TaskDescriptionException("Tier contains illegal characters: " + config.getTier()); + } + if (config.isSetJob()) { if (!JobKeys.isValid(config.getJob())) { // Job key is set but invalid http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java b/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java index 956c508..9879836 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java @@ -48,6 +48,7 @@ public final class DbTaskConfig { private ExecutorConfig executorConfig; private List<Metadata> metadata; private DbContainer container; + private String tier; private DbTaskConfig() { } @@ -69,6 +70,7 @@ public final class DbTaskConfig { .setPriority(priority) .setMaxTaskFailures(maxTaskFailures) .setProduction(production) + .setTier(tier) .setConstraints(constraints.stream() .map(DbConstraint::toThrift) .collect(GuavaUtils.toImmutableSet())) http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/python/apache/aurora/client/config.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/client/config.py b/src/main/python/apache/aurora/client/config.py index 59703ef..f5ac084 100644 --- a/src/main/python/apache/aurora/client/config.py +++ b/src/main/python/apache/aurora/client/config.py @@ -22,6 +22,8 @@ import math import re import sys +from pystachio import Empty + from apache.aurora.client import binding_helper from apache.aurora.client.base import die from apache.aurora.config import AuroraConfig @@ -55,13 +57,25 @@ def _validate_announce_configuration(config): STAGING_RE = re.compile(r'^staging\d*$') +#TODO(maxim): Merge env and tier and move definitions to scheduler: AURORA-1443. +def __validate_env(name, config_name): + if STAGING_RE.match(name): + return + if name not in ('prod', 'devel', 'test'): + raise ValueError('%s should be one of "prod", "devel", "test" or ' + 'staging<number>! Got %s' % (config_name, name)) + + def _validate_environment_name(config): env_name = str(config.raw().environment()) - if STAGING_RE.match(env_name): - return - if env_name not in ('prod', 'devel', 'test'): - raise ValueError('Environment name should be one of "prod", "devel", "test" or ' - 'staging<number>! Got %s' % env_name) + __validate_env(env_name, 'Environment') + + +def _validate_tier(config): + tier_raw = config.raw().tier() + tier_name = str(tier_raw) if tier_raw is not Empty else None + if tier_name is not None: + __validate_env(tier_name, 'Tier') UPDATE_CONFIG_MAX_FAILURES_ERROR = ''' @@ -114,6 +128,7 @@ def validate_config(config, env=None): _validate_update_config(config) _validate_announce_configuration(config) _validate_environment_name(config) + _validate_tier(config) class GlobalHookRegistry(object): http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/python/apache/aurora/config/schema/base.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/config/schema/base.py b/src/main/python/apache/aurora/config/schema/base.py index 214d559..f8a1f05 100644 --- a/src/main/python/apache/aurora/config/schema/base.py +++ b/src/main/python/apache/aurora/config/schema/base.py @@ -110,6 +110,7 @@ class MesosJob(Struct): instances = Default(Integer, 1) task = Required(Task) announce = Announcer + tier = String cron_schedule = String cron_collision_policy = Default(String, "KILL_EXISTING") http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/python/apache/aurora/config/thrift.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/config/thrift.py b/src/main/python/apache/aurora/config/thrift.py index adf53bb..f69977c 100644 --- a/src/main/python/apache/aurora/config/thrift.py +++ b/src/main/python/apache/aurora/config/thrift.py @@ -197,6 +197,7 @@ def convert(job, metadata=frozenset(), ports=frozenset()): task.maxTaskFailures = fully_interpolated(job.max_task_failures()) task.priority = fully_interpolated(job.priority()) task.contactEmail = not_empty_or(job.contact(), None) + task.tier = not_empty_or(job.tier(), None) # Add metadata to a task, to display in the scheduler UI. task.metadata = frozenset(Metadata(key=str(key), value=str(value)) for key, value in metadata) http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml ---------------------------------------------------------------------- diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml b/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml index bfad339..c70e859 100644 --- a/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml +++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml @@ -32,6 +32,7 @@ contact_email, executor_name, executor_data, + tier ) VALUES ( ( SELECT ID @@ -51,6 +52,7 @@ #{config.contactEmail}, #{config.executorConfig.name}, #{config.executorConfig.data}, + #{config.tier} ) </insert> @@ -167,6 +169,7 @@ c.contact_email AS contact_email, c.executor_name AS executor_name, c.executor_data AS executor_data, + c.tier AS tier, j.role AS j_role, j.environment AS j_environment, j.name AS j_name, http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql ---------------------------------------------------------------------- diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql b/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql index 7634047..d971aa1 100644 --- a/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql +++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql @@ -187,7 +187,8 @@ CREATE TABLE task_configs( production BOOLEAN NOT NULL, contact_email VARCHAR, executor_name VARCHAR NOT NULL, - executor_data VARCHAR NOT NULL + executor_data VARCHAR NOT NULL, + tier VARCHAR ); CREATE TABLE task_constraints( http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java b/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java index 37e19ac..eb02071 100644 --- a/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java +++ b/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java @@ -23,7 +23,7 @@ import static org.junit.Assert.assertEquals; public class TierManagerTest { @Test - public void testIsRevocable() { + public void testIsNotRevocable() { TierInfo expected = new TierInfo(false); assertEquals(expected, new TierManagerImpl().getTier(ITaskConfig.build(new TaskConfig()))); } http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java b/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java index b231827..f3b62cc 100644 --- a/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java +++ b/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java @@ -28,6 +28,7 @@ import org.apache.aurora.gen.LimitConstraint; import org.apache.aurora.gen.TaskConfig; import org.apache.aurora.gen.TaskConstraint; import org.apache.aurora.gen.ValueConstraint; +import org.apache.aurora.scheduler.configuration.ConfigurationManager.TaskDescriptionException; import org.apache.aurora.scheduler.storage.entities.ITaskConfig; import org.junit.Test; @@ -110,11 +111,21 @@ public class ConfigurationManagerTest { assertTrue(copy.isSetKey()); } - @Test(expected = ConfigurationManager.TaskDescriptionException.class) - public void testBadContainerConfig() throws ConfigurationManager.TaskDescriptionException { + @Test(expected = TaskDescriptionException.class) + public void testBadContainerConfig() throws TaskDescriptionException { TaskConfig taskConfig = CONFIG_WITH_CONTAINER.deepCopy(); taskConfig.getContainer().getDocker().setImage(null); ConfigurationManager.validateAndPopulate(ITaskConfig.build(taskConfig)); } + + @Test(expected = TaskDescriptionException.class) + public void testInvalidTier() throws TaskDescriptionException { + ITaskConfig config = ITaskConfig.build(UNSANITIZED_JOB_CONFIGURATION.deepCopy().getTaskConfig() + .setJobName("job") + .setEnvironment("env") + .setTier("pr/d")); + + ConfigurationManager.validateAndPopulate(config); + } } http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/test/python/apache/aurora/client/cli/test_create.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/client/cli/test_create.py b/src/test/python/apache/aurora/client/cli/test_create.py index 69039b6..8c27e2b 100644 --- a/src/test/python/apache/aurora/client/cli/test_create.py +++ b/src/test/python/apache/aurora/client/cli/test_create.py @@ -404,10 +404,7 @@ class TestClientCreateCommand(AuroraClientCommandTest): fp.name]) assert result == EXIT_INVALID_CONFIGURATION assert mock_context.get_out() == [] - assert mock_context.get_err() == [ - "Error loading configuration: " - "TypeCheck(FAILED): MesosJob[update_config] failed: " - "UpdateConfig[batch_size] failed: u'{{TEST_BATCH}}' not an integer"] + assert "Error loading configuration: TypeCheck(FAILED):" in mock_context.get_err()[0] def test_create_cron_job_fails(self): """Test a cron job is not accepted.""" http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/test/python/apache/aurora/client/test_config.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/client/test_config.py b/src/test/python/apache/aurora/client/test_config.py index 986061b..37459f5 100644 --- a/src/test/python/apache/aurora/client/test_config.py +++ b/src/test/python/apache/aurora/client/test_config.py @@ -138,9 +138,11 @@ def test_include(): get_aurora_config('hello_world', hello_include_fname_fp) +BAD_ENV = ('Prod', ' prod', 'prod ', 'tEst', 'production', 'staging 2', 'stagingA') +GOOD_ENV = ('prod', 'devel', 'test', 'staging', 'staging001', 'staging1', 'staging1234') + + def test_environment_names(): - BAD = ('Prod', ' prod', 'prod ', 'tEst', 'production', 'staging 2', 'stagingA') - GOOD = ('prod', 'devel', 'test', 'staging', 'staging001', 'staging1', 'staging1234') base_job = Job( name='hello_world', role='john_doe', cluster='test-cluster', task=Task(name='main', processes=[], @@ -148,13 +150,28 @@ def test_environment_names(): with pytest.raises(ValueError): config._validate_environment_name(AuroraConfig(base_job)) - for env_name in GOOD: + for env_name in GOOD_ENV: config._validate_environment_name(AuroraConfig(base_job(environment=env_name))) - for env_name in BAD: + for env_name in BAD_ENV: with pytest.raises(ValueError): config._validate_environment_name(AuroraConfig(base_job(environment=env_name))) +def test_tier_names(): + base_job = Job( + name='hello_world', role='john_doe', cluster='test-cluster', + task=Task(name='main', processes=[])) + + # Make sure empty value does not raise. + config._validate_tier(AuroraConfig(base_job)) + + for tier in GOOD_ENV: + config._validate_tier(AuroraConfig(base_job(tier=tier))) + for tier in BAD_ENV: + with pytest.raises(ValueError): + config._validate_tier(AuroraConfig(base_job(tier=tier))) + + def test_dedicated_portmap(): base_job = Job( name='hello_world', role='john_doe', cluster='test-cluster', http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/test/python/apache/aurora/config/test_thrift.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/config/test_thrift.py b/src/test/python/apache/aurora/config/test_thrift.py index 061864e..0c0432c 100644 --- a/src/test/python/apache/aurora/config/test_thrift.py +++ b/src/test/python/apache/aurora/config/test_thrift.py @@ -71,6 +71,13 @@ def test_simple_config(): assert tti.constraints == set() assert tti.metadata == set() assert tti.environment == HELLO_WORLD.environment().get() + assert tti.tier is None + + +def test_config_with_tier(): + config = HELLO_WORLD(tier='devel') + job = convert_pystachio_to_thrift(config) + assert job.taskConfig.tier == 'devel' def test_docker_with_parameters():
