Repository: aurora Updated Branches: refs/heads/master 2e1ca4288 -> f1d9caf36
Add metadata field to Job object in DSL Bugs closed: AURORA-1898 Reviewed at https://reviews.apache.org/r/64341/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/f1d9caf3 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/f1d9caf3 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/f1d9caf3 Branch: refs/heads/master Commit: f1d9caf36dea2dbab6ccc44b9ba08a5572d7bbc8 Parents: 2e1ca42 Author: Jing Chen <milantr...@gmail.com> Authored: Sun Dec 17 08:26:33 2017 -0800 Committer: Bill Farner <wfar...@apache.org> Committed: Sun Dec 17 08:26:33 2017 -0800 ---------------------------------------------------------------------- RELEASE-NOTES.md | 1 + docs/reference/configuration.md | 9 +++++ .../python/apache/aurora/config/schema/base.py | 6 +++ src/main/python/apache/aurora/config/thrift.py | 10 ++++- .../apache/aurora/client/cli/test_inspect.py | 1 + .../python/apache/aurora/config/test_thrift.py | 41 ++++++++++++++++++++ 6 files changed, 67 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/f1d9caf3/RELEASE-NOTES.md ---------------------------------------------------------------------- diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 54dcc75..3053e54 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -47,6 +47,7 @@ - The scheduler no longer uses an internal H2 database for storage. - There is a new Scheduler UI which, in addition to the facelift, provides the ability to inject your own custom UI components. +- Introduce a metadata field in the Job object of the DSL, which will populate TaskConfig.metadata. ### Deprecations and removals: http://git-wip-us.apache.org/repos/asf/aurora/blob/f1d9caf3/docs/reference/configuration.md ---------------------------------------------------------------------- diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 67d9914..725e073 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -357,6 +357,7 @@ Job Schema ```announce``` | ```Announcer``` object | Optionally enable Zookeeper ServerSet announcements. See [Announcer Objects] for more information. ```enable_hooks``` | Boolean | Whether to enable [Client Hooks](client-hooks.md) for this job. (Default: False) ```partition_policy``` | ```PartitionPolicy``` object | An optional partition policy that allows job owners to define how to handle partitions for running tasks (in partition-aware Aurora clusters) + ```metadata``` | list of ```Metadata``` objects | list of ```Metadata``` objects for user's customized metadata information. ### UpdateConfig Objects @@ -410,6 +411,14 @@ Parameters for controlling a task's health checks via HTTP or a shell command. | ```reschedule``` | Boolean | Whether or not to reschedule when running tasks become partitioned (Default: True) | ```delay_secs``` | Integer | How long to delay transitioning to LOST when running tasks are partitioned. (Default: 0) +### Metadata Objects + +Describes a piece of user metadata in a key value pair + + param | type | description + ----- | :----: | ----------- + ```key``` | String | Indicate which metadata the user provides + ```value``` | String | Provide the metadata content for corresponding key ### Announcer Objects http://git-wip-us.apache.org/repos/asf/aurora/blob/f1d9caf3/src/main/python/apache/aurora/config/schema/base.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/config/schema/base.py b/src/main/python/apache/aurora/config/schema/base.py index a466e78..3d57d6a 100644 --- a/src/main/python/apache/aurora/config/schema/base.py +++ b/src/main/python/apache/aurora/config/schema/base.py @@ -159,6 +159,11 @@ class PartitionPolicy(Struct): delay_secs = Default(Integer, 0) +class Metadata(Struct): + key = Required(String) + value = Required(String) + + class MesosJob(Struct): name = Default(String, '{{task.name}}') role = Required(String) @@ -176,6 +181,7 @@ class MesosJob(Struct): update_config = Default(UpdateConfig, UpdateConfig()) constraints = Map(String, String) + metadata = Default(List(Metadata), []) service = Default(Boolean, False) max_task_failures = Default(Integer, 1) production = Default(Boolean, False) http://git-wip-us.apache.org/repos/asf/aurora/blob/f1d9caf3/src/main/python/apache/aurora/config/thrift.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/config/thrift.py b/src/main/python/apache/aurora/config/thrift.py index eb00144..dcabb03 100644 --- a/src/main/python/apache/aurora/config/thrift.py +++ b/src/main/python/apache/aurora/config/thrift.py @@ -275,7 +275,15 @@ def convert(job, metadata=frozenset(), ports=frozenset()): fully_interpolated(job.partition_policy().delay_secs())) # Add metadata to a task, to display in the scheduler UI. - task.metadata = frozenset(Metadata(key=str(key), value=str(value)) for key, value in metadata) + metadata_set = frozenset() + if job.has_metadata(): + customized_metadata = job.metadata() + metadata_set |= frozenset( + (str(fully_interpolated(key_value_metadata.key())), + str(fully_interpolated(key_value_metadata.value()))) + for key_value_metadata in customized_metadata) + metadata_set |= frozenset((str(key), str(value)) for key, value in metadata) + task.metadata = frozenset(Metadata(key=key, value=value) for key, value in metadata_set) # task components if not task_raw.has_resources(): http://git-wip-us.apache.org/repos/asf/aurora/blob/f1d9caf3/src/test/python/apache/aurora/client/cli/test_inspect.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/client/cli/test_inspect.py b/src/test/python/apache/aurora/client/cli/test_inspect.py index ecefc18..8c62480 100644 --- a/src/test/python/apache/aurora/client/cli/test_inspect.py +++ b/src/test/python/apache/aurora/client/cli/test_inspect.py @@ -138,6 +138,7 @@ Process 'process': "production": False, "role": "bozo", "contact": "bozo@the.clown", + "metadata": [], "lifecycle": { "http": { "graceful_shutdown_endpoint": "/quitquitquit", http://git-wip-us.apache.org/repos/asf/aurora/blob/f1d9caf3/src/test/python/apache/aurora/config/test_thrift.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/config/test_thrift.py b/src/test/python/apache/aurora/config/test_thrift.py index 76d0ad6..7bf0508 100644 --- a/src/test/python/apache/aurora/config/test_thrift.py +++ b/src/test/python/apache/aurora/config/test_thrift.py @@ -27,6 +27,7 @@ from apache.aurora.config.schema.base import ( HealthCheckConfig, Job, Mesos, + Metadata, Mode, Parameter, SimpleTask, @@ -269,6 +270,46 @@ def test_metadata_in_config(): assert pi.value == '1' +def test_config_with_metadata(): + expected_metadata_tuples = frozenset([("city", "LA"), ("city", "SF")]) + job = convert_pystachio_to_thrift( + HELLO_WORLD(metadata=[ + Metadata(key=key, value=value) + for key, value in expected_metadata_tuples])) + tti = job.taskConfig + + metadata_tuples = frozenset((key_value.key, key_value.value) + for key_value in tti.metadata) + assert metadata_tuples == expected_metadata_tuples + + +def test_config_with_key_collision_metadata(): + input_metadata_tuples = frozenset([("city", "LA")]) + job = convert_pystachio_to_thrift( + HELLO_WORLD(metadata=[ + Metadata(key=key, value=value) + for key, value in input_metadata_tuples]), metadata=[('city', "SF")]) + tti = job.taskConfig + + metadata_tuples = frozenset((key_value.key, key_value.value) + for key_value in tti.metadata) + expected_metadata_tuples = frozenset([("city", "LA"), ("city", "SF")]) + assert metadata_tuples == expected_metadata_tuples + + +def test_config_with_duplicate_metadata(): + expected_metadata_tuples = frozenset([("city", "LA")]) + job = convert_pystachio_to_thrift( + HELLO_WORLD(metadata=[ + Metadata(key=key, value=value) + for key, value in expected_metadata_tuples]), metadata=[('city', "LA")]) + tti = job.taskConfig + + metadata_tuples = frozenset((key_value.key, key_value.value) + for key_value in tti.metadata) + assert metadata_tuples == expected_metadata_tuples + + def test_task_instance_from_job(): instance = task_instance_from_job( Job(health_check_config=HealthCheckConfig(interval_secs=30)), 0, '')