Repository: aurora Updated Branches: refs/heads/master 0f3dc939e -> 21af250c9
Remove task level resource fields from thrift interface Bugs closed: AURORA-1707 Reviewed at https://reviews.apache.org/r/60942/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/21af250c Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/21af250c Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/21af250c Branch: refs/heads/master Commit: 21af250c9e781d7d4a121c74f561cbda36245040 Parents: 0f3dc93 Author: Nicolás Donatucci <ndonatu...@medallia.com> Authored: Mon Nov 27 17:01:03 2017 -0800 Committer: Bill Farner <wfar...@apache.org> Committed: Mon Nov 27 17:01:03 2017 -0800 ---------------------------------------------------------------------- RELEASE-NOTES.md | 1 + .../thrift/org/apache/aurora/gen/api.thrift | 8 --- .../aurora/scheduler/base/TaskTestUtil.java | 4 -- .../apache/aurora/client/cli/diff_formatter.py | 5 +- .../python/apache/aurora/client/cli/jobs.py | 6 +- .../apache/aurora/scheduler/http/TestUtils.java | 13 ++++- .../aurora/scheduler/thrift/Fixtures.java | 4 -- .../updater/UpdateAgentReserverImplTest.java | 8 ++- .../apache/aurora/client/cli/test_status.py | 59 +++++++++++++------- .../python/apache/aurora/client/cli/util.py | 8 ++- .../apache/aurora/config/test_resources.py | 11 +--- 11 files changed, 64 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/21af250c/RELEASE-NOTES.md ---------------------------------------------------------------------- diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index e622a8d..2d3c423 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -17,6 +17,7 @@ - Removed the ability to recover from SQL-based backups and snapshots. An 0.20.0 scheduler will not be able to recover backups or replicated log data created prior to 0.19.0. +- Removed task level resource fields (`numCpus`, `ramMb`, `diskMb`, `requestedPorts`). 0.19.0 ====== http://git-wip-us.apache.org/repos/asf/aurora/blob/21af250c/api/src/main/thrift/org/apache/aurora/gen/api.thrift ---------------------------------------------------------------------- diff --git a/api/src/main/thrift/org/apache/aurora/gen/api.thrift b/api/src/main/thrift/org/apache/aurora/gen/api.thrift index 2978f6d..c9f4210 100644 --- a/api/src/main/thrift/org/apache/aurora/gen/api.thrift +++ b/api/src/main/thrift/org/apache/aurora/gen/api.thrift @@ -232,12 +232,6 @@ struct TaskConfig { /** contains the role component of JobKey */ 17: Identity owner 7: bool isService - // TODO(maxim): Deprecated. See AURORA-1707. - 8: double numCpus - // TODO(maxim): Deprecated. See AURORA-1707. - 9: i64 ramMb - // TODO(maxim): Deprecated. See AURORA-1707. - 10: i64 diskMb 11: i32 priority 13: i32 maxTaskFailures // TODO(mnurolahzade): Deprecated. See AURORA-1708. @@ -249,8 +243,6 @@ struct TaskConfig { 32: set<Resource> resources 20: set<Constraint> constraints - /** a list of named ports this task requests */ - 21: set<string> requestedPorts /** Resources to retrieve with Mesos Fetcher */ 33: optional set<MesosFetcherURI> mesosFetcherUris /** http://git-wip-us.apache.org/repos/asf/aurora/blob/21af250c/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java b/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java index dd64a5b..5fe7b9b 100644 --- a/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java +++ b/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java @@ -116,9 +116,6 @@ public final class TaskTestUtil { .setJob(job.newBuilder()) .setOwner(new Identity().setUser(job.getRole() + "-user")) .setIsService(true) - .setNumCpus(1.0) - .setRamMb(1024) - .setDiskMb(1024) .setPriority(1) .setMaxTaskFailures(-1) .setProduction(true) @@ -132,7 +129,6 @@ public final class TaskTestUtil { new Constraint( "limitConstraint", TaskConstraint.limit(new LimitConstraint(5))))) - .setRequestedPorts(ImmutableSet.of("http")) .setTaskLinks(ImmutableMap.of("http", "link", "admin", "otherLink")) .setContactEmail("f...@bar.com") .setMetadata(ImmutableSet.of(new Metadata("key", "value"))) http://git-wip-us.apache.org/repos/asf/aurora/blob/21af250c/src/main/python/apache/aurora/client/cli/diff_formatter.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/client/cli/diff_formatter.py b/src/main/python/apache/aurora/client/cli/diff_formatter.py index 7871777..2f66e29 100644 --- a/src/main/python/apache/aurora/client/cli/diff_formatter.py +++ b/src/main/python/apache/aurora/client/cli/diff_formatter.py @@ -63,9 +63,6 @@ class DiffFormatter(object): if task.resources: task.resources = sorted(task.resources, key=str) - if task.requestedPorts: - task.requestedPorts = sorted(task.requestedPorts, key=str) - if task.mesosFetcherUris: task.mesosFetcherUris = sorted(task.mesosFetcherUris, key=str) @@ -129,6 +126,8 @@ class DiffFormatter(object): format_ranges(r for r in chain.from_iterable(s.instances for s in summaries))) def diff_no_update_details(self, local_tasks): + # Deepcopy is important here as tasks will be modified for printing. + local_tasks = [deepcopy(t) for t in local_tasks] api = self.context.get_api(self.cluster) resp = api.query(api.build_query(self.role, self.name, env=self.env, statuses=ACTIVE_STATES)) self.context.log_response_and_raise( http://git-wip-us.apache.org/repos/asf/aurora/blob/21af250c/src/main/python/apache/aurora/client/cli/jobs.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/client/cli/jobs.py b/src/main/python/apache/aurora/client/cli/jobs.py index cbae387..536d04a 100644 --- a/src/main/python/apache/aurora/client/cli/jobs.py +++ b/src/main/python/apache/aurora/client/cli/jobs.py @@ -20,7 +20,6 @@ import pprint import textwrap import webbrowser from collections import namedtuple -from copy import deepcopy from datetime import datetime from thrift.protocol import TJSONProtocol @@ -201,10 +200,7 @@ class DiffCommand(Verb): err_code=EXIT_INVALID_CONFIGURATION, err_msg="Error loading configuration") local_task = resp.result.populateJobResult.taskConfig - # Deepcopy is important here as tasks will be modified for printing. - local_tasks = [ - deepcopy(local_task) for _ in range(config.instances()) - ] + local_tasks = [local_task for _ in range(config.instances())] instances = (None if context.options.instance_spec.instance == ALL_INSTANCES else context.options.instance_spec.instance) formatter = DiffFormatter(context, config, cluster, role, env, name) http://git-wip-us.apache.org/repos/asf/aurora/blob/21af250c/src/test/java/org/apache/aurora/scheduler/http/TestUtils.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/http/TestUtils.java b/src/test/java/org/apache/aurora/scheduler/http/TestUtils.java index 689482c..eb6b39f 100644 --- a/src/test/java/org/apache/aurora/scheduler/http/TestUtils.java +++ b/src/test/java/org/apache/aurora/scheduler/http/TestUtils.java @@ -15,6 +15,8 @@ package org.apache.aurora.scheduler.http; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; + import org.apache.aurora.gen.AssignedTask; import org.apache.aurora.gen.ScheduleStatus; import org.apache.aurora.gen.ScheduledTask; @@ -22,6 +24,10 @@ import org.apache.aurora.gen.TaskConfig; import org.apache.aurora.scheduler.storage.entities.IJobKey; import org.apache.aurora.scheduler.storage.entities.IScheduledTask; +import static org.apache.aurora.gen.Resource.diskMb; +import static org.apache.aurora.gen.Resource.numCpus; +import static org.apache.aurora.gen.Resource.ramMb; + public final class TestUtils { private TestUtils() { } @@ -49,8 +55,9 @@ public final class TestUtils { .setTaskId(id) .setTask(new TaskConfig() .setJob(jobKey.newBuilder()) - .setNumCpus(numCPUs) - .setRamMb(ramMB) - .setDiskMb(diskMB)))); + .setResources(ImmutableSet.of( + numCpus(numCPUs), + ramMb(ramMB), + diskMb(diskMB)))))); } } http://git-wip-us.apache.org/repos/asf/aurora/blob/21af250c/src/test/java/org/apache/aurora/scheduler/thrift/Fixtures.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/Fixtures.java b/src/test/java/org/apache/aurora/scheduler/thrift/Fixtures.java index d1b5e40..e1b8eaa 100644 --- a/src/test/java/org/apache/aurora/scheduler/thrift/Fixtures.java +++ b/src/test/java/org/apache/aurora/scheduler/thrift/Fixtures.java @@ -102,12 +102,8 @@ final class Fixtures { .setOwner(IDENTITY) .setContactEmail("test...@twitter.com") .setExecutorConfig(new ExecutorConfig(apiConstants.AURORA_EXECUTOR_NAME, "data")) - .setNumCpus(1) - .setRamMb(1024) - .setDiskMb(1024) .setProduction(production) .setTier(production ? TaskTestUtil.PROD_TIER_NAME : TaskTestUtil.DEV_TIER_NAME) - .setRequestedPorts(ImmutableSet.of()) .setTaskLinks(ImmutableMap.of()) .setMaxTaskFailures(1) .setConstraints(ImmutableSet.of()) http://git-wip-us.apache.org/repos/asf/aurora/blob/21af250c/src/test/java/org/apache/aurora/scheduler/updater/UpdateAgentReserverImplTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/updater/UpdateAgentReserverImplTest.java b/src/test/java/org/apache/aurora/scheduler/updater/UpdateAgentReserverImplTest.java index 1bc2a77..7f17be0 100644 --- a/src/test/java/org/apache/aurora/scheduler/updater/UpdateAgentReserverImplTest.java +++ b/src/test/java/org/apache/aurora/scheduler/updater/UpdateAgentReserverImplTest.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.apache.aurora.common.testing.easymock.EasyMockTest; +import org.apache.aurora.gen.Resource; import org.apache.aurora.gen.TaskConfig; import org.apache.aurora.scheduler.base.InstanceKeys; import org.apache.aurora.scheduler.base.JobKeys; @@ -46,9 +47,10 @@ public class UpdateAgentReserverImplTest extends EasyMockTest { return TaskGroupKey.from(ITaskConfig.build( new TaskConfig() .setJob(key.getJobKey().newBuilder()) - .setNumCpus(1.0) - .setRamMb(1L) - .setDiskMb(1L))); + .setResources(ImmutableSet.of( + Resource.numCpus(1.0), + Resource.ramMb(1L), + Resource.diskMb(1L))))); } @Before http://git-wip-us.apache.org/repos/asf/aurora/blob/21af250c/src/test/python/apache/aurora/client/cli/test_status.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/client/cli/test_status.py b/src/test/python/apache/aurora/client/cli/test_status.py index b0b7f96..d462dc0 100644 --- a/src/test/python/apache/aurora/client/cli/test_status.py +++ b/src/test/python/apache/aurora/client/cli/test_status.py @@ -31,6 +31,7 @@ from gen.apache.aurora.api.ttypes import ( JobConfiguration, JobKey, Metadata, + Resource, ResponseCode, Result, ScheduledTask, @@ -148,13 +149,13 @@ class TestJobStatus(AuroraClientCommandTest): task=TaskConfig( job=JobKey(role="nobody", environment="prod", name='flibber'), isService=False, - numCpus=2, - ramMb=2048, - diskMb=4096, + resources=frozenset( + [Resource(numCpus=2), + Resource(ramMb=2048), + Resource(diskMb=4096)]), priority=7, maxTaskFailures=3, - production=False, - requestedPorts=["http"]), + production=False), assignedPorts={"http": 1001}, instanceId=instance), status=2, @@ -426,6 +427,7 @@ class TestJobStatus(AuroraClientCommandTest): cmd = AuroraCommandLine() cmd.execute(['job', 'status', '--write-json', 'west/bozo/test/hello']) actual = re.sub("\\d\\d:\\d\\d:\\d\\d", "##:##:##", '\n'.join(mock_context.get_out())) + actual_sorted = json.loads(actual) expected = [ { "active": [ @@ -437,9 +439,7 @@ class TestJobStatus(AuroraClientCommandTest): "container": { "mesos": {} }, - "requestedPorts": [ - "http" - ], + "maxTaskFailures": 3, "priority": 7, "job": { "environment": "prod", @@ -447,10 +447,17 @@ class TestJobStatus(AuroraClientCommandTest): "name": "flibber" }, "production": False, - "diskMb": 4096, - "ramMb": 2048, - "maxTaskFailures": 3, - "numCpus": 2 + "resources": [ + { + "numCpus": 2 + }, + { + "ramMb": 2048 + }, + { + "diskMb": 4096 + } + ] }, "taskId": "task_0", "instanceId": 0, @@ -488,9 +495,7 @@ class TestJobStatus(AuroraClientCommandTest): "container": { "mesos": {} }, - "requestedPorts": [ - "http" - ], + "maxTaskFailures": 3, "priority": 7, "job": { "environment": "prod", @@ -498,10 +503,17 @@ class TestJobStatus(AuroraClientCommandTest): "name": "flibber" }, "production": False, - "diskMb": 4096, - "ramMb": 2048, - "maxTaskFailures": 3, - "numCpus": 2 + "resources": [ + { + "numCpus": 2 + }, + { + "ramMb": 2048 + }, + { + "diskMb": 4096 + } + ] }, "taskId": "task_1", "instanceId": 1, @@ -536,7 +548,14 @@ class TestJobStatus(AuroraClientCommandTest): "inactive": [] } ] - assert json.loads(actual) == expected + for entry in actual_sorted[0]["active"]: + entry["assignedTask"]["task"]["resources"] = sorted( + entry["assignedTask"]["task"]["resources"], key=str) + for entry in expected[0]["active"]: + entry["assignedTask"]["task"]["resources"] = sorted( + entry["assignedTask"]["task"]["resources"], key=str) + + assert actual_sorted == expected def test_status_job_not_found(self): """Regression test: there was a crasher bug when metadata was None.""" http://git-wip-us.apache.org/repos/asf/aurora/blob/21af250c/src/test/python/apache/aurora/client/cli/util.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/client/cli/util.py b/src/test/python/apache/aurora/client/cli/util.py index 476b8c2..8b3b4f5 100644 --- a/src/test/python/apache/aurora/client/cli/util.py +++ b/src/test/python/apache/aurora/client/cli/util.py @@ -32,6 +32,7 @@ from gen.apache.aurora.api.ttypes import ( GetTierConfigResult, JobKey, JobUpdateSummary, + Resource, Response, ResponseCode, ResponseDetail, @@ -227,9 +228,10 @@ class AuroraClientCommandTest(unittest.TestCase): executorConfig=ExecutorConfig(data='{"fake": "data"}'), metadata=[], job=JobKey(role=cls.TEST_ROLE, environment=cls.TEST_ENV, name=name), - numCpus=2, - ramMb=2, - diskMb=2) + resources=frozenset( + [Resource(numCpus=2), + Resource(ramMb=2), + Resource(diskMb=2)])) @classmethod def create_scheduled_tasks(cls): http://git-wip-us.apache.org/repos/asf/aurora/blob/21af250c/src/test/python/apache/aurora/config/test_resources.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/config/test_resources.py b/src/test/python/apache/aurora/config/test_resources.py index 25a20f0..f43bad7 100644 --- a/src/test/python/apache/aurora/config/test_resources.py +++ b/src/test/python/apache/aurora/config/test_resources.py @@ -15,7 +15,7 @@ import unittest from apache.aurora.config.resource import ResourceDetails, ResourceManager, ResourceType -from gen.apache.aurora.api.ttypes import Resource, ResourceAggregate, TaskConfig +from gen.apache.aurora.api.ttypes import Resource, ResourceAggregate class TestResourceType(unittest.TestCase): @@ -49,12 +49,3 @@ class TestResourceManager(unittest.TestCase): ResourceDetails(ResourceType.RAM_MB, 2), ResourceDetails(ResourceType.DISK_MB, 3) ] - - def test_backfill_task(self): - task = TaskConfig(numCpus=1.0, ramMb=2, diskMb=3, requestedPorts=frozenset(['http'])) - assert ResourceManager.resource_details_from_quota(task) == [ - ResourceDetails(ResourceType.CPUS, 1.0), - ResourceDetails(ResourceType.RAM_MB, 2), - ResourceDetails(ResourceType.DISK_MB, 3), - ResourceDetails(ResourceType.PORTS, 'http'), - ]