Repository: aurora Updated Branches: refs/heads/master 06d8666f0 -> 02df5eeea
Deleting unused function deprecation_warning Adding test coverage to base, job_monitor, quota_check, scheduler_client Bugs closed: AURORA-1536 Reviewed at https://reviews.apache.org/r/39958/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/02df5eee Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/02df5eee Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/02df5eee Branch: refs/heads/master Commit: 02df5eeea51aa8b8ac8ed22635c29cd66c81f90b Parents: 06d8666 Author: Dmitriy Shirchenko <[email protected]> Authored: Mon Nov 9 18:17:36 2015 -0800 Committer: Bill Farner <[email protected]> Committed: Mon Nov 9 18:17:36 2015 -0800 ---------------------------------------------------------------------- src/main/python/apache/aurora/client/base.py | 10 - src/test/python/apache/aurora/client/BUILD | 1 + .../aurora/client/api/test_job_monitor.py | 7 + .../aurora/client/api/test_quota_check.py | 13 + .../aurora/client/api/test_scheduler_client.py | 282 ++++++++++--------- .../python/apache/aurora/client/test_base.py | 70 ++++- 6 files changed, 238 insertions(+), 145 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/02df5eee/src/main/python/apache/aurora/client/base.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/client/base.py b/src/main/python/apache/aurora/client/base.py index 487f8e7..91c276b 100644 --- a/src/main/python/apache/aurora/client/base.py +++ b/src/main/python/apache/aurora/client/base.py @@ -68,16 +68,6 @@ def check_and_log_locked_response(resp): log.info(LOCKED_WARNING) -def deprecation_warning(text): - log.warning('') - log.warning('*' * 80) - log.warning('* The command you ran is deprecated and will soon break!') - for line in text.split('\n'): - log.warning('* %s' % line) - log.warning('*' * 80) - log.warning('') - - class requires(object): # noqa @classmethod def wrap_function(cls, fn, fnargs, comparator): http://git-wip-us.apache.org/repos/asf/aurora/blob/02df5eee/src/test/python/apache/aurora/client/BUILD ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/client/BUILD b/src/test/python/apache/aurora/client/BUILD index dc7912e..84c5c84 100644 --- a/src/test/python/apache/aurora/client/BUILD +++ b/src/test/python/apache/aurora/client/BUILD @@ -37,6 +37,7 @@ python_tests( python_tests(name = 'base', sources = ['test_base.py'], dependencies = [ + '3rdparty/python:mock', 'src/main/python/apache/aurora/client', ], ) http://git-wip-us.apache.org/repos/asf/aurora/blob/02df5eee/src/test/python/apache/aurora/client/api/test_job_monitor.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/client/api/test_job_monitor.py b/src/test/python/apache/aurora/client/api/test_job_monitor.py index ccc8b55..537abd3 100644 --- a/src/test/python/apache/aurora/client/api/test_job_monitor.py +++ b/src/test/python/apache/aurora/client/api/test_job_monitor.py @@ -13,6 +13,7 @@ # import unittest +import mock from mock import create_autospec from apache.aurora.client.api.job_monitor import JobMonitor @@ -126,3 +127,9 @@ class JobMonitorTest(unittest.TestCase): self._event.set() monitor = JobMonitor(self._scheduler, self._job_key, terminating_event=self._event) assert monitor.wait_until(monitor.terminal) + + def test_terminate(self): + mock_event = mock.Mock() + monitor = JobMonitor(self._scheduler, self._job_key, terminating_event=mock_event) + monitor.terminate() + mock_event.set.assert_called_once_with() http://git-wip-us.apache.org/repos/asf/aurora/blob/02df5eee/src/test/python/apache/aurora/client/api/test_quota_check.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/client/api/test_quota_check.py b/src/test/python/apache/aurora/client/api/test_quota_check.py index 7035bb5..0b97334 100644 --- a/src/test/python/apache/aurora/client/api/test_quota_check.py +++ b/src/test/python/apache/aurora/client/api/test_quota_check.py @@ -127,3 +127,16 @@ class QuotaCheckTest(unittest.TestCase): call((acquired - released).quota(), 'Requested', self._name), call(additional, 'Additional quota required', self._role) ] + + def test_op_not_cap_request(self): + released = CapacityRequest(ResourceAggregate(numCpus=5.0, ramMb=100, diskMb=100)) + # Should return self so a noop. + out = released._op('not_a_real_op', 'not_a_CapacityRequest_obj') + self.assertEqual(out, released) + + def test_right_add(self): + released = CapacityRequest(ResourceAggregate(numCpus=5.0, ramMb=100, diskMb=100)) + # Testing __radd__ which is called when object on the ride side isn't our CapacityRequest + # instance. + out = 1 + released + self.assertEqual(out, released) http://git-wip-us.apache.org/repos/asf/aurora/blob/02df5eee/src/test/python/apache/aurora/client/api/test_scheduler_client.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/client/api/test_scheduler_client.py b/src/test/python/apache/aurora/client/api/test_scheduler_client.py index 44671fa..d244b3a 100644 --- a/src/test/python/apache/aurora/client/api/test_scheduler_client.py +++ b/src/test/python/apache/aurora/client/api/test_scheduler_client.py @@ -389,137 +389,151 @@ def test_url_when_not_connected_and_cluster_has_no_proxy_url(scheme): client._connect_scheduler.assert_has_calls([]) [email protected]('apache.aurora.client.api.scheduler_client.TRequestsTransport', spec=TRequestsTransport) -def test_connect_scheduler(mock_client): - mock_client.return_value.open.side_effect = [TTransport.TTransportException, True] - mock_time = mock.create_autospec(spec=time, instance=True) - - client = scheduler_client.SchedulerClient(mock_auth(), 'Some-User-Agent', verbose=True) - client._connect_scheduler('https://scheduler.example.com:1337', mock_time) - - assert mock_client.return_value.open.has_calls(mock.call(), mock.call()) - mock_time.sleep.assert_called_once_with( - scheduler_client.SchedulerClient.RETRY_TIMEOUT.as_(Time.SECONDS)) - - [email protected]('apache.aurora.client.api.scheduler_client.TRequestsTransport', spec=TRequestsTransport) -def test_connect_scheduler_with_user_agent(mock_transport): - mock_transport.return_value.open.side_effect = [TTransport.TTransportException, True] - mock_time = mock.create_autospec(spec=time, instance=True) - - auth = mock_auth() - user_agent = 'Some-User-Agent' - - client = scheduler_client.SchedulerClient(auth, user_agent, verbose=True) - - uri = 'https://scheduler.example.com:1337' - client._connect_scheduler(uri, mock_time) - - mock_transport.assert_called_once_with(uri, auth=auth.auth(), user_agent=user_agent) - - [email protected]('apache.aurora.client.api.scheduler_client.SchedulerClient', - spec=scheduler_client.SchedulerClient) [email protected]('threading._Event.wait') -def test_transient_error(_, client): - mock_scheduler_client = mock.create_autospec( - spec=scheduler_client.SchedulerClient, - spec_set=False, - instance=True) - mock_thrift_client = mock.create_autospec(spec=AuroraAdmin.Client, instance=True) - mock_thrift_client.killTasks.side_effect = [ - Response(responseCode=ResponseCode.ERROR_TRANSIENT, - details=[ResponseDetail(message="message1"), ResponseDetail(message="message2")]), - Response(responseCode=ResponseCode.ERROR_TRANSIENT), - Response(responseCode=ResponseCode.OK)] - - mock_scheduler_client.get_thrift_client.return_value = mock_thrift_client - client.get.return_value = mock_scheduler_client - - proxy = TestSchedulerProxy(Cluster(name='local')) - proxy.killTasks(TaskQuery(), None) - - assert mock_thrift_client.killTasks.call_count == 3 - - [email protected]('apache.aurora.client.api.scheduler_client.SchedulerClient', - spec=scheduler_client.SchedulerClient) -def test_unknown_connection_error(client): - mock_scheduler_client = mock.create_autospec(spec=scheduler_client.SchedulerClient, instance=True) - client.get.return_value = mock_scheduler_client - proxy = TestSchedulerProxy(Cluster(name='local')) - - # unknown, transient connection error - mock_scheduler_client.get_thrift_client.side_effect = RuntimeError - with pytest.raises(Exception): - proxy.client() - - # successful connection on re-attempt - mock_scheduler_client.get_thrift_client.side_effect = None - assert proxy.client() is not None - - [email protected]('apache.aurora.client.api.scheduler_client.TRequestsTransport', spec=TRequestsTransport) -def test_connect_direct_scheduler_with_user_agent(mock_transport): - mock_transport.return_value.open.side_effect = [TTransport.TTransportException, True] - mock_time = mock.create_autospec(spec=time, instance=True) - - auth = mock_auth() - user_agent = 'Some-User-Agent' - uri = 'https://scheduler.example.com:1337' - - client = scheduler_client.DirectSchedulerClient( - uri, - auth=auth, - verbose=True, - user_agent=user_agent) - - client._connect_scheduler(uri, mock_time) - - mock_transport.assert_called_once_with(uri, auth=auth.auth(), user_agent=user_agent) - - [email protected]('apache.aurora.client.api.scheduler_client.TRequestsTransport', spec=TRequestsTransport) -def test_connect_zookeeper_client_with_auth(mock_transport): - mock_transport.return_value.open.side_effect = [TTransport.TTransportException, True] - mock_time = mock.create_autospec(spec=time, instance=True) - - user_agent = 'Some-User-Agent' - uri = 'https://scheduler.example.com:1337' - auth = mock_auth() - cluster = Cluster(zk='zk', zk_port='2181') - - def auth_factory(_): - return auth - - client = scheduler_client.SchedulerClient.get( - cluster, - auth_factory=auth_factory, - user_agent=user_agent) - - client._connect_scheduler(uri, mock_time) - - mock_transport.assert_called_once_with(uri, auth=auth.auth(), user_agent=user_agent) - - [email protected]('apache.aurora.client.api.scheduler_client.TRequestsTransport', spec=TRequestsTransport) -def test_connect_direct_client_with_auth(mock_transport): - mock_transport.return_value.open.side_effect = [TTransport.TTransportException, True] - mock_time = mock.create_autospec(spec=time, instance=True) - - user_agent = 'Some-User-Agent' - uri = 'https://scheduler.example.com:1337' - auth = mock_auth() - cluster = Cluster(scheduler_uri='uri') - - def auth_factory(_): - return auth - - client = scheduler_client.SchedulerClient.get( - cluster, - auth_factory=auth_factory, - user_agent=user_agent) - - client._connect_scheduler(uri, mock_time) - - mock_transport.assert_called_once_with(uri, auth=auth.auth(), user_agent=user_agent) +class TestSchedulerClient(unittest.TestCase): + + @mock.patch('apache.aurora.client.api.scheduler_client.TRequestsTransport', + spec=TRequestsTransport) + def test_connect_scheduler(self, mock_client): + mock_client.return_value.open.side_effect = [TTransport.TTransportException, True] + mock_time = mock.create_autospec(spec=time, instance=True) + + client = scheduler_client.SchedulerClient(mock_auth(), 'Some-User-Agent', verbose=True) + client._connect_scheduler('https://scheduler.example.com:1337', mock_time) + + assert mock_client.return_value.open.has_calls(mock.call(), mock.call()) + mock_time.sleep.assert_called_once_with( + scheduler_client.SchedulerClient.RETRY_TIMEOUT.as_(Time.SECONDS)) + + @mock.patch('apache.aurora.client.api.scheduler_client.TRequestsTransport', + spec=TRequestsTransport) + def test_connect_scheduler_with_user_agent(self, mock_transport): + mock_transport.return_value.open.side_effect = [TTransport.TTransportException, True] + mock_time = mock.create_autospec(spec=time, instance=True) + + auth = mock_auth() + user_agent = 'Some-User-Agent' + + client = scheduler_client.SchedulerClient(auth, user_agent, verbose=True) + + uri = 'https://scheduler.example.com:1337' + client._connect_scheduler(uri, mock_time) + + mock_transport.assert_called_once_with(uri, auth=auth.auth(), user_agent=user_agent) + + @mock.patch('apache.aurora.client.api.scheduler_client.SchedulerClient', + spec=scheduler_client.SchedulerClient) + @mock.patch('threading._Event.wait') + def test_transient_error(self, _, client): + mock_scheduler_client = mock.create_autospec( + spec=scheduler_client.SchedulerClient, + spec_set=False, + instance=True) + mock_thrift_client = mock.create_autospec(spec=AuroraAdmin.Client, instance=True) + mock_thrift_client.killTasks.side_effect = [ + Response(responseCode=ResponseCode.ERROR_TRANSIENT, + details=[ResponseDetail(message="message1"), ResponseDetail(message="message2")]), + Response(responseCode=ResponseCode.ERROR_TRANSIENT), + Response(responseCode=ResponseCode.OK)] + + mock_scheduler_client.get_thrift_client.return_value = mock_thrift_client + client.get.return_value = mock_scheduler_client + + proxy = TestSchedulerProxy(Cluster(name='local')) + proxy.killTasks(TaskQuery(), None) + + assert mock_thrift_client.killTasks.call_count == 3 + + @mock.patch('apache.aurora.client.api.scheduler_client.SchedulerClient', + spec=scheduler_client.SchedulerClient) + def test_unknown_connection_error(self, client): + mock_scheduler_client = mock.create_autospec(spec=scheduler_client.SchedulerClient, + instance=True) + client.get.return_value = mock_scheduler_client + proxy = TestSchedulerProxy(Cluster(name='local')) + + # unknown, transient connection error + mock_scheduler_client.get_thrift_client.side_effect = RuntimeError + with pytest.raises(Exception): + proxy.client() + + # successful connection on re-attempt + mock_scheduler_client.get_thrift_client.side_effect = None + assert proxy.client() is not None + + @mock.patch('apache.aurora.client.api.scheduler_client.TRequestsTransport', + spec=TRequestsTransport) + def test_connect_direct_scheduler_with_user_agent(self, mock_transport): + mock_transport.return_value.open.side_effect = [TTransport.TTransportException, True] + mock_time = mock.create_autospec(spec=time, instance=True) + + auth = mock_auth() + user_agent = 'Some-User-Agent' + uri = 'https://scheduler.example.com:1337' + + client = scheduler_client.DirectSchedulerClient( + uri, + auth=auth, + verbose=True, + user_agent=user_agent) + + client._connect_scheduler(uri, mock_time) + + mock_transport.assert_called_once_with(uri, auth=auth.auth(), user_agent=user_agent) + + @mock.patch('apache.aurora.client.api.scheduler_client.TRequestsTransport', + spec=TRequestsTransport) + def test_connect_zookeeper_client_with_auth(self, mock_transport): + mock_transport.return_value.open.side_effect = [TTransport.TTransportException, True] + mock_time = mock.create_autospec(spec=time, instance=True) + + user_agent = 'Some-User-Agent' + uri = 'https://scheduler.example.com:1337' + auth = mock_auth() + cluster = Cluster(zk='zk', zk_port='2181') + + def auth_factory(_): + return auth + + client = scheduler_client.SchedulerClient.get( + cluster, + auth_factory=auth_factory, + user_agent=user_agent) + + client._connect_scheduler(uri, mock_time) + + mock_transport.assert_called_once_with(uri, auth=auth.auth(), user_agent=user_agent) + + @mock.patch('apache.aurora.client.api.scheduler_client.TRequestsTransport', + spec=TRequestsTransport) + def test_connect_direct_client_with_auth(self, mock_transport): + mock_transport.return_value.open.side_effect = [TTransport.TTransportException, True] + mock_time = mock.create_autospec(spec=time, instance=True) + + user_agent = 'Some-User-Agent' + uri = 'https://scheduler.example.com:1337' + auth = mock_auth() + cluster = Cluster(scheduler_uri='uri') + + def auth_factory(_): + return auth + + client = scheduler_client.SchedulerClient.get( + cluster, + auth_factory=auth_factory, + user_agent=user_agent) + + client._connect_scheduler(uri, mock_time) + + mock_transport.assert_called_once_with(uri, auth=auth.auth(), user_agent=user_agent) + + def test_no_zk_or_scheduler_uri(self): + cluster = None + with self.assertRaises(TypeError): + scheduler_client.SchedulerClient.get(cluster) + cluster = Cluster() + with self.assertRaises(ValueError): + scheduler_client.SchedulerClient.get(cluster) + + def test__internal_connect(self): + client = scheduler_client.SchedulerClient(mock_auth(), 'Some-User-Agent', verbose=True) + self.assertIsNone(client._connect()) http://git-wip-us.apache.org/repos/asf/aurora/blob/02df5eee/src/test/python/apache/aurora/client/test_base.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/client/test_base.py b/src/test/python/apache/aurora/client/test_base.py index fa5eb07..8c88008 100644 --- a/src/test/python/apache/aurora/client/test_base.py +++ b/src/test/python/apache/aurora/client/test_base.py @@ -13,7 +13,10 @@ # import unittest +import mock + from apache.aurora.client import base +from apache.aurora.common.pex_version import UnknownVersion from gen.apache.aurora.api.ttypes import ( PopulateJobResult, @@ -57,7 +60,9 @@ class TestBase(unittest.TestCase): taskConfig=config))) assert config == resp.result.populateJobResult.taskConfig - def test_synthesize_url(self): + @mock.patch('apache.aurora.client.base.die') + @mock.patch('apache.aurora.client.base.log') + def test_synthesize_url(self, mock_log, mock_die): base_url = 'http://example.com' role = 'some-role' environment = 'some-environment' @@ -78,3 +83,66 @@ class TestBase(unittest.TestCase): assert (('%s/scheduler/%s' % (base_url, role)) == base.synthesize_url(base_url, role)) + + mock_log.reset_mock() + scheduler_url = '' + out = base.synthesize_url(scheduler_url) + self.assertIsNone(out) + mock_log.warning.assert_called_once_with('Unable to find scheduler web UI!') + + mock_log.reset_mock() + mock_die.reset_mock() + scheduler_url = 'foo' + env = 'foo' + out = base.synthesize_url(scheduler_url, env=env) + expected = 'scheduler' + self.assertEqual(out, expected) + mock_die.assert_called_once_with('If env specified, must specify role') + + mock_log.reset_mock() + mock_die.reset_mock() + scheduler_url = 'foo' + job = 'bar' + out = base.synthesize_url(scheduler_url, job=job) + expected = 'scheduler' + self.assertEqual(out, expected) + mock_die.assert_called_once_with('If job specified, must specify role and env') + + @mock.patch('apache.aurora.client.base.pex_version') + def test_user_agent(self, mock_pex_version): + mock_pex_version.return_value = ('sha', '2015-11-3') + out = base.user_agent() + expected = 'Aurora;sha-2015-11-3' + self.assertEqual(out, expected) + + mock_pex_version.reset_mock() + mock_pex_version.side_effect = UnknownVersion + out = base.user_agent() + expected = 'Aurora;Unknown Version' + self.assertEqual(out, expected) + + @mock.patch('apache.aurora.client.base.log.info') + @mock.patch('apache.aurora.client.base.sys.exit') + def test_check_and_log_response(self, mock_sys_exit, mock_log): + resp = Response(responseCode=ResponseCode.LOCK_ERROR) + out = base.check_and_log_response(resp) + self.assertIsNone(out) + mock_sys_exit.assert_called_once_with(1) + mock_log.assert_any_call(base.LOCKED_WARNING) + mock_log.assert_any_call('Response from scheduler: LOCK_ERROR (message: )') + + @mock.patch('apache.aurora.client.base.log.info') + def test_check_and_log_locked_response(self, mock_log): + resp = Response(responseCode=ResponseCode.LOCK_ERROR) + out = base.check_and_log_locked_response(resp) + self.assertIsNone(out) + mock_log.assert_called_once_with(base.LOCKED_WARNING) + + @mock.patch('apache.aurora.client.base.sys.exit') + @mock.patch('apache.aurora.client.base.log.fatal') + def test_die(self, mock_log, mock_sys_exit): + msg = 'fatal message' + out = base.die(msg) + self.assertIsNone(out) + mock_sys_exit.assert_called_once_with(1) + mock_log.assert_called_once_with(msg)
