Repository: aurora Updated Branches: refs/heads/master 8def0a90d -> af7e1a731
Fix symlink duplicates from MesosPathDetector. Also adds thread ids to TaskResourceMonitor for better /threads debugging. Testing Done: Verified we realpath the paths prior to returning from path detector. Bugs closed: AURORA-1353 Reviewed at https://reviews.apache.org/r/35580/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/af7e1a73 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/af7e1a73 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/af7e1a73 Branch: refs/heads/master Commit: af7e1a731f7502eff805047bab9cb2c6753fe2ae Parents: 8def0a9 Author: Brian Wickman <[email protected]> Authored: Wed Jun 17 14:30:34 2015 -0700 Committer: Brian Wickman <[email protected]> Committed: Wed Jun 17 14:30:34 2015 -0700 ---------------------------------------------------------------------- .../aurora/executor/common/path_detector.py | 4 +- .../apache/thermos/monitoring/resource.py | 2 +- .../executor/common/test_path_detector.py | 48 ++++++++++++-------- .../apache/thermos/monitoring/test_resource.py | 1 + 4 files changed, 32 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/af7e1a73/src/main/python/apache/aurora/executor/common/path_detector.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/executor/common/path_detector.py b/src/main/python/apache/aurora/executor/common/path_detector.py index ba85d5c..ed264d7 100644 --- a/src/main/python/apache/aurora/executor/common/path_detector.py +++ b/src/main/python/apache/aurora/executor/common/path_detector.py @@ -30,5 +30,5 @@ class MesosPathDetector(PathDetector): def get_paths(self): def iterate(): for scan_result in self._detector: - yield os.path.join(ExecutorDetector.path(scan_result), self._sandbox_path) - return [path for path in iterate() if os.path.exists(path)] + yield os.path.join(os.path.realpath(ExecutorDetector.path(scan_result)), self._sandbox_path) + return list(set(path for path in iterate() if os.path.exists(path))) http://git-wip-us.apache.org/repos/asf/aurora/blob/af7e1a73/src/main/python/apache/thermos/monitoring/resource.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/thermos/monitoring/resource.py b/src/main/python/apache/thermos/monitoring/resource.py index c1d7804..53d0ff1 100644 --- a/src/main/python/apache/thermos/monitoring/resource.py +++ b/src/main/python/apache/thermos/monitoring/resource.py @@ -147,7 +147,7 @@ class TaskResourceMonitor(ResourceMonitorBase, ExceptionalThread): log.debug("Initialising ResourceHistory of length %s" % history_length) self._history = ResourceHistory(history_length) self._kill_signal = threading.Event() - ExceptionalThread.__init__(self) + ExceptionalThread.__init__(self, name='%s[%s]' % (self.__class__.__name__, task_id)) self.daemon = True def sample(self): http://git-wip-us.apache.org/repos/asf/aurora/blob/af7e1a73/src/test/python/apache/aurora/executor/common/test_path_detector.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/executor/common/test_path_detector.py b/src/test/python/apache/aurora/executor/common/test_path_detector.py index 86460bb..7b5ef0c 100644 --- a/src/test/python/apache/aurora/executor/common/test_path_detector.py +++ b/src/test/python/apache/aurora/executor/common/test_path_detector.py @@ -34,27 +34,35 @@ def test_path_detector(): FAKE_ROOT = '/var/blah/blah' FAKE_CHECKPOINT_DIR = 'ckpt' - path1, path2 = ( + path1_symlink, path1, path2 = ( ExecutorDetector.path(Match(ROOTS[0], 'slave001', 'framework1', 'executor1', 'latest')), + ExecutorDetector.path(Match(ROOTS[0], 'slave001', 'framework1', 'executor1', 'asdf-ghjk')), ExecutorDetector.path(Match(ROOTS[1], 'slave002', 'framework2', 'executor2', 'latest')), ) - with mock.patch('glob.glob', return_value=(path1, path2)) as glob: - with mock.patch('os.path.exists', side_effect=(True, False)) as exists: - mpd = MesosPathDetector(root=FAKE_ROOT, sandbox_path=FAKE_CHECKPOINT_DIR) - paths = list(mpd.get_paths()) - assert len(paths) == 1 - assert paths == [os.path.join(path1, FAKE_CHECKPOINT_DIR)] - - expected_glob_pattern = os.path.join(*ExecutorDetector.PATTERN) % { - 'root': FAKE_ROOT, - 'slave_id': '*', - 'framework_id': '*', - 'executor_id': '*', - 'run': '*' - } - glob.assert_called_once_with(expected_glob_pattern) - assert exists.mock_calls == [ - mock.call(os.path.join(path1, FAKE_CHECKPOINT_DIR)), - mock.call(os.path.join(path2, FAKE_CHECKPOINT_DIR)), - ] + with mock.patch('glob.glob', return_value=(path1_symlink, path1, path2)) as glob: + with mock.patch('os.path.realpath', side_effect=(path1, path1, path2)) as realpath: + with mock.patch('os.path.exists', side_effect=(True, True, False)) as exists: + mpd = MesosPathDetector(root=FAKE_ROOT, sandbox_path=FAKE_CHECKPOINT_DIR) + paths = list(mpd.get_paths()) + assert len(paths) == 1 + assert paths == [os.path.join(path1, FAKE_CHECKPOINT_DIR)] + + expected_glob_pattern = os.path.join(*ExecutorDetector.PATTERN) % { + 'root': FAKE_ROOT, + 'slave_id': '*', + 'framework_id': '*', + 'executor_id': '*', + 'run': '*' + } + assert glob.mock_calls == [mock.call(expected_glob_pattern)] + assert realpath.mock_calls == [ + mock.call(os.path.join(path1_symlink)), + mock.call(os.path.join(path1)), + mock.call(os.path.join(path2)), + ] + assert exists.mock_calls == [ + mock.call(os.path.join(path1, FAKE_CHECKPOINT_DIR)), + mock.call(os.path.join(path1, FAKE_CHECKPOINT_DIR)), + mock.call(os.path.join(path2, FAKE_CHECKPOINT_DIR)), + ] http://git-wip-us.apache.org/repos/asf/aurora/blob/af7e1a73/src/test/python/apache/thermos/monitoring/test_resource.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/thermos/monitoring/test_resource.py b/src/test/python/apache/thermos/monitoring/test_resource.py index a7ab360..d794a99 100644 --- a/src/test/python/apache/thermos/monitoring/test_resource.py +++ b/src/test/python/apache/thermos/monitoring/test_resource.py @@ -71,6 +71,7 @@ class TestTaskResouceMonitor(TestCase): mock_sample.return_value = fake_process_sample task_resource_monitor = TaskResourceMonitor('fake-task-id', task_monitor) + assert task_resource_monitor.name == 'TaskResourceMonitor[fake-task-id]' assert fake_process_sample == task_resource_monitor.sample_by_process(fake_process_name) assert mock_get_active_processes.mock_calls == [mock.call(task_monitor)]
