AMBARI-18846 - Custom services should be able to easily specify their own dashboards
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b413d921 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b413d921 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b413d921 Branch: refs/heads/branch-dev-patch-upgrade Commit: b413d921cdeca51ad8d36088100ee13e853bb925 Parents: d2118f5 Author: Tim Thorpe <[email protected]> Authored: Tue Nov 15 13:03:39 2016 -0800 Committer: Tim Thorpe <[email protected]> Committed: Tue Nov 15 13:03:39 2016 -0800 ---------------------------------------------------------------------- .../ambari_agent/CustomServiceOrchestrator.py | 3 +- .../src/main/python/ambari_agent/FileCache.py | 10 ++++ .../TestCustomServiceOrchestrator.py | 11 +++- .../test/python/ambari_agent/TestFileCache.py | 12 ++++ .../python/ambari_server/resourceFilesKeeper.py | 12 ++-- .../python/ambari_server/serverConfiguration.py | 11 ++++ .../package/scripts/metrics_grafana_util.py | 8 +-- .../0.1.0/package/scripts/params.py | 3 + .../0.1.0/package/scripts/split_points.py | 29 ++++++---- .../stacks/HDP/2.0.6/services/stack_advisor.py | 3 +- .../src/test/python/TestResourceFilesKeeper.py | 61 +++++++++++++++++--- 11 files changed, 131 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b413d921/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py index 57416a4..fbc1509 100644 --- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py +++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py @@ -153,7 +153,8 @@ class CustomServiceOrchestrator(): self.file_cache.get_host_scripts_base_dir(server_url_prefix) hook_dir = self.file_cache.get_hook_base_dir(command, server_url_prefix) base_dir = self.file_cache.get_service_base_dir(command, server_url_prefix) - + self.file_cache.get_dashboard_base_dir(server_url_prefix) + script_path = self.resolve_script_path(base_dir, script) script_tuple = (script_path, base_dir) http://git-wip-us.apache.org/repos/asf/ambari/blob/b413d921/ambari-agent/src/main/python/ambari_agent/FileCache.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/FileCache.py b/ambari-agent/src/main/python/ambari_agent/FileCache.py index b7c5dee..83ac373 100644 --- a/ambari-agent/src/main/python/ambari_agent/FileCache.py +++ b/ambari-agent/src/main/python/ambari_agent/FileCache.py @@ -45,6 +45,7 @@ class FileCache(): STACKS_CACHE_DIRECTORY="stacks" COMMON_SERVICES_DIRECTORY="common-services" CUSTOM_ACTIONS_CACHE_DIRECTORY="custom_actions" + DASHBOARD_DIRECTORY="dashboards" HOST_SCRIPTS_CACHE_DIRECTORY="host_scripts" HASH_SUM_FILE=".hash" ARCHIVE_NAME="archive.zip" @@ -99,6 +100,15 @@ class FileCache(): server_url_prefix) + def get_dashboard_base_dir(self, server_url_prefix): + """ + Returns a base directory for dashboards + """ + return self.provide_directory(self.cache_dir, + self.DASHBOARD_DIRECTORY, + server_url_prefix) + + def get_host_scripts_base_dir(self, server_url_prefix): """ Returns a base directory for host scripts (host alerts, etc) which http://git-wip-us.apache.org/repos/asf/ambari/blob/b413d921/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py index c9724b7..9f1241e 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py +++ b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py @@ -220,7 +220,7 @@ class TestCustomServiceOrchestrator(TestCase): except AgentException: pass # Expected - + @patch.object(FileCache, "get_dashboard_base_dir") @patch.object(CustomServiceOrchestrator, "resolve_script_path") @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path") @patch.object(FileCache, "get_host_scripts_base_dir") @@ -234,7 +234,8 @@ class TestCustomServiceOrchestrator(TestCase): get_hook_base_dir_mock, get_service_base_dir_mock, get_host_scripts_base_dir_mock, resolve_hook_script_path_mock, - resolve_script_path_mock): + resolve_script_path_mock, + get_dashboard_base_dir_mock): FileCache_mock.return_value = None command = { @@ -265,6 +266,7 @@ class TestCustomServiceOrchestrator(TestCase): unix_process_id = 111 orchestrator.commands_in_progress = {command['taskId']: unix_process_id} get_hook_base_dir_mock.return_value = "/hooks/" + get_dashboard_base_dir_mock.return_value = "/dashboards/" # normal run case run_file_mock.return_value = { 'stdout' : 'sss', @@ -309,6 +311,7 @@ class TestCustomServiceOrchestrator(TestCase): pass + @patch.object(FileCache, "get_dashboard_base_dir") @patch("ambari_commons.shell.kill_process_with_children") @patch.object(CustomServiceOrchestrator, "resolve_script_path") @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path") @@ -323,7 +326,8 @@ class TestCustomServiceOrchestrator(TestCase): get_hook_base_dir_mock, get_service_base_dir_mock, get_host_scripts_base_dir_mock, resolve_hook_script_path_mock, resolve_script_path_mock, - kill_process_with_children_mock): + kill_process_with_children_mock, + get_dashboard_base_dir_mock): FileCache_mock.return_value = None command = { 'role' : 'REGION_SERVER', @@ -353,6 +357,7 @@ class TestCustomServiceOrchestrator(TestCase): unix_process_id = 111 orchestrator.commands_in_progress = {command['taskId']: unix_process_id} get_hook_base_dir_mock.return_value = "/hooks/" + get_dashboard_base_dir_mock.return_value = "/dashboards/" run_file_mock_return_value = { 'stdout' : 'killed', 'stderr' : 'killed', http://git-wip-us.apache.org/repos/asf/ambari/blob/b413d921/ambari-agent/src/test/python/ambari_agent/TestFileCache.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestFileCache.py b/ambari-agent/src/test/python/ambari_agent/TestFileCache.py index 5933daa..fbefc2b 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestFileCache.py +++ b/ambari-agent/src/test/python/ambari_agent/TestFileCache.py @@ -117,6 +117,18 @@ class TestFileCache(TestCase): "('/var/lib/ambari-agent/cache', 'custom_actions', 'server_url_pref')") self.assertEquals(res, "dummy value") + + @patch.object(FileCache, "provide_directory") + def test_get_dashboard_base_dir(self, provide_directory_mock): + provide_directory_mock.return_value = "dummy value" + fileCache = FileCache(self.config) + res = fileCache.get_dashboard_base_dir("server_url_pref") + self.assertEquals( + pprint.pformat(provide_directory_mock.call_args_list[0][0]), + "('/var/lib/ambari-agent/cache', 'dashboards', 'server_url_pref')") + self.assertEquals(res, "dummy value") + + @patch.object(FileCache, "build_download_url") def test_provide_directory_no_update(self, build_download_url_mock): try: http://git-wip-us.apache.org/repos/asf/ambari/blob/b413d921/ambari-server/src/main/python/ambari_server/resourceFilesKeeper.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/resourceFilesKeeper.py b/ambari-server/src/main/python/ambari_server/resourceFilesKeeper.py index f2a3ebd..8047e7d 100644 --- a/ambari-server/src/main/python/ambari_server/resourceFilesKeeper.py +++ b/ambari-server/src/main/python/ambari_server/resourceFilesKeeper.py @@ -39,6 +39,7 @@ class ResourceFilesKeeper(): COMMON_SERVICES_DIR="common-services" CUSTOM_ACTIONS_DIR="custom_actions" HOST_SCRIPTS_DIR="host_scripts" + DASHBOARDS_DIR="dashboards" # For these directories archives are created ARCHIVABLE_DIRS = [HOOKS_DIR, PACKAGE_DIR] @@ -68,7 +69,7 @@ class ResourceFilesKeeper(): """ Performs housekeeping operations on resource files """ - self.update_directory_archieves() + self.update_directory_archives() # probably, later we will need some additional operations @@ -87,7 +88,7 @@ class ResourceFilesKeeper(): # update the directories so that the .hash is generated self.update_directory_archive(archive_root) - def update_directory_archieves(self): + def update_directory_archives(self): """ Please see AMBARI-4481 for more details """ @@ -112,6 +113,9 @@ class ResourceFilesKeeper(): # agent host scripts self._update_resources_subdir_archive(self.HOST_SCRIPTS_DIR) + # custom service dashboards + self._update_resources_subdir_archive(self.DASHBOARDS_DIR) + def _list_metainfo_dirs(self, root_dir): valid_items = [] # Format: <stack_dir, ignore(True|False)> @@ -153,7 +157,7 @@ class ResourceFilesKeeper(): if not self.nozip: self.zip_directory(directory, skip_empty_directory) # Skip generation of .hash file is directory is empty - if (skip_empty_directory and not os.listdir(directory)): + if (skip_empty_directory and (not os.path.exists(directory) or not os.listdir(directory))): self.dbg_out("Empty directory. Skipping generation of hash file for {0}".format(directory)) else: self.write_hash_sum(directory, cur_hash) @@ -228,7 +232,7 @@ class ResourceFilesKeeper(): self.dbg_out("creating archive for directory {0}".format(directory)) try: if skip_if_empty: - if not os.listdir(directory): + if not os.path.exists(directory) or not os.listdir(directory): self.dbg_out("Empty directory. Skipping archive creation for {0}".format(directory)) return http://git-wip-us.apache.org/repos/asf/ambari/blob/b413d921/ambari-server/src/main/python/ambari_server/serverConfiguration.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py index 04509cf..3086003 100644 --- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py +++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py @@ -397,6 +397,7 @@ class ServerConfigDefaults(object): self.EXTENSION_LOCATION_DEFAULT = "" self.COMMON_SERVICES_LOCATION_DEFAULT = "" self.MPACKS_STAGING_LOCATION_DEFAULT = "" + self.DASHBOARD_LOCATION_DEFAULT = "" self.SERVER_TMP_DIR_DEFAULT = "" self.DEFAULT_VIEWS_DIR = "" @@ -468,6 +469,7 @@ class ServerConfigDefaultsWindows(ServerConfigDefaults): self.EXTENSION_LOCATION_DEFAULT = "resources\\extensions" self.COMMON_SERVICES_LOCATION_DEFAULT = "resources\\common-services" self.MPACKS_STAGING_LOCATION_DEFAULT = "resources\\mpacks" + self.DASHBOARD_LOCATION_DEFAULT = "resources\\dashboards" self.SERVER_TMP_DIR_DEFAULT = "data\\tmp" self.DEFAULT_VIEWS_DIR = "resources\\views" @@ -554,6 +556,7 @@ class ServerConfigDefaultsLinux(ServerConfigDefaults): self.EXTENSION_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/extensions") self.COMMON_SERVICES_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/common-services") self.MPACKS_STAGING_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/mpacks") + self.DASHBOARD_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/dashboards") self.SERVER_TMP_DIR_DEFAULT = AmbariPath.get("/var/lib/ambari-server/data/tmp") self.DEFAULT_VIEWS_DIR = AmbariPath.get("/var/lib/ambari-server/resources/views") @@ -1439,6 +1442,14 @@ def get_mpacks_staging_location(properties): mpacks_staging_location = configDefaults.MPACKS_STAGING_LOCATION_DEFAULT return mpacks_staging_location + +# +# Dashboard location +# +def get_dashboard_location(properties): + dashboard_location = configDefaults.DASHBOARD_LOCATION_DEFAULT + return dashboard_location + # # Server temp location # http://git-wip-us.apache.org/repos/asf/ambari/blob/b413d921/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py index b98dc1d..6030fc4 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py @@ -36,7 +36,7 @@ import os GRAFANA_CONNECT_TRIES = 5 GRAFANA_CONNECT_TIMEOUT = 10 -GRAFANA_SEARCH_BULTIN_DASHBOARDS = "/api/search?tag=builtin" +GRAFANA_SEARCH_BUILTIN_DASHBOARDS = "/api/search?tag=builtin" GRAFANA_DATASOURCE_URL = "/api/datasources" GRAFANA_DASHBOARDS_URL = "/api/dashboards/db" METRICS_GRAFANA_DATASOURCE_NAME = "AMBARI_METRICS" @@ -313,14 +313,14 @@ def create_ams_dashboards(): Dashboard = namedtuple('Dashboard', ['uri', 'id', 'title', 'tags']) existing_dashboards = [] - response = perform_grafana_get_call(GRAFANA_SEARCH_BULTIN_DASHBOARDS, server) + response = perform_grafana_get_call(GRAFANA_SEARCH_BUILTIN_DASHBOARDS, server) if response and response.status == 200: data = response.read() try: dashboards = json.loads(data) except: Logger.error("Unable to parse JSON response from grafana request: %s" % - GRAFANA_SEARCH_BULTIN_DASHBOARDS) + GRAFANA_SEARCH_BUILTIN_DASHBOARDS) Logger.info(data) return @@ -336,7 +336,7 @@ def create_ams_dashboards(): else: Logger.error("Failed to execute search query on Grafana dashboards. " "query = %s\n statuscode = %s\n reason = %s\n data = %s\n" % - (GRAFANA_SEARCH_BULTIN_DASHBOARDS, response.status, response.reason, response.read())) + (GRAFANA_SEARCH_BUILTIN_DASHBOARDS, response.status, response.reason, response.read())) return Logger.debug('Dashboard definitions found = %s' % str(dashboard_files)) http://git-wip-us.apache.org/repos/asf/ambari/blob/b413d921/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py index f04f5c2..acc3763 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py @@ -83,6 +83,9 @@ dashboards_dirs.append(os.path.join(agent_cache_dir, service_package_folder, dashboards_dirs.append(os.path.join(agent_cache_dir, service_package_folder, 'files', 'grafana-dashboards', 'default')) +# Custom services +dashboards_dirs.append(os.path.join(agent_cache_dir, 'dashboards', 'grafana-dashboards')) + def get_grafana_dashboard_defs(): dashboard_defs = [] for dashboards_dir in dashboards_dirs: http://git-wip-us.apache.org/repos/asf/ambari/blob/b413d921/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/split_points.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/split_points.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/split_points.py index aa03d197..ffe10b5 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/split_points.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/split_points.py @@ -67,11 +67,12 @@ def format_Xmx_size_to_bytes(value, default='b'): # pre-splits based on selected services also passed as a parameter to the class. class FindSplitPointsForAMSRegions(): - def __init__(self, ams_hbase_site, ams_hbase_env, serviceMetricsDir, + def __init__(self, ams_hbase_site, ams_hbase_env, serviceMetricsDir, customServiceMetricsDir, operation_mode = 'embedded', services = None): self.ams_hbase_site = ams_hbase_site self.ams_hbase_env = ams_hbase_env self.serviceMetricsDir = serviceMetricsDir + self.customServiceMetricsDir = customServiceMetricsDir self.services = services self.mode = operation_mode # Add host metrics if not present as input @@ -117,18 +118,28 @@ class FindSplitPointsForAMSRegions(): pass def initialize_ordered_set_of_metrics(self): - onlyServicefiles = [ f for f in os.listdir(self.serviceMetricsDir) if - os.path.isfile(os.path.join(self.serviceMetricsDir, f)) ] - metrics = set() + self.gatherMetrics(metrics, self.serviceMetricsDir) + self.gatherMetrics(metrics, self.customServiceMetricsDir) + + self.metrics = sorted(metrics) + print 'metrics length: %s' % len(self.metrics) - for file in onlyServicefiles: - # Process for services selected at deploy time or all services if + + def gatherMetrics(self, metrics, dir): + if os.path.exists(dir): + files = [ f for f in os.listdir(dir) if + os.path.isfile(os.path.join(dir, f)) ] + else: + return + + for file in files: + # Process for stack services selected at deploy time or all stack services if # services arg is not passed if self.services is None or file.rstrip(metric_filename_ext) in self.services: - print 'Processing file: %s' % os.path.join(self.serviceMetricsDir, file) + print 'Processing file: %s' % os.path.join(dir, file) service_metrics = set() - with open(os.path.join(self.serviceMetricsDir, file), 'r') as f: + with open(os.path.join(dir, file), 'r') as f: for metric in f: service_metrics.add(metric.strip()) pass @@ -137,8 +148,6 @@ class FindSplitPointsForAMSRegions(): pass pass - self.metrics = sorted(metrics) - print 'metrics length: %s' % len(self.metrics) # Pick 50 metric points for each service that are equidistant from # each other for a service http://git-wip-us.apache.org/repos/asf/ambari/blob/b413d921/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index c7c3b2b..9b5ff68 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -832,6 +832,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): scriptDir = os.path.dirname(os.path.abspath(__file__)) metricsDir = os.path.join(scriptDir, '../../../../common-services/AMBARI_METRICS/0.1.0/package') serviceMetricsDir = os.path.join(metricsDir, 'files', 'service-metrics') + customServiceMetricsDir = os.path.join(scriptDir, '../../../../dashboards/service-metrics') sys.path.append(os.path.join(metricsDir, 'scripts')) servicesList = [service["StackServices"]["service_name"] for service in services["services"]] @@ -853,7 +854,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): ams_hbase_env = configurations["ams-hbase-env"]["properties"] split_point_finder = FindSplitPointsForAMSRegions( - ams_hbase_site, ams_hbase_env, serviceMetricsDir, operatingMode, servicesList) + ams_hbase_site, ams_hbase_env, serviceMetricsDir, customServiceMetricsDir, operatingMode, servicesList) result = split_point_finder.get_split_points() precision_splits = ' ' http://git-wip-us.apache.org/repos/asf/ambari/blob/b413d921/ambari-server/src/test/python/TestResourceFilesKeeper.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestResourceFilesKeeper.py b/ambari-server/src/test/python/TestResourceFilesKeeper.py index 966d3b1..4801fad 100644 --- a/ambari-server/src/test/python/TestResourceFilesKeeper.py +++ b/ambari-server/src/test/python/TestResourceFilesKeeper.py @@ -82,7 +82,8 @@ class TestResourceFilesKeeper(TestCase): "call('../resources/TestAmbaryServer.samples/" \ "dummy_common_services/HIVE/0.11.0.2.0.5.0/package'),\n " \ "call('../resources/custom_actions'),\n " \ - "call('../resources/host_scripts')]" + "call('../resources/host_scripts'),\n " \ + "call('../resources/dashboards')]" else: UPDATE_DIRECTORY_ARCHIVE_CALL_LIST = \ "[call('..\\\\resources\\\\TestAmbaryServer.samples\\\\dummy_stack\\\\HIVE\\\\package'),\n " \ @@ -91,17 +92,18 @@ class TestResourceFilesKeeper(TestCase): "call('..\\\\resources\\\\TestAmbaryServer.samples\\\\dummy_common_services\\\\HIVE\\\\0.11.0.2.0.5.0\\\\package'),\n " \ "call('..\\\\resources\\\\TestAmbaryServer.samples\\\\dummy_common_services\\\\HIVE\\\\0.11.0.2.0.5.0\\\\package'),\n " \ "call('..\\\\resources\\\\custom_actions'),\n " \ - "call('..\\\\resources\\\\host_scripts')]" + "call('..\\\\resources\\\\host_scripts'),\n " \ + "call('..\\\\resources\\\\dashboards')]" def setUp(self): logging.basicConfig(level=logging.ERROR) - @patch.object(ResourceFilesKeeper, "update_directory_archieves") - def test_perform_housekeeping(self, update_directory_archieves_mock): + @patch.object(ResourceFilesKeeper, "update_directory_archives") + def test_perform_housekeeping(self, update_directory_archives_mock): resource_files_keeper = ResourceFilesKeeper(os.sep + "dummy-resources", os.sep + "dummy-path") resource_files_keeper.perform_housekeeping() - update_directory_archieves_mock.assertCalled() + update_directory_archives_mock.assertCalled() pass @@ -109,7 +111,7 @@ class TestResourceFilesKeeper(TestCase): @patch.object(ResourceFilesKeeper, "list_common_services") @patch.object(ResourceFilesKeeper, "list_stacks") @patch("os.path.abspath") - def test_update_directory_archieves(self, abspath_mock, + def test_update_directory_archives(self, abspath_mock, list_active_stacks_mock, list_common_services_mock, update_directory_archive_mock): @@ -120,7 +122,7 @@ class TestResourceFilesKeeper(TestCase): self.DUMMY_UNCHANGEABLE_COMMON_SERVICES] abspath_mock.side_effect = lambda s : s resource_files_keeper = ResourceFilesKeeper(self.TEST_RESOURCES_DIR, self.TEST_STACKS_DIR) - resource_files_keeper.update_directory_archieves() + resource_files_keeper.update_directory_archives() self.assertEquals(pprint.pformat( update_directory_archive_mock.call_args_list), self.UPDATE_DIRECTORY_ARCHIVE_CALL_LIST) @@ -168,6 +170,7 @@ class TestResourceFilesKeeper(TestCase): except Exception, e: self.fail('Unexpected exception thrown:' + str(e)) + @patch("os.path.exists") @patch("os.listdir") @patch.object(ResourceFilesKeeper, "count_hash_sum") @patch.object(ResourceFilesKeeper, "read_hash_sum") @@ -176,11 +179,12 @@ class TestResourceFilesKeeper(TestCase): def test_update_directory_archive(self, write_hash_sum_mock, zip_directory_mock, read_hash_sum_mock, count_hash_sum_mock, - os_listdir_mock): + os_listdir_mock, os_path_exists_mock): os_listdir_mock.return_value = ['file1', 'dir1'] # Test situation when there is no saved directory hash read_hash_sum_mock.return_value = None count_hash_sum_mock.return_value = self.YA_HASH + os_path_exists_mock.return_value = True resource_files_keeper = ResourceFilesKeeper(self.TEST_RESOURCES_DIR, self.SOME_PATH) resource_files_keeper.update_directory_archive(self.SOME_PATH) self.assertTrue(read_hash_sum_mock.called) @@ -196,6 +200,7 @@ class TestResourceFilesKeeper(TestCase): # Test situation when saved directory hash == current hash read_hash_sum_mock.return_value = self.DUMMY_HASH count_hash_sum_mock.return_value = self.YA_HASH + os_path_exists_mock.return_value = True resource_files_keeper.update_directory_archive(self.SOME_PATH) self.assertTrue(read_hash_sum_mock.called) self.assertTrue(count_hash_sum_mock.called) @@ -210,6 +215,7 @@ class TestResourceFilesKeeper(TestCase): # Test situation when saved directory hash == current hash read_hash_sum_mock.return_value = self.DUMMY_HASH count_hash_sum_mock.return_value = self.DUMMY_HASH + os_path_exists_mock.return_value = True resource_files_keeper.update_directory_archive(self.SOME_PATH) self.assertTrue(read_hash_sum_mock.called) self.assertTrue(count_hash_sum_mock.called) @@ -225,6 +231,7 @@ class TestResourceFilesKeeper(TestCase): zip_directory_mock.side_effect = self.keeper_exc_side_effect read_hash_sum_mock.return_value = self.DUMMY_HASH count_hash_sum_mock.return_value = self.YA_HASH + os_path_exists_mock.return_value = True try: resource_files_keeper.update_directory_archive(self.SOME_PATH) self.fail('KeeperException not thrown') @@ -245,6 +252,7 @@ class TestResourceFilesKeeper(TestCase): # Test nozip option read_hash_sum_mock.return_value = None count_hash_sum_mock.return_value = self.YA_HASH + os_path_exists_mock.return_value = True resource_files_keeper = ResourceFilesKeeper(self.TEST_RESOURCES_DIR, self.SOME_PATH, nozip=True) resource_files_keeper.update_directory_archive(self.SOME_PATH) self.assertTrue(read_hash_sum_mock.called) @@ -262,6 +270,26 @@ class TestResourceFilesKeeper(TestCase): os_listdir_mock.return_value = [] # Empty dir zip_directory_mock.side_effect = None read_hash_sum_mock.return_value = None # hash read from .hash file + os_path_exists_mock.return_value = True + resource_files_keeper = ResourceFilesKeeper(self.TEST_RESOURCES_DIR, self.SOME_PATH) + resource_files_keeper.update_directory_archive(self.SOME_PATH) + + self.assertTrue(read_hash_sum_mock.called) + self.assertTrue(count_hash_sum_mock.called) + self.assertTrue(zip_directory_mock.called) + self.assertFalse(write_hash_sum_mock.called) + pass + + # Test no directory + read_hash_sum_mock.reset_mock() + count_hash_sum_mock.reset_mock() + zip_directory_mock.reset_mock() + write_hash_sum_mock.reset_mock() + + # If the input directory doesn't exist, then write_hash_sum() should not be called + zip_directory_mock.side_effect = None + read_hash_sum_mock.return_value = None # hash read from .hash file + os_path_exists_mock.return_value = False resource_files_keeper = ResourceFilesKeeper(self.TEST_RESOURCES_DIR, self.SOME_PATH) resource_files_keeper.update_directory_archive(self.SOME_PATH) @@ -347,8 +375,10 @@ class TestResourceFilesKeeper(TestCase): self.fail('Unexpected exception thrown:' + str(e)) - def test_zip_directory(self): + @patch("os.path.exists") + def test_zip_directory(self, os_path_exists_mock): # Test normal flow + os_path_exists_mock.return_value = True resource_files_keeper = ResourceFilesKeeper(self.TEST_RESOURCES_DIR, self.DUMMY_UNCHANGEABLE_PACKAGE) resource_files_keeper.zip_directory(self.DUMMY_UNCHANGEABLE_PACKAGE) arc_file = os.path.join(self.DUMMY_UNCHANGEABLE_PACKAGE, @@ -384,6 +414,19 @@ class TestResourceFilesKeeper(TestCase): self.fail('Unexpected exception thrown: ' + str(e)) pass + # Test skip zipping of a missing directory + with patch("os.listdir") as os_listdir_mock: + os_path_exists_mock.return_value = False + os_listdir_mock.return_value = False # Empty dir + try: + skip_empty_directory = True + resource_files_keeper.zip_directory("empty-to-directory", skip_empty_directory) + self.assertTrue(os_path_exists_mock.called) + except Exception, e: + self.fail('Unexpected exception thrown: ' + str(e)) + pass + + def test_is_ignored(self): resource_files_keeper = ResourceFilesKeeper(self.TEST_RESOURCES_DIR, self.DUMMY_UNCHANGEABLE_PACKAGE) self.assertTrue(resource_files_keeper.is_ignored(".hash"))
