Repository: ambari Updated Branches: refs/heads/branch-3.0-perf 32552c9b4 -> 2e579f11c
AMBARI-20869. Changes to events format (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2e579f11 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2e579f11 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2e579f11 Branch: refs/heads/branch-3.0-perf Commit: 2e579f11c25953e777abcfaf11facd0157635be3 Parents: 32552c9 Author: Andrew Onishuk <[email protected]> Authored: Wed May 3 12:12:28 2017 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Wed May 3 12:12:28 2017 +0300 ---------------------------------------------------------------------- .../main/python/ambari_agent/ClusterCache.py | 22 +- .../ambari_agent/ClusterConfigurationCache.py | 6 +- .../ambari_agent/ComponentStatusExecutor.py | 14 +- .../src/main/python/ambari_agent/Constants.py | 8 +- .../main/python/ambari_agent/HeartbeatThread.py | 4 +- .../src/main/python/ambari_agent/Utils.py | 5 +- .../ambari_agent/TestAgentStompResponses.py | 24 +-- .../stomp/components_status_report.json | 46 +++-- .../stomp/configurations_update.json | 44 ++-- .../dummy_files/stomp/execution_commands.json | 206 +++++++++---------- .../stomp/metadata_after_registration.json | 60 +++--- .../dummy_files/stomp/metadata_update.json | 13 +- .../dummy_files/stomp/registration_request.json | 2 +- .../stomp/registration_response.json | 4 +- .../dummy_files/stomp/topology_update.json | 126 ++++++------ 15 files changed, 288 insertions(+), 296 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2e579f11/ambari-agent/src/main/python/ambari_agent/ClusterCache.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/ClusterCache.py b/ambari-agent/src/main/python/ambari_agent/ClusterCache.py index d5e724c..70b44b4 100644 --- a/ambari-agent/src/main/python/ambari_agent/ClusterCache.py +++ b/ambari-agent/src/main/python/ambari_agent/ClusterCache.py @@ -58,35 +58,35 @@ class ClusterCache(dict): with open(self.__current_cache_json_file, 'r') as fp: cache_dict = json.load(fp) - for cluster_name, cache in cache_dict.iteritems(): + for cluster_id, cache in cache_dict.iteritems(): immutable_cache = Utils.make_immutable(cache) - cache_dict[cluster_name] = immutable_cache + cache_dict[cluster_id] = immutable_cache super(ClusterCache, self).__init__(cache_dict) - def get_cluster_names(self): + def get_cluster_ids(self): return self.keys() def update_cache(self, cache): - for cluster_name, cluster_cache in cache['clusters'].iteritems(): - self.update_cluster_cache(cluster_name, cluster_cache) + for cluster_id, cluster_cache in cache.iteritems(): + self.update_cluster_cache(cluster_id, cluster_cache) - def update_cluster_cache(self, cluster_name, cache): + def update_cluster_cache(self, cluster_id, cache): """ Thread-safe method for writing out the specified cluster cache and updating the in-memory representation. - :param cluster_name: + :param cluster_id: :param cache: :return: """ - logger.info("Updating cache {0} for cluster {1}".format(self.__class__.__name__, cluster_name)) + logger.info("Updating cache {0} for cluster {1}".format(self.__class__.__name__, cluster_id)) # The cache should contain exactly the data received from server. # Modifications on agent-side will lead to unnecessary cache sync every agent registration. Which is a big concern on perf clusters! # Also immutability can lead to multithreading issues. immutable_cache = Utils.make_immutable(cache) with self._cache_lock: - self[cluster_name] = immutable_cache + self[cluster_id] = immutable_cache # ensure that our cache directory exists @@ -97,11 +97,11 @@ class ClusterCache(dict): with os.fdopen(os.open(self.__current_cache_json_file, os.O_WRONLY | os.O_CREAT, 0o600), "w") as f: json.dump(self, f, indent=2) - def get_md5_hashsum(self, cluster_name): + def get_md5_hashsum(self, cluster_id): """ Thread-safe method for writing out the specified cluster cache and updating the in-memory representation. - :param cluster_name: + :param cluster_id: :param cache: :return: """ http://git-wip-us.apache.org/repos/asf/ambari/blob/2e579f11/ambari-agent/src/main/python/ambari_agent/ClusterConfigurationCache.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/ClusterConfigurationCache.py b/ambari-agent/src/main/python/ambari_agent/ClusterConfigurationCache.py index 03e9645..f0c3945 100644 --- a/ambari-agent/src/main/python/ambari_agent/ClusterConfigurationCache.py +++ b/ambari-agent/src/main/python/ambari_agent/ClusterConfigurationCache.py @@ -41,7 +41,7 @@ class ClusterConfigurationCache(ClusterCache): def get_cache_name(self): return 'configurations' - def get_configuration_value(self, cluster_name, key): + def get_configuration_value(self, cluster_id, key): """ Gets a value from the cluster configuration map for the given cluster and key. The key is expected to be of the form 'foo-bar/baz' or @@ -51,14 +51,14 @@ class ClusterConfigurationCache(ClusterCache): """ self._cache_lock.acquire() try: - dictionary = self[cluster_name] + dictionary = self[cluster_id] for layer_key in key.split('/'): dictionary = dictionary[layer_key] return dictionary except KeyError: - logger.debug("Cache miss for configuration property {0} in cluster {1}".format(key, cluster_name)) + logger.debug("Cache miss for configuration property {0} in cluster {1}".format(key, cluster_id)) return None finally: self._cache_lock.release() http://git-wip-us.apache.org/repos/asf/ambari/blob/2e579f11/ambari-agent/src/main/python/ambari_agent/ComponentStatusExecutor.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/ComponentStatusExecutor.py b/ambari-agent/src/main/python/ambari_agent/ComponentStatusExecutor.py index 70cd82a..ebe9156 100644 --- a/ambari-agent/src/main/python/ambari_agent/ComponentStatusExecutor.py +++ b/ambari-agent/src/main/python/ambari_agent/ComponentStatusExecutor.py @@ -45,11 +45,11 @@ class ComponentStatusExecutor(threading.Thread): try: cluster_reports = defaultdict(lambda:[]) - for cluster_id in self.topology_cache.get_cluster_names(): + for cluster_id in self.topology_cache.get_cluster_ids(): # TODO: check if we can make clusters immutable too try: - cluster_cache = self.topology_cache[cluster_id] - cluster_metadata = self.metadata_cache[cluster_id] + topology_cache = self.topology_cache[cluster_id] + metadata_cache = self.metadata_cache[cluster_id] except KeyError: # multithreading: if cluster was deleted during iteration continue @@ -62,17 +62,17 @@ class ComponentStatusExecutor(threading.Thread): # TODO STOMP: read this from metadata status_commands_to_run = ['STATUS', 'SECURITY_STATUS'] - cluster_components = cluster_cache.topology.components - for component_name in cluster_components: + cluster_components = topology_cache.components + for component_dict in cluster_components: for command in status_commands_to_run: if self.stop_event.is_set(): break - component_info = cluster_components[component_name] + component_name = component_dict.componentName # TODO STOMP: run real command - logger.info("Running {0}/{1}".format(component_info.statusCommandsParams.service_package_folder, component_info.statusCommandsParams.script)) + logger.info("Running {0}/{1}".format(component_dict.statusCommandsParams.service_package_folder, component_dict.statusCommandsParams.script)) #self.customServiceOrchestrator.requestComponentStatus(command) status = random.choice(["INSTALLED","STARTED"]) result = { http://git-wip-us.apache.org/repos/asf/ambari/blob/2e579f11/ambari-agent/src/main/python/ambari_agent/Constants.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/Constants.py b/ambari-agent/src/main/python/ambari_agent/Constants.py index 5ea916c..3fbb485 100644 --- a/ambari-agent/src/main/python/ambari_agent/Constants.py +++ b/ambari-agent/src/main/python/ambari_agent/Constants.py @@ -19,10 +19,10 @@ limitations under the License. ''' -COMMANDS_TOPIC = '/events/commands' -CONFIGURATIONS_TOPIC = '/events/configurations' -METADATA_TOPIC = '/events/metadata' -TOPOLOGIES_TOPIC = '/events/topologies' +COMMANDS_TOPIC = '/user/commands' +CONFIGURATIONS_TOPIC = '/user/configs' +METADATA_TOPIC = '/user/metadata' +TOPOLOGIES_TOPIC = '/user/topologies' SERVER_RESPONSES_TOPIC = '/user/' TOPICS_TO_SUBSCRIBE = [SERVER_RESPONSES_TOPIC, COMMANDS_TOPIC, CONFIGURATIONS_TOPIC, METADATA_TOPIC, TOPOLOGIES_TOPIC] http://git-wip-us.apache.org/repos/asf/ambari/blob/2e579f11/ambari-agent/src/main/python/ambari_agent/HeartbeatThread.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/HeartbeatThread.py b/ambari-agent/src/main/python/ambari_agent/HeartbeatThread.py index 6b68f27..1f7d576 100644 --- a/ambari-agent/src/main/python/ambari_agent/HeartbeatThread.py +++ b/ambari-agent/src/main/python/ambari_agent/HeartbeatThread.py @@ -104,8 +104,8 @@ class HeartbeatThread(threading.Thread): for cache in self.caches: cache_key_name = cache.get_cache_name() + '_hash' - for cluster_name in cache.get_cluster_names(): - request['clusters'][cluster_name][cache_key_name] = cache.get_md5_hashsum(cluster_name) + for cluster_id in cache.get_cluster_ids(): + request['clusters'][cluster_id][cache_key_name] = cache.get_md5_hashsum(cluster_id) return request http://git-wip-us.apache.org/repos/asf/ambari/blob/2e579f11/ambari-agent/src/main/python/ambari_agent/Utils.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/Utils.py b/ambari-agent/src/main/python/ambari_agent/Utils.py index 456b2b3..d6f0294 100644 --- a/ambari-agent/src/main/python/ambari_agent/Utils.py +++ b/ambari-agent/src/main/python/ambari_agent/Utils.py @@ -61,8 +61,9 @@ class Utils(object): def make_immutable(value): if isinstance(value, dict): return ImmutableDictionary(value) - if isinstance(value, list): - return tuple(value) + if isinstance(value, (list, tuple)): + return tuple([Utils.make_immutable(x) for x in value]) + return value class ImmutableDictionary(dict): http://git-wip-us.apache.org/repos/asf/ambari/blob/2e579f11/ambari-agent/src/test/python/ambari_agent/TestAgentStompResponses.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestAgentStompResponses.py b/ambari-agent/src/test/python/ambari_agent/TestAgentStompResponses.py index 7ea0646..c05f350 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestAgentStompResponses.py +++ b/ambari-agent/src/test/python/ambari_agent/TestAgentStompResponses.py @@ -36,7 +36,7 @@ class TestAgentStompResponses(BaseStompServerTestCase): def test_mock_server_can_start(self): self.init_stdout_logger() - self.remove(['/tmp/configurations.json', '/tmp/metadata.json', '/tmp/topology.json']) + self.remove(['/tmp/cluster_cache/configurations.json', '/tmp/cluster_cache/metadata.json', '/tmp/cluster_cache/topology.json']) initializer_module = InitializerModule() heartbeat_thread = HeartbeatThread.HeartbeatThread(initializer_module) @@ -60,16 +60,16 @@ class TestAgentStompResponses(BaseStompServerTestCase): f = Frame(frames.MESSAGE, headers={'destination': '/user/', 'correlationId': '0'}, body=self.get_json("registration_response.json")) self.server.topic_manager.send(f) - f = Frame(frames.MESSAGE, headers={'destination': '/events/configurations'}, body=self.get_json("configurations_update.json")) + f = Frame(frames.MESSAGE, headers={'destination': '/user/configs'}, body=self.get_json("configurations_update.json")) self.server.topic_manager.send(f) - f = Frame(frames.MESSAGE, headers={'destination': '/events/commands'}, body=self.get_json("execution_commands.json")) + f = Frame(frames.MESSAGE, headers={'destination': '/user/commands'}, body=self.get_json("execution_commands.json")) self.server.topic_manager.send(f) - f = Frame(frames.MESSAGE, headers={'destination': '/events/metadata'}, body=self.get_json("metadata_after_registration.json")) + f = Frame(frames.MESSAGE, headers={'destination': '/user/metadata'}, body=self.get_json("metadata_after_registration.json")) self.server.topic_manager.send(f) - f = Frame(frames.MESSAGE, headers={'destination': '/events/topologies'}, body=self.get_json("topology_update.json")) + f = Frame(frames.MESSAGE, headers={'destination': '/user/topologies'}, body=self.get_json("topology_update.json")) self.server.topic_manager.send(f) heartbeat_frame = self.server.frames_queue.get() @@ -81,9 +81,9 @@ class TestAgentStompResponses(BaseStompServerTestCase): heartbeat_thread.join() component_status_executor.join() - self.assertEquals(initializer_module.topology_cache['cl1']['topology']['hosts'][0]['hostname'], 'c6401.ambari.apache.org') - self.assertEquals(initializer_module.metadata_cache['cl1']['metadata']['status_commands_to_run'], ('STATUS',)) - self.assertEquals(initializer_module.configurations_cache['cl1']['configurations']['zoo.cfg']['clientPort'], '2181') + self.assertEquals(initializer_module.topology_cache['0']['hosts'][0]['hostname'], 'c6401.ambari.apache.org') + self.assertEquals(initializer_module.metadata_cache['0']['status_commands_to_run'], ('STATUS',)) + self.assertEquals(initializer_module.configurations_cache['0']['configurations']['zoo.cfg']['clientPort'], '2181') """ ============================================================================================ @@ -104,16 +104,16 @@ class TestAgentStompResponses(BaseStompServerTestCase): metadata_subscribe_frame = self.server.frames_queue.get() topologies_subscribe_frame = self.server.frames_queue.get() registration_frame_json = json.loads(self.server.frames_queue.get().body) - clusters_hashes = registration_frame_json['clusters']['cl1'] + clusters_hashes = registration_frame_json['clusters']['0'] component_status_executor = ComponentStatusExecutor(initializer_module) component_status_executor.start() status_reports_frame = self.server.frames_queue.get() - self.assertEquals(clusters_hashes['metadata_hash'], '20089c8c8682cf03e361cdab3e668ed1') - self.assertEquals(clusters_hashes['configurations_hash'], 'bc54fe976cade95c48eafbfdff188661') - self.assertEquals(clusters_hashes['topology_hash'], 'd14ca943e4a69ad0dd640f32d713d2b9') + self.assertEquals(clusters_hashes['metadata_hash'], '21724f6ffa7aff0fe91a0c0c5b765dba') + self.assertEquals(clusters_hashes['configurations_hash'], '04c968412ded7c8ffe7858036bae03ce') + self.assertEquals(clusters_hashes['topology_hash'], '0de1df56fd594873fe594cf02ea61f4b') # server sends registration response f = Frame(frames.MESSAGE, headers={'destination': '/user/', 'correlationId': '0'}, body=self.get_json("registration_response.json")) http://git-wip-us.apache.org/repos/asf/ambari/blob/2e579f11/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/components_status_report.json ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/components_status_report.json b/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/components_status_report.json index c1ce406..5b55869 100644 --- a/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/components_status_report.json +++ b/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/components_status_report.json @@ -1,24 +1,28 @@ { - "clusters":{ - "cl1":{ - "componentStatuses":[ - { - "componentName":"HDFS_CLIENT", - "command":"SECURITY_STATUS", - "stdout":"", - "stderr":"Python script killed due to timeout", - "structuredOutput":"{}", - "exitCode":-15 - }, - { - "componentName":"ZOOKEEPER_SERVER", - "command":"STATUS", - "stdout":"", - "stderr":"", - "structuredOutput":"{}", - "exitcode":0 - } - ] + "0":[ + { + "status":"INSTALLED", + "componentName":"ZOOKEEPER_SERVER", + "clusterId": 0, + "command":"STATUS" + }, + { + "status":"STARTED", + "componentName":"ZOOKEEPER_SERVER", + "clusterId": 0, + "command":"SECURITY_STATUS" + }, + { + "status":"INSTALLED", + "componentName":"ZOOKEEPER_CLIENT", + "clusterId": 0, + "command":"STATUS" + }, + { + "status":"INSTALLED", + "componentName":"ZOOKEEPER_CLIENT", + "clusterId": 0, + "command":"SECURITY_STATUS" } - } + ] } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/2e579f11/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/configurations_update.json ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/configurations_update.json b/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/configurations_update.json index 41cef61..cc1dda0 100644 --- a/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/configurations_update.json +++ b/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/configurations_update.json @@ -1,28 +1,26 @@ { - "clusters":{ - "cl1":{ - "configurations":{ - "zookeeper-env":{ - "zk_user":"zookeeper", - "zk_log_dir":"/var/log/zookeeper", - "content":"\nexport JAVA_HOME={{java64_home}}\nexport ZOO_LOG_DIR={{zk_log_dir}}\nexport ZOOPIDFILE={{zk_pid_file}}\nexport SERVER_JVMFLAGS={{zk_server_heapsize}}\nexport JAVA=$JAVA_HOME/bin/java\nexport CLASSPATH=$CLASSPATH:/usr/share/zookeeper/*\n\n{% if security_enabled %}\nexport SERVER_JVMFLAGS=\"$SERVER_JVMFLAGS -Djava.security.auth.login.config={{zk_server_jaas_file}}\"\nexport CLIENT_JVMFLAGS=\"$CLIENT_JVMFLAGS -Djava.security.auth.login.config={{zk_client_jaas_file}}\"\n{% endif %}", - "zk_pid_dir":"/var/run/zookeeper", - "zookeeper_principal_name":"zookeeper/[email protected]", - "zookeeper_keytab_path":"/etc/security/keytabs/zk.service.keytab" - }, - "zoo.cfg":{ - "clientPort":"2181", - "syncLimit":"5", - "initLimit":"10", - "dataDir":"/hadoop/zookeeper", - "tickTime":"2000" - } + "0":{ + "configurations":{ + "zookeeper-env":{ + "zk_user":"zookeeper", + "zk_log_dir":"/var/log/zookeeper", + "content":"\nexport JAVA_HOME={{java64_home}}\nexport ZOO_LOG_DIR={{zk_log_dir}}\nexport ZOOPIDFILE={{zk_pid_file}}\nexport SERVER_JVMFLAGS={{zk_server_heapsize}}\nexport JAVA=$JAVA_HOME/bin/java\nexport CLASSPATH=$CLASSPATH:/usr/share/zookeeper/*\n\n{% if security_enabled %}\nexport SERVER_JVMFLAGS=\"$SERVER_JVMFLAGS -Djava.security.auth.login.config={{zk_server_jaas_file}}\"\nexport CLIENT_JVMFLAGS=\"$CLIENT_JVMFLAGS -Djava.security.auth.login.config={{zk_client_jaas_file}}\"\n{% endif %}", + "zk_pid_dir":"/var/run/zookeeper", + "zookeeper_principal_name":"zookeeper/[email protected]", + "zookeeper_keytab_path":"/etc/security/keytabs/zk.service.keytab" }, - "configuration_attributes":{ - "core-site":{ - "final":{ - "fs.defaultFS":"true" - } + "zoo.cfg":{ + "clientPort":"2181", + "syncLimit":"5", + "initLimit":"10", + "dataDir":"/hadoop/zookeeper", + "tickTime":"2000" + } + }, + "configurationAttributes":{ + "core-site":{ + "final":{ + "fs.defaultFS":"true" } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/2e579f11/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/execution_commands.json ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/execution_commands.json b/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/execution_commands.json index 1eeeb7c..bf54b97 100644 --- a/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/execution_commands.json +++ b/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/execution_commands.json @@ -1,113 +1,107 @@ { - "clusters":{ - "cl1":{ - "commandsStage":{ - "hostLevelParams":{ - "agent_stack_retry_on_unavailability":"false", - "unlimited_key_jce_required":"false", - "group_list":"[\"hadoop\",\"users\"]", - "host_sys_prepped":"false", - "ambari_db_rca_username":"mapred", - "jdk_name":"jdk-8u112-linux-x64.tar.gz", - "mysql_jdbc_url":"http://gc6401:8080/resources//mysql-connector-java.jar";, - "agent_stack_retry_count":"5", - "user_groups":"{}", - "stack_version":"2.5", - "stack_name":"HDP", - "ambari_db_rca_driver":"org.postgresql.Driver", - "java_home":"/usr/jdk64/jdk1.8.0_112", - "jdk_location":"http://gc6401:8080/resources/";, - "not_managed_hdfs_path_list":"[\"/tmp\"]", - "ambari_db_rca_url":"jdbc:postgresql://gc6401/ambarirca", - "java_version":"8", - "repo_info":"...", - "db_name":"ambari", - "agentCacheDir":"/var/lib/ambari-agent/cache", - "ambari_db_rca_password":"mapred", - "jce_name":"jce_policy-8.zip", - "oracle_jdbc_url":"http://gc6401:8080/resources//ojdbc6.jar";, - "db_driver_filename":"mysql-connector-java.jar", - "user_list":"[\"zookeeper\",\"ambari-qa\",\"hdfs\"]", - "clientsToUpdateConfigs":"[\"*\"]" - }, - "commands":[ - { - "requestId":5, - "taskId":9, - "serviceName":"HDFS", - "role":"DATANODE", - "commandType":"EXECUTION_COMMAND", - "roleCommand":"START", - "configuration_credentials":{ - - }, - "commandParams":{ - "service_package_folder":"common-services/HDFS/2.1.0.2.0/package", - "hooks_folder":"HDP/2.0.6/hooks", - "script":"scripts/datanode.py", - "phase":"INITIAL_START", - "max_duration_for_retries":"600", - "command_retry_enabled":"true", - "command_timeout":"1200", - "refresh_topology":"True", - "script_type":"PYTHON" - } - }, - { - "requestId":6, - "taskId":9, - "serviceName":"ZOOKEEPER", - "role":"ZOOKEEPER_SERVER", - "commandType":"EXECUTION_COMMAND", - "roleCommand":"START", - "configuration_credentials":{ + "0":{ + "hostLevelParams":{ + "agent_stack_retry_on_unavailability":"false", + "unlimited_key_jce_required":"false", + "group_list":"[\"hadoop\",\"users\"]", + "host_sys_prepped":"false", + "ambari_db_rca_username":"mapred", + "jdk_name":"jdk-8u112-linux-x64.tar.gz", + "mysql_jdbc_url":"http://gc6401:8080/resources//mysql-connector-java.jar";, + "agent_stack_retry_count":"5", + "user_groups":"{}", + "stack_version":"2.5", + "stack_name":"HDP", + "ambari_db_rca_driver":"org.postgresql.Driver", + "java_home":"/usr/jdk64/jdk1.8.0_112", + "jdk_location":"http://gc6401:8080/resources/";, + "not_managed_hdfs_path_list":"[\"/tmp\"]", + "ambari_db_rca_url":"jdbc:postgresql://gc6401/ambarirca", + "java_version":"8", + "repo_info":"...", + "db_name":"ambari", + "agentCacheDir":"/var/lib/ambari-agent/cache", + "ambari_db_rca_password":"mapred", + "jce_name":"jce_policy-8.zip", + "oracle_jdbc_url":"http://gc6401:8080/resources//ojdbc6.jar";, + "db_driver_filename":"mysql-connector-java.jar", + "user_list":"[\"zookeeper\",\"ambari-qa\",\"hdfs\"]", + "clientsToUpdateConfigs":"[\"*\"]" + }, + "commands":[ + { + "requestId":5, + "taskId":9, + "serviceName":"HDFS", + "role":"DATANODE", + "commandType":"EXECUTION_COMMAND", + "roleCommand":"START", + "configuration_credentials":{ - }, - "commandParams":{ - "service_package_folder":"common-services/ZOOKEEPER/3.4.5/package", - "hooks_folder":"HDP/2.0.6/hooks", - "script":"scripts/datanode.py", - "phase":"INITIAL_START", - "max_duration_for_retries":"600", - "command_retry_enabled":"true", - "command_timeout":"1200", - "refresh_topology":"True", - "script_type":"PYTHON" - } - } - ] + }, + "commandParams":{ + "service_package_folder":"common-services/HDFS/2.1.0.2.0/package", + "hooks_folder":"HDP/2.0.6/hooks", + "script":"scripts/datanode.py", + "phase":"INITIAL_START", + "max_duration_for_retries":"600", + "command_retry_enabled":"true", + "command_timeout":"1200", + "refresh_topology":"True", + "script_type":"PYTHON" + } }, - "CLUSTER_INDEPENDENT":{ - "commandsStage":{ - "hostLevelParams":{ - "agent_stack_retry_count":"5", - "agent_stack_retry_on_unavailability":"false", - "agentCacheDir":"/var/lib/ambari-agent/cache" - }, - "commands":[ - { - "roleCommand":"ACTIONEXECUTE", - "serviceName":"null", - "role":"check_host", - "commandType":"EXECUTION_COMMAND", - "taskId":2, - "commandParams":{ - "script":"check_host.py", - "check_execute_list":"host_resolution_check", - "threshold":"20", - "hosts":"gc6401", - "command_timeout":"900", - "script_type":"PYTHON" - }, - "roleParams":{ - "threshold":"20", - "check_execute_list":"host_resolution_check", - "hosts":"gc6401" - } - } - ] + { + "requestId":6, + "taskId":9, + "serviceName":"ZOOKEEPER", + "role":"ZOOKEEPER_SERVER", + "commandType":"EXECUTION_COMMAND", + "roleCommand":"START", + "configuration_credentials":{ + + }, + "commandParams":{ + "service_package_folder":"common-services/ZOOKEEPER/3.4.5/package", + "hooks_folder":"HDP/2.0.6/hooks", + "script":"scripts/datanode.py", + "phase":"INITIAL_START", + "max_duration_for_retries":"600", + "command_retry_enabled":"true", + "command_timeout":"1200", + "refresh_topology":"True", + "script_type":"PYTHON" + } + } + ] + }, + "-1":{ + "hostLevelParams":{ + "agent_stack_retry_count":"5", + "agent_stack_retry_on_unavailability":"false", + "agentCacheDir":"/var/lib/ambari-agent/cache" + }, + "commands":[ + { + "roleCommand":"ACTIONEXECUTE", + "serviceName":"null", + "role":"check_host", + "commandType":"EXECUTION_COMMAND", + "taskId":2, + "commandParams":{ + "script":"check_host.py", + "check_execute_list":"host_resolution_check", + "threshold":"20", + "hosts":"gc6401", + "command_timeout":"900", + "script_type":"PYTHON" + }, + "roleParams":{ + "threshold":"20", + "check_execute_list":"host_resolution_check", + "hosts":"gc6401" } } - } + ] } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/2e579f11/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/metadata_after_registration.json ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/metadata_after_registration.json b/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/metadata_after_registration.json index 8ccd8e8..14d7c04 100644 --- a/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/metadata_after_registration.json +++ b/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/metadata_after_registration.json @@ -1,37 +1,33 @@ { - "clusters":{ - "cl1":{ - "metadata":{ - "status_commands_to_run":[ - "STATUS" - ], - "status_commands_timeout":"900", - "hooks_folder":"HDP/2.0.6/hooks", - "credentialStoreEnabled":"false", - "availableServices":{ - "SQOOP":"1.4.6.2.5", - "AMBARI_METRICS":"0.1.0", - "KERBEROS":"1.10.3-10", - "RANGER":"0.6.0.2.5", - "ZEPPELIN":"0.6.0.2.5", - "HDFS":"2.7.3.2.6", - "ZOOKEEPER":"3.4.6" - }, - "agentConfigParams":{ - "agent":{ - "parallel_execution":0, - "use_system_proxy_settings":true - } - }, - "recoveryConfig":{ - "type":"DEFAULT|AUTO_START|AUTO_INSTALL_START|FULL", - "maxCount":10, - "windowInMinutes":60, - "retryGap":0, - "components":"ZOOKEEPER_CLIENT,ZOOKEEPER_SERVER", - "recoveryTimestamp":1458150424380 - } + "0":{ + "status_commands_to_run":[ + "STATUS" + ], + "status_commands_timeout":"900", + "hooks_folder":"HDP/2.0.6/hooks", + "credentialStoreEnabled":"false", + "availableServices":{ + "SQOOP":"1.4.6.2.5", + "AMBARI_METRICS":"0.1.0", + "KERBEROS":"1.10.3-10", + "RANGER":"0.6.0.2.5", + "ZEPPELIN":"0.6.0.2.5", + "HDFS":"2.7.3.2.6", + "ZOOKEEPER":"3.4.6" + }, + "agentConfigParams":{ + "agent":{ + "parallel_execution":0, + "use_system_proxy_settings":true } + }, + "recoveryConfig":{ + "type":"DEFAULT|AUTO_START|AUTO_INSTALL_START|FULL", + "maxCount":10, + "windowInMinutes":60, + "retryGap":0, + "components":"ZOOKEEPER_CLIENT,ZOOKEEPER_SERVER", + "recoveryTimestamp":1458150424380 } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/2e579f11/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/metadata_update.json ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/metadata_update.json b/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/metadata_update.json index 96b929b..4efcd31 100644 --- a/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/metadata_update.json +++ b/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/metadata_update.json @@ -1,10 +1,9 @@ { - "clusters":{ - "cl1":{ - "metadata":{ - "status_commands_to_run":["STATUS", "SECURITY_STATUS"], - "security_type":"kerberos" - } - } + "0":{ + "status_commands_to_run":[ + "STATUS", + "SECURITY_STATUS" + ], + "security_type":"kerberos" } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/2e579f11/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/registration_request.json ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/registration_request.json b/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/registration_request.json index 64077b3..865ad5e 100644 --- a/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/registration_request.json +++ b/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/registration_request.json @@ -1,6 +1,6 @@ { "clusters": { - "cl1": { + "0": { "topologyHash": "fagrewtqwfasfa", "configurationsHash": "mgasdeg443fas", "metadataHash": "1rgw23rfwfaas", http://git-wip-us.apache.org/repos/asf/ambari/blob/2e579f11/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/registration_response.json ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/registration_response.json b/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/registration_response.json index f744410..6873d7b 100644 --- a/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/registration_response.json +++ b/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/registration_response.json @@ -2,7 +2,7 @@ "responseId":0, "exitstatus":0, "errorMessage":"", - "clusters_list":[ - "c1" + "clustersList":[ + "0" ] } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/2e579f11/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/topology_update.json ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/topology_update.json b/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/topology_update.json index 291249c..b403e4d 100644 --- a/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/topology_update.json +++ b/ambari-agent/src/test/python/ambari_agent/dummy_files/stomp/topology_update.json @@ -1,69 +1,69 @@ { - "clusters":{ - "cl1":{ - "topology":{ - "components":{ - "ZOOKEEPER_SERVER":{ - "hosts":[ - 0, - 1, - 4 - ], - "statusCommandsParams":{ - "script":"scripts/zookeeper_server.py", - "service_package_folder":"common-services/ZOOKEEPER/3.4.5/package" - } - }, - "ZOOKEEPER_CLIENT":{ - "hosts":[ - 0, - 1, - 2, - 3, - 4 - ], - "statusCommandsParams":{ - "script":"scripts/zookeeper_client.py", - "service_package_folder":"common-services/ZOOKEEPER/3.4.5/package" - } - } - }, + "0":{ + "components":[ + { + "serviceName":"ZOOKEEPER", + "componentName":"ZOOKEEPER_SERVER", "hosts":[ - { - "hostname":"c6401.ambari.apache.org", - "rack_id":0, - "ipv4_ip":"10.240.0.240" - }, - { - "hostname":"c6402.ambari.apache.org", - "rack_id":1, - "ipv4_ip":"10.240.0.241" - }, - { - "hostname":"c6403.ambari.apache.org", - "rack_id":0, - "ipv4_ip":"10.240.0.242" - }, - { - "hostname":"c6404.ambari.apache.org", - "rack_id":0, - "ipv4_ip":"10.240.0.243" - }, - { - "hostname":"c6405.ambari.apache.org", - "rack_id":1, - "ipv4_ip":"10.240.0.244" - } + 0, + 1, + 4 ], - "racks":[ - { - "name":"/default-rack" - }, - { - "name":"/another-rack" - } - ] + "statusCommandsParams":{ + "script":"scripts/zookeeper_server.py", + "service_package_folder":"common-services/ZOOKEEPER/3.4.5/package" + } + }, + { + "serviceName":"ZOOKEEPER", + "componentName":"ZOOKEEPER_CLIENT", + "hosts":[ + 0, + 1, + 2, + 3, + 4 + ], + "statusCommandsParams":{ + "script":"scripts/zookeeper_client.py", + "service_package_folder":"common-services/ZOOKEEPER/3.4.5/package" + } + } + ], + "hosts":[ + { + "hostname":"c6401.ambari.apache.org", + "rack_id":0, + "ipv4_ip":"10.240.0.240" + }, + { + "hostname":"c6402.ambari.apache.org", + "rack_id":1, + "ipv4_ip":"10.240.0.241" + }, + { + "hostname":"c6403.ambari.apache.org", + "rack_id":0, + "ipv4_ip":"10.240.0.242" + }, + { + "hostname":"c6404.ambari.apache.org", + "rack_id":0, + "ipv4_ip":"10.240.0.243" + }, + { + "hostname":"c6405.ambari.apache.org", + "rack_id":1, + "ipv4_ip":"10.240.0.244" + } + ], + "racks":[ + { + "name":"/default-rack" + }, + { + "name":"/another-rack" } - } + ] } } \ No newline at end of file
