http://git-wip-us.apache.org/repos/asf/ambari/blob/8e5eeb4d/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_upgrade.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_upgrade.py deleted file mode 100644 index bc245c4..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_upgrade.py +++ /dev/null @@ -1,177 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. -import os - -from ambari_commons import yaml_utils -from resource_management.core.logger import Logger -from resource_management.core.exceptions import Fail -from resource_management.core.resources.system import Directory -from resource_management.core.resources.system import File -from resource_management.core.resources.system import Execute -from resource_management.libraries.script.script import Script -from resource_management.libraries.functions.default import default -from resource_management.libraries.functions.format import format - -class StormUpgrade(Script): - """ - Applies to Rolling/Express Upgrade from HDP 2.1 or 2.2 to 2.3 or higher. - - Requirements: Needs to run from a host with ZooKeeper Client. - - This class helps perform some of the upgrade tasks needed for Storm during - a Rolling or Express upgrade. Storm writes data to disk locally and to ZooKeeper. - If any HDP 2.1 or 2.2 bits exist in these directories when an HDP 2.3 instance - starts up, it will fail to start properly. Because the upgrade framework in - Ambari doesn't yet have a mechanism to say "stop all" before starting to - upgrade each component, we need to rely on a Storm trick to bring down - running daemons. By removing the ZooKeeper data with running daemons, those - daemons will die. - """ - - def delete_storm_zookeeper_data(self, env): - """ - Deletes the Storm data from ZooKeeper, effectively bringing down all - Storm daemons. - :return: - """ - import params - - Logger.info('Clearing Storm data from ZooKeeper') - - storm_zookeeper_root_dir = params.storm_zookeeper_root_dir - if storm_zookeeper_root_dir is None: - raise Fail("The storm ZooKeeper directory specified by storm-site/storm.zookeeper.root must be specified") - - # The zookeeper client must be given a zookeeper host to contact. Guaranteed to have at least one host. - storm_zookeeper_server_list = yaml_utils.get_values_from_yaml_array(params.storm_zookeeper_servers) - if storm_zookeeper_server_list is None: - Logger.info("Unable to extract ZooKeeper hosts from '{0}', assuming localhost").format(params.storm_zookeeper_servers) - storm_zookeeper_server_list = ["localhost"] - - # For every zk server, try to remove /storm - zookeeper_data_cleared = False - for storm_zookeeper_server in storm_zookeeper_server_list: - # Determine where the zkCli.sh shell script is - zk_command_location = os.path.join(params.stack_root, "current", "zookeeper-client", "bin", "zkCli.sh") - if params.version is not None: - zk_command_location = os.path.join(params.stack_root, params.version, "zookeeper", "bin", "zkCli.sh") - - # create the ZooKeeper delete command - command = "{0} -server {1}:{2} rmr /storm".format( - zk_command_location, storm_zookeeper_server, params.storm_zookeeper_port) - - # clean out ZK - try: - # the ZK client requires Java to run; ensure it's on the path - env_map = { - 'JAVA_HOME': params.java64_home - } - - # AMBARI-12094: if security is enabled, then we need to tell zookeeper where the - # JAAS file is located since we don't use kinit directly with STORM - if params.security_enabled: - env_map['JVMFLAGS'] = "-Djava.security.auth.login.config={0}".format(params.storm_jaas_file) - - Execute(command, user=params.storm_user, environment=env_map, - logoutput=True, tries=1) - - zookeeper_data_cleared = True - break - except: - # the command failed, try a different ZK server - pass - - # fail if the ZK data could not be cleared - if not zookeeper_data_cleared: - raise Fail("Unable to clear ZooKeeper Storm data on any of the following ZooKeeper hosts: {0}".format( - storm_zookeeper_server_list)) - - - def delete_storm_local_data(self, env): - """ - Deletes Storm data from local directories. This will create a marker file - with JSON data representing the upgrade stack and request/stage ID. This - will prevent multiple Storm components on the same host from removing - the local directories more than once. - :return: - """ - import params - - Logger.info('Clearing Storm data from local directories...') - - storm_local_directory = params.local_dir - if storm_local_directory is None: - raise Fail("The storm local directory specified by storm-site/storm.local.dir must be specified") - - request_id = default("/requestId", None) - - stack_name = params.stack_name - stack_version = params.version - upgrade_direction = params.upgrade_direction - - json_map = {} - json_map["requestId"] = request_id - json_map["stackName"] = stack_name - json_map["stackVersion"] = stack_version - json_map["direction"] = upgrade_direction - - temp_directory = params.tmp_dir - marker_file = os.path.join(temp_directory, "storm-upgrade-{0}.json".format(stack_version)) - Logger.info("Marker file for upgrade/downgrade of Storm, {0}".format(marker_file)) - - if os.path.exists(marker_file): - Logger.info("The marker file exists.") - try: - with open(marker_file) as file_pointer: - existing_json_map = json.load(file_pointer) - - if cmp(json_map, existing_json_map) == 0: - Logger.info("The storm upgrade has already removed the local directories for {0}-{1} for " - "request {2} and direction {3}. Nothing else to do.".format(stack_name, stack_version, request_id, upgrade_direction)) - - # Nothing else to do here for this as it appears to have already been - # removed by another component being upgraded - return - else: - Logger.info("The marker file differs from the new value. Will proceed to delete Storm local dir, " - "and generate new file. Current marker file: {0}".format(str(existing_json_map))) - except Exception, e: - Logger.error("The marker file {0} appears to be corrupt; removing it. Error: {1}".format(marker_file, str(e))) - File(marker_file, action="delete") - else: - Logger.info('The marker file {0} does not exist; will attempt to delete local Storm directory if it exists.'.format(marker_file)) - - # Delete from local directory - if os.path.isdir(storm_local_directory): - Logger.info("Deleting storm local directory, {0}".format(storm_local_directory)) - Directory(storm_local_directory, action="delete", create_parents = True) - - # Recreate storm local directory - Logger.info("Recreating storm local directory, {0}".format(storm_local_directory)) - Directory(storm_local_directory, mode=0755, owner=params.storm_user, - group=params.user_group, create_parents = True) - - # The file doesn't exist, so create it - Logger.info("Saving marker file to {0} with contents: {1}".format(marker_file, str(json_map))) - with open(marker_file, 'w') as file_pointer: - json.dump(json_map, file_pointer, indent=2) - -if __name__ == "__main__": - StormUpgrade().execute() \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/8e5eeb4d/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_yaml_utils.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_yaml_utils.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_yaml_utils.py deleted file mode 100644 index 9d78e71..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_yaml_utils.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import os -import resource_management - -from ambari_commons.yaml_utils import escape_yaml_property -from resource_management.core.source import InlineTemplate -from resource_management.core.resources.system import File - -def replace_jaas_placeholder(name, security_enabled, conf_dir): - if name.find('_JAAS_PLACEHOLDER') > -1: - if security_enabled: - return name.replace('_JAAS_PLACEHOLDER', '-Djava.security.auth.login.config=' + conf_dir + '/storm_jaas.conf') - else: - return name.replace('_JAAS_PLACEHOLDER', '') - else: - return name - -storm_yaml_template = """{% for key, value in configurations|dictsort if not key.startswith('_') %}{{key}} : {{ escape_yaml_property(replace_jaas_placeholder(resource_management.core.source.InlineTemplate(value).get_content().strip(), security_enabled, conf_dir)) }} -{% endfor %}""" - -def yaml_config_template(configurations): - return InlineTemplate(storm_yaml_template, configurations=configurations, - extra_imports=[escape_yaml_property, replace_jaas_placeholder, resource_management, - resource_management.core, resource_management.core.source]) - -def yaml_config(filename, configurations = None, conf_dir = None, owner = None, group = None): - import params - config_content = InlineTemplate('''{% for key, value in configurations_dict|dictsort %}{{ key }}: {{ escape_yaml_property(resource_management.core.source.InlineTemplate(value).get_content()) }} -{% endfor %}''', configurations_dict=configurations, extra_imports=[escape_yaml_property, resource_management, resource_management.core, resource_management.core.source]) - - File (os.path.join(params.conf_dir, filename), - content = config_content, - owner = owner, - mode = "f" - ) http://git-wip-us.apache.org/repos/asf/ambari/blob/8e5eeb4d/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor.py deleted file mode 100644 index da900f9..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import sys -from resource_management.libraries.functions import check_process_status -from resource_management.libraries.script import Script -from resource_management.libraries.functions import conf_select -from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions import format -from resource_management.core.resources.system import Execute -from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions import StackFeature -from storm import storm -from service import service -from ambari_commons import OSConst -from ambari_commons.os_family_impl import OsFamilyImpl -from resource_management.core.resources.service import Service - - -class Supervisor(Script): - def get_component_name(self): - return "storm-supervisor" - - def install(self, env): - self.install_packages(env) - self.configure(env) - - def configure(self, env): - import params - env.set_params(params) - storm("supervisor") - - -@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) -class SupervisorWindows(Supervisor): - def start(self, env): - import status_params - env.set_params(status_params) - self.configure(env) - Service(status_params.supervisor_win_service_name, action="start") - - def stop(self, env): - import status_params - env.set_params(status_params) - Service(status_params.supervisor_win_service_name, action="stop") - - def status(self, env): - import status_params - from resource_management.libraries.functions.windows_service_utils import check_windows_service_status - env.set_params(status_params) - check_windows_service_status(status_params.supervisor_win_service_name) - - -@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) -class SupervisorDefault(Supervisor): - - def pre_upgrade_restart(self, env, upgrade_type=None): - import params - env.set_params(params) - - if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): - conf_select.select(params.stack_name, "storm", params.version) - stack_select.select("storm-client", params.version) - stack_select.select("storm-supervisor", params.version) - - def start(self, env, upgrade_type=None): - import params - env.set_params(params) - self.configure(env) - - service("supervisor", action="start") - service("logviewer", action="start") - - def stop(self, env, upgrade_type=None): - import params - env.set_params(params) - - service("supervisor", action="stop") - service("logviewer", action="stop") - - def status(self, env): - import status_params - env.set_params(status_params) - check_process_status(status_params.pid_supervisor) - - def get_log_folder(self): - import params - return params.log_dir - - def get_user(self): - import params - return params.storm_user - -if __name__ == "__main__": - Supervisor().execute() - http://git-wip-us.apache.org/repos/asf/ambari/blob/8e5eeb4d/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor_prod.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor_prod.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor_prod.py deleted file mode 100644 index d6c3545..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor_prod.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import sys -from storm import storm -from service import service -from supervisord_service import supervisord_service, supervisord_check_status -from resource_management.libraries.script import Script -from resource_management.libraries.functions import conf_select -from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions import format -from resource_management.core.resources.system import Execute -from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions import StackFeature - - -class Supervisor(Script): - - def get_component_name(self): - return "storm-supervisor" - - def install(self, env): - self.install_packages(env) - self.configure(env) - - def configure(self, env): - import params - env.set_params(params) - storm() - - def pre_upgrade_restart(self, env, upgrade_type=None): - import params - env.set_params(params) - - if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): - conf_select.select(params.stack_name, "storm", params.version) - stack_select.select("storm-client", params.version) - stack_select.select("storm-supervisor", params.version) - - def start(self, env, upgrade_type=None): - import params - env.set_params(params) - self.configure(env) - - supervisord_service("supervisor", action="start") - service("logviewer", action="start") - - def stop(self, env, upgrade_type=None): - import params - env.set_params(params) - - supervisord_service("supervisor", action="stop") - service("logviewer", action="stop") - - def status(self, env): - supervisord_check_status("supervisor") - - def get_log_folder(self): - import params - return params.log_dir - - def get_user(self): - import params - return params.storm_user - -if __name__ == "__main__": - Supervisor().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/8e5eeb4d/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisord_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisord_service.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisord_service.py deleted file mode 100644 index 6ff9f9c..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisord_service.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.core.resources.system import Execute -from resource_management.libraries.functions.format import format - -def supervisord_service(component_name, action): - Execute(format("supervisorctl {action} storm-{component_name}"), - wait_for_finish=False - ) - -def supervisord_check_status(component_name): - try: - Execute(format("supervisorctl status storm-{component_name} | grep RUNNING")) - except Fail: - raise ComponentIsNotRunning() http://git-wip-us.apache.org/repos/asf/ambari/blob/8e5eeb4d/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/ui_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/ui_server.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/ui_server.py deleted file mode 100644 index 6551067..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/ui_server.py +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import sys -from storm import storm -from service import service -from service_check import ServiceCheck -from resource_management.libraries.functions import check_process_status -from resource_management.libraries.script import Script -from resource_management.libraries.functions import conf_select -from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions import format -from resource_management.core.resources.system import Link -from resource_management.core.resources.system import Execute -from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions import StackFeature -from resource_management.libraries.functions.security_commons import build_expectations, \ - cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ - FILE_TYPE_JAAS_CONF -from setup_ranger_storm import setup_ranger_storm -from ambari_commons import OSConst -from ambari_commons.os_family_impl import OsFamilyImpl -from resource_management.core.resources.service import Service - - -class UiServer(Script): - - def get_component_name(self): - return "storm-client" - - def install(self, env): - self.install_packages(env) - self.configure(env) - - def configure(self, env): - import params - env.set_params(params) - storm("ui") - -@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) -class UiServerWindows(UiServer): - def start(self, env): - import status_params - env.set_params(status_params) - self.configure(env) - Service(status_params.ui_win_service_name, action="start") - - def stop(self, env): - import status_params - env.set_params(status_params) - Service(status_params.ui_win_service_name, action="stop") - - def status(self, env): - import status_params - env.set_params(status_params) - from resource_management.libraries.functions.windows_service_utils import check_windows_service_status - check_windows_service_status(status_params.ui_win_service_name) - - -@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) -class UiServerDefault(UiServer): - - def pre_upgrade_restart(self, env, upgrade_type=None): - import params - env.set_params(params) - if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): - conf_select.select(params.stack_name, "storm", params.version) - stack_select.select("storm-client", params.version) - - def link_metrics_sink_jar(self): - # Add storm metrics reporter JAR to storm-ui-server classpath. - # Remove symlinks. They can be there, if you doing upgrade from HDP < 2.2 to HDP >= 2.2 - Link(format("{storm_lib_dir}/ambari-metrics-storm-sink.jar"), - action="delete") - # On old HDP 2.1 versions, this symlink may also exist and break EU to newer versions - Link("/usr/lib/storm/lib/ambari-metrics-storm-sink.jar", action="delete") - - Execute(format("{sudo} ln -s {metric_collector_sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar"), - not_if=format("ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"), - only_if=format("ls {metric_collector_sink_jar}") - ) - - def start(self, env, upgrade_type=None): - import params - env.set_params(params) - self.configure(env) - self.link_metrics_sink_jar() - setup_ranger_storm(upgrade_type=upgrade_type) - service("ui", action="start") - - def stop(self, env, upgrade_type=None): - import params - env.set_params(params) - service("ui", action="stop") - - def status(self, env): - import status_params - env.set_params(status_params) - check_process_status(status_params.pid_ui) - - def security_status(self, env): - import status_params - - env.set_params(status_params) - - if status_params.security_enabled: - # Expect the following files to be available in status_params.config_dir: - # storm_jaas.conf - - try: - props_value_check = None - props_empty_check = ['storm_ui_principal_name', 'storm_ui_keytab'] - props_read_check = ['storm_ui_keytab'] - storm_env_expectations = build_expectations('storm_ui', props_value_check, props_empty_check, - props_read_check) - - storm_expectations = {} - storm_expectations.update(storm_env_expectations) - - security_params = {} - security_params['storm_ui'] = {} - security_params['storm_ui']['storm_ui_principal_name'] = status_params.storm_ui_principal - security_params['storm_ui']['storm_ui_keytab'] = status_params.storm_ui_keytab - - result_issues = validate_security_config_properties(security_params, storm_expectations) - if not result_issues: # If all validations passed successfully - # Double check the dict before calling execute - if ( 'storm_ui' not in security_params - or 'storm_ui_principal_name' not in security_params['storm_ui'] - or 'storm_ui_keytab' not in security_params['storm_ui']): - self.put_structured_out({"securityState": "ERROR"}) - self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."}) - return - - cached_kinit_executor(status_params.kinit_path_local, - status_params.storm_user, - security_params['storm_ui']['storm_ui_keytab'], - security_params['storm_ui']['storm_ui_principal_name'], - status_params.hostname, - status_params.tmp_dir) - self.put_structured_out({"securityState": "SECURED_KERBEROS"}) - else: - issues = [] - for cf in result_issues: - issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) - self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) - self.put_structured_out({"securityState": "UNSECURED"}) - except Exception as e: - self.put_structured_out({"securityState": "ERROR"}) - self.put_structured_out({"securityStateErrorInfo": str(e)}) - else: - self.put_structured_out({"securityState": "UNSECURED"}) - - def get_log_folder(self): - import params - return params.log_dir - - def get_user(self): - import params - return params.storm_user - -if __name__ == "__main__": - UiServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/8e5eeb4d/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/client_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/client_jaas.conf.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/client_jaas.conf.j2 deleted file mode 100644 index e8dc122..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/client_jaas.conf.j2 +++ /dev/null @@ -1,24 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -StormClient { - com.sun.security.auth.module.Krb5LoginModule required - useTicketCache=true - renewTicket=true - serviceName="{{nimbus_bare_jaas_principal}}"; -}; http://git-wip-us.apache.org/repos/asf/ambari/blob/8e5eeb4d/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/config.yaml.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/config.yaml.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/config.yaml.j2 deleted file mode 100644 index a9760cb..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/config.yaml.j2 +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -nimbusHost: {{nimbus_host}} -nimbusPort: {{nimbus_port}} - -# HTTP-specific options. -http: - - # The port on which the HTTP server listens for service requests. - port: {{rest_api_port}} - - # The port on which the HTTP server listens for administrative requests. - adminPort: {{rest_api_admin_port}} - -{% if ganglia_installed %} -enableGanglia: {{ganglia_installed}} - -# ganglia configuration (necessary if ganglia reporting is enabled) -ganglia: - - # how often to report to ganglia metrics (in seconds) - reportInterval: {{ganglia_report_interval}} - - # the hostname of the gmond server where storm cluster metrics will be sent - host: "{{ganglia_server}}" - - # address mode - # default is MULTICAST - addressMode: "UNICAST" - - # an <IP>:<HOSTNAME> pair to spoof - # this allows us to simulate storm cluster metrics coming from a specific host - #spoof: "192.168.1.1:storm" -{% endif %} - -{% if has_metric_collector and stack_supports_storm_ams %} -enableGanglia: False - -ganglia: - reportInterval: {{metric_collector_report_interval}} - -enableMetricsSink: True - -metrics_collector: - - reportInterval: {{metric_collector_report_interval}} - collector: "{{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}}" - appId: "{{metric_collector_app_id}}" - - # HTTPS settings - truststore.path : "{{metric_truststore_path}}" - truststore.type : "{{metric_truststore_type}}" - truststore.password : "{{metric_truststore_password}}" - -{% endif %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/8e5eeb4d/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm-metrics2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm-metrics2.properties.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm-metrics2.properties.j2 deleted file mode 100644 index 9acf173..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm-metrics2.properties.j2 +++ /dev/null @@ -1,26 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}} -maxRowCacheSize=10000 -sendInterval={{metrics_report_interval}}000 - -# HTTPS properties -truststore.path = {{metric_truststore_path}} -truststore.type = {{metric_truststore_type}} -truststore.password = {{metric_truststore_password}} http://git-wip-us.apache.org/repos/asf/ambari/blob/8e5eeb4d/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm.conf.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm.conf.j2 deleted file mode 100644 index 82a26fe..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm.conf.j2 +++ /dev/null @@ -1,35 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{storm_user}} - nofile {{storm_user_nofile_limit}} -{{storm_user}} - nproc {{storm_user_nproc_limit}} http://git-wip-us.apache.org/repos/asf/ambari/blob/8e5eeb4d/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm_jaas.conf.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm_jaas.conf.j2 deleted file mode 100644 index 8116492..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm_jaas.conf.j2 +++ /dev/null @@ -1,57 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} -{% if stack_supports_storm_kerberos %} -StormServer { - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - keyTab="{{nimbus_keytab_path}}" - storeKey=true - useTicketCache=false - principal="{{nimbus_jaas_principal}}"; -}; -StormClient { - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - keyTab="{{storm_keytab_path}}" - storeKey=true - useTicketCache=false - serviceName="{{nimbus_bare_jaas_principal}}" - principal="{{storm_jaas_principal}}"; -}; -{% endif %} -Client { - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - keyTab="{{storm_keytab_path}}" - storeKey=true - useTicketCache=false - serviceName="zookeeper" - principal="{{storm_jaas_principal}}"; -}; - -{% if kafka_bare_jaas_principal %} -KafkaClient { - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - keyTab="{{storm_keytab_path}}" - storeKey=true - useTicketCache=false - serviceName="{{kafka_bare_jaas_principal}}" - principal="{{storm_jaas_principal}}"; -}; -{% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/8e5eeb4d/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/worker-launcher.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/worker-launcher.cfg.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/worker-launcher.cfg.j2 deleted file mode 100644 index 2228601..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/worker-launcher.cfg.j2 +++ /dev/null @@ -1,19 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} -storm.worker-launcher.group={{user_group}} -min.user.id={{min_user_ruid}} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/8e5eeb4d/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/quicklinks/quicklinks.json b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/quicklinks/quicklinks.json deleted file mode 100644 index 492f0a0..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/quicklinks/quicklinks.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "default", - "description": "default quick links configuration", - "configuration": { - "protocol": - { - "type":"HTTP_ONLY" - }, - - "links": [ - { - "name": "storm_ui", - "label": "Storm UI", - "requires_user_name": "false", - "component_name": "STORM_UI_SERVER", - "url":"%@://%@:%@/", - "port":{ - "http_property": "ui.port", - "http_default_port": "8744", - "https_property": "ui.port", - "https_default_port": "8744", - "regex": "^(\\d+)$", - "site": "storm-site" - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/8e5eeb4d/ambari-server/src/main/resources/common-services/STORM/0.9.1/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/alerts.json b/ambari-server/src/main/resources/common-services/STORM/0.9.1/alerts.json new file mode 100644 index 0000000..ae03617 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/alerts.json @@ -0,0 +1,194 @@ +{ + "STORM": { + "service": [ + { + "name": "storm_supervisor_process_percent", + "label": "Percent Supervisors Available", + "interval": 1, + "scope": "SERVICE", + "enabled": true, + "source": { + "type": "AGGREGATE", + "alert_name": "storm_supervisor_process", + "reporting": { + "ok": { + "text": "affected: [{1}], total: [{0}]" + }, + "warning": { + "text": "affected: [{1}], total: [{0}]", + "value": 10 + }, + "critical": { + "text": "affected: [{1}], total: [{0}]", + "value": 30 + }, + "units" : "%", + "type": "PERCENT" + } + } + } + ], + "STORM_UI_SERVER": [ + { + "name": "storm_server_process", + "label": "Storm Server Process", + "interval": 1, + "scope": "ANY", + "enabled": true, + "source": { + "type": "PORT", + "uri": "{{storm-site/ui.port}}", + "default_port": 8744, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + }, + { + "name": "storm_webui", + "label": "Storm Web UI", + "interval": 1, + "scope": "ANY", + "enabled": true, + "source": { + "type": "WEB", + "uri": { + "http": "{{storm-site/ui.port}}", + "kerberos_keytab": "{{storm-env/storm_ui_keytab}}", + "kerberos_principal": "{{storm-env/storm_ui_principal_name}}", + "connection_timeout": 5.0 + }, + "reporting": { + "ok": { + "text": "HTTP {0} response in {2:.3f}s" + }, + "warning":{ + "text": "HTTP {0} response from {1} in {2:.3f}s ({3})" + }, + "critical": { + "text": "Connection failed to {1} ({3})" + } + } + } + } + ], + "NIMBUS": [ + { + "name": "storm_nimbus_process", + "label": "Nimbus Process", + "interval": 1, + "scope": "ANY", + "enabled": true, + "source": { + "type": "PORT", + "uri": "{{storm-site/nimbus.thrift.port}}", + "default_port": 6627, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + } + ], + "DRPC_SERVER": [ + { + "name": "storm_drpc_server", + "label": "DRPC Server Process", + "interval": 1, + "scope": "ANY", + "enabled": true, + "source": { + "type": "PORT", + "uri": "{{storm-site/drpc.port}}", + "default_port": 3772, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + } + ], + "STORM_REST_API": [ + { + "name": "storm_rest_api", + "label": "Storm REST API", + "interval": 1, + "scope": "ANY", + "enabled": true, + "source": { + "type": "PORT", + "uri": "8745", + "default_port": 8745, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + } + ], + "SUPERVISOR": [ + { + "name": "storm_supervisor_process", + "label": "Supervisor Process", + "interval": 1, + "scope": "HOST", + "enabled": true, + "source": { + "type": "PORT", + "uri": "{{storm-env/jmxremote_port}}", + "default_port": 56431, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + } + ] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/8e5eeb4d/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-env.xml b/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-env.xml new file mode 100644 index 0000000..75b080a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-env.xml @@ -0,0 +1,139 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_adding_forbidden="true"> + <property> + <name>storm_user</name> + <display-name>Storm User</display-name> + <value>storm</value> + <property-type>USER</property-type> + <description></description> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> + </property> + <property> + <name>storm_log_dir</name> + <value>/var/log/storm</value> + <description></description> + <value-attributes> + <type>directory</type> + </value-attributes> + </property> + <property> + <name>storm_pid_dir</name> + <value>/var/run/storm</value> + <description></description> + <value-attributes> + <type>directory</type> + </value-attributes> + </property> + <property> + <name>jmxremote_port</name> + <value>56431</value> + <description></description> + </property> + + + <property> + <name>storm_principal_name</name> + <description>Storm principal name</description> + </property> + + <property> + <name>storm_principal_name</name> + <description>Storm principal name</description> + </property> + + <property> + <name>storm_keytab</name> + <description>Storm keytab path</description> + </property> + + <property> + <name>storm_ui_principal_name</name> + <description>Storm UI principal name</description> + </property> + + <property> + <name>storm_ui_keytab</name> + <description>Storm UI keytab path</description> + </property> + + <property> + <name>nimbus_keytab</name> + <description>Nimbus keytab path</description> + </property> + + <property> + <name>nimbus_principal_name</name> + <description>Nimbus principal name</description> + </property> + + <property> + <name>storm_user_nofile_limit</name> + <value>128000</value> + <description>Max open files limit setting for STORM user.</description> + </property> + <property> + <name>storm_user_nproc_limit</name> + <value>65536</value> + <description>Max number of processes limit setting for STORM user.</description> + </property> + + <!-- storm-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for storm-env.sh file</description> + <value> +#!/bin/bash + +# Set Storm specific environment variables here. + +# The java implementation to use. +export JAVA_HOME={{java64_home}} + +# Storm log folder +export STORM_LOG_DIR={{log_dir}} + +export STORM_CONF_DIR={{conf_dir}} +export STORM_HOME={{storm_component_home_dir}} + </value> + <value-attributes> + <type>content</type> + </value-attributes> + </property> + + <property> + <name>nimbus_seeds_supported</name> + <value>false</value> + <description></description> + </property> + <property> + <name>storm_logs_supported</name> + <value>false</value> + <description></description> + </property> + + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/8e5eeb4d/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-site.xml b/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-site.xml new file mode 100644 index 0000000..01b3f4e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-site.xml @@ -0,0 +1,740 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_final="true"> + <property> + <name>java.library.path</name> + <value>/usr/local/lib:/opt/local/lib:/usr/lib</value> + <description>This value is passed to spawned JVMs (e.g., Nimbus, Supervisor, and Workers) + for the java.library.path value. java.library.path tells the JVM where + to look for native libraries. It is necessary to set this config correctly since + Storm uses the ZeroMQ and JZMQ native libs. </description> + </property> + <property> + <name>storm.local.dir</name> + <value>/hadoop/storm</value> + <description>A directory on the local filesystem used by Storm for any local + filesystem usage it needs. The directory must exist and the Storm daemons must + have permission to read/write from this location.</description> + <value-attributes> + <type>directory</type> + </value-attributes> + </property> + <property> + <name>storm.zookeeper.servers</name> + <value>['localhost']</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> + <description>A list of hosts of ZooKeeper servers used to manage the cluster.</description> + <value-attributes> + <type>multiLine</type> + <overridable>false</overridable> + </value-attributes> + </property> + <property> + <name>storm.zookeeper.port</name> + <value>2181</value> + <description>The port Storm will use to connect to each of the ZooKeeper servers.</description> + <value-attributes> + <type>int</type> + </value-attributes> + </property> + <property> + <name>storm.zookeeper.root</name> + <value>/storm</value> + <description>The root location at which Storm stores data in ZooKeeper.</description> + <value-attributes> + <type>directory</type> + </value-attributes> + </property> + <property> + <name>storm.zookeeper.session.timeout</name> + <value>20000</value> + <description>The session timeout for clients to ZooKeeper.</description> + <value-attributes> + <type>int</type> + </value-attributes> + </property> + <property> + <name>storm.zookeeper.connection.timeout</name> + <value>15000</value> + <description>The connection timeout for clients to ZooKeeper.</description> + <value-attributes> + <type>int</type> + </value-attributes> + </property> + <property> + <name>storm.zookeeper.retry.times</name> + <value>5</value> + <description>The number of times to retry a Zookeeper operation.</description> + <value-attributes> + <type>int</type> + </value-attributes> + </property> + <property> + <name>storm.zookeeper.retry.interval</name> + <value>1000</value> + <description>The interval between retries of a Zookeeper operation.</description> + <value-attributes> + <unit>ms</unit> + <type>int</type> + </value-attributes> + </property> + <property> + <name>storm.zookeeper.retry.intervalceiling.millis</name> + <value>30000</value> + <description>The ceiling of the interval between retries of a Zookeeper operation.</description> + <value-attributes> + <type>int</type> + <unit>ms</unit> + </value-attributes> + </property> + <property> + <name>storm.cluster.mode</name> + <value>distributed</value> + <description>The mode this Storm cluster is running in. Either "distributed" or "local".</description> + </property> + <property> + <name>storm.local.mode.zmq</name> + <value>false</value> + <description>Whether or not to use ZeroMQ for messaging in local mode. If this is set + to false, then Storm will use a pure-Java messaging system. The purpose + of this flag is to make it easy to run Storm in local mode by eliminating + the need for native dependencies, which can be difficult to install. + </description> + <value-attributes> + <type>boolean</type> + </value-attributes> + </property> + <property> + <name>storm.thrift.transport</name> + <value>backtype.storm.security.auth.SimpleTransportPlugin</value> + <description>The transport plug-in for Thrift client/server communication.</description> + </property> + <property> + <name>storm.messaging.transport</name> + <value>backtype.storm.messaging.netty.Context</value> + <description>The transporter for communication among Storm tasks.</description> + </property> + <property> + <name>nimbus.host</name> + <value>localhost</value> + <description>The host that the master server is running on.</description> + <value-attributes> + <type>componentHost</type> + <editable-only-at-install>true</editable-only-at-install> + <overridable>false</overridable> + </value-attributes> + </property> + <property> + <name>nimbus.thrift.port</name> + <value>6627</value> + <description> Which port the Thrift interface of Nimbus should run on. Clients should + connect to this port to upload jars and submit topologies.</description> + <value-attributes> + <type>int</type> + </value-attributes> + </property> + <property> + <name>nimbus.thrift.max_buffer_size</name> + <value>1048576</value> + <description>The maximum buffer size thrift should use when reading messages.</description> + <value-attributes> + <type>int</type> + <unit>bytes</unit> + </value-attributes> + </property> + <property> + <name>nimbus.childopts</name> + <value>-Xmx1024m -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8649,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM</value> + <description>This parameter is used by the storm-deploy project to configure the jvm options for the nimbus daemon.</description> + <value-attributes> + <type>multiLine</type> + <overridable>false</overridable> + </value-attributes> + </property> + <property> + <name>nimbus.task.timeout.secs</name> + <value>30</value> + <description>How long without heartbeating a task can go before nimbus will consider the task dead and reassign it to another location.</description> + <value-attributes> + <type>int</type> + <unit>seconds</unit> + </value-attributes> + </property> + <property> + <name>nimbus.supervisor.timeout.secs</name> + <value>60</value> + <description>How long before a supervisor can go without heartbeating before nimbus considers it dead and stops assigning new work to it.</description> + <value-attributes> + <type>int</type> + </value-attributes> + </property> + <property> + <name>nimbus.monitor.freq.secs</name> + <value>120</value> + <description> + How often nimbus should wake up to check heartbeats and do reassignments. Note + that if a machine ever goes down Nimbus will immediately wake up and take action. + This parameter is for checking for failures when there's no explicit event like that occuring. + </description> + <value-attributes> + <type>int</type> + <unit>seconds</unit> + </value-attributes> + </property> + <property> + <name>nimbus.cleanup.inbox.freq.secs</name> + <value>600</value> + <description>How often nimbus should wake the cleanup thread to clean the inbox.</description> + <value-attributes> + <type>int</type> + <unit>seconds</unit> + </value-attributes> + </property> + <property> + <name>nimbus.inbox.jar.expiration.secs</name> + <value>3600</value> + <description> + The length of time a jar file lives in the inbox before being deleted by the cleanup thread. + + Probably keep this value greater than or equal to NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS. + Note that the time it takes to delete an inbox jar file is going to be somewhat more than + NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS (depending on how often NIMBUS_CLEANUP_FREQ_SECS is set to). + </description> + <value-attributes> + <type>int</type> + <unit>seconds</unit> + </value-attributes> + </property> + <property> + <name>nimbus.task.launch.secs</name> + <value>120</value> + <description>A special timeout used when a task is initially launched. During launch, this is the timeout + used until the first heartbeat, overriding nimbus.task.timeout.secs.</description> + <value-attributes> + <type>int</type> + <unit>seconds</unit> + </value-attributes> + </property> + <property> + <name>nimbus.reassign</name> + <value>true</value> + <description>Whether or not nimbus should reassign tasks if it detects that a task goes down. + Defaults to true, and it's not recommended to change this value.</description> + <value-attributes> + <type>boolean</type> + </value-attributes> + </property> + <property> + <name>nimbus.file.copy.expiration.secs</name> + <value>600</value> + <description>During upload/download with the master, how long an upload or download connection is idle + before nimbus considers it dead and drops the connection.</description> + <value-attributes> + <type>int</type> + <unit>seconds</unit> + </value-attributes> + </property> + <property> + <name>nimbus.topology.validator</name> + <value>backtype.storm.nimbus.DefaultTopologyValidator</value> + <description>A custom class that implements ITopologyValidator that is run whenever a + topology is submitted. Can be used to provide business-specific logic for + whether topologies are allowed to run or not.</description> + </property> + <property> + <name>ui.port</name> + <value>8744</value> + <description>Storm UI binds to this port.</description> + <value-attributes> + <type>int</type> + </value-attributes> + </property> + <property> + <name>ui.childopts</name> + <value>-Xmx768m</value> + <description>Childopts for Storm UI Java process.</description> + </property> + <property> + <name>logviewer.port</name> + <value>8000</value> + <description>HTTP UI port for log viewer.</description> + </property> + <property> + <name>logviewer.childopts</name> + <value>-Xmx128m</value> + <description>Childopts for log viewer java process.</description> + </property> + <property> + <name>logviewer.appender.name</name> + <value>A1</value> + <description>Appender name used by log viewer to determine log directory.</description> + </property> + <property> + <name>drpc.port</name> + <value>3772</value> + <description>This port is used by Storm DRPC for receiving DPRC requests from clients.</description> + <value-attributes> + <type>int</type> + </value-attributes> + </property> + <property> + <name>drpc.worker.threads</name> + <value>64</value> + <description>DRPC thrift server worker threads.</description> + <value-attributes> + <type>int</type> + </value-attributes> + </property> + <property> + <name>drpc.queue.size</name> + <value>128</value> + <description>DRPC thrift server queue size.</description> + <value-attributes> + <type>int</type> + </value-attributes> + </property> + <property> + <name>drpc.invocations.port</name> + <value>3773</value> + <description>This port on Storm DRPC is used by DRPC topologies to receive function invocations and send results back.</description> + <value-attributes> + <type>int</type> + </value-attributes> + </property> + <property> + <name>drpc.request.timeout.secs</name> + <value>600</value> + <description>The timeout on DRPC requests within the DRPC server. Defaults to 10 minutes. Note that requests can also + timeout based on the socket timeout on the DRPC client, and separately based on the topology message + timeout for the topology implementing the DRPC function.</description> + <value-attributes> + <type>int</type> + <unit>seconds</unit> + </value-attributes> + </property> + <property> + <name>drpc.childopts</name> + <value>-Xmx768m</value> + <description>Childopts for Storm DRPC Java process.</description> + </property> + <property> + <name>transactional.zookeeper.root</name> + <value>/transactional</value> + <description>The root directory in ZooKeeper for metadata about TransactionalSpouts.</description> + </property> + <property> + <name>transactional.zookeeper.servers</name> + <value>null</value> + <description>The list of zookeeper servers in which to keep the transactional state. If null (which is default), + will use storm.zookeeper.servers</description> + </property> + <property> + <name>transactional.zookeeper.port</name> + <value>null</value> + <description>The port to use to connect to the transactional zookeeper servers. If null (which is default), + will use storm.zookeeper.port</description> + </property> + <property> + <name>supervisor.slots.ports</name> + <value>[6700, 6701]</value> + <description>A list of ports that can run workers on this supervisor. Each worker uses one port, and + the supervisor will only run one worker per port. Use this configuration to tune + how many workers run on each machine.</description> + </property> + <property> + <name>supervisor.childopts</name> + <value>-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port={{jmxremote_port}} -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM</value> + <description>This parameter is used by the storm-deploy project to configure the jvm options for the supervisor daemon.</description> + <value-attributes> + <type>multiLine</type> + <overridable>false</overridable> + </value-attributes> + </property> + <property> + <name>supervisor.worker.start.timeout.secs</name> + <value>120</value> + <description>How long a worker can go without heartbeating during the initial launch before + the supervisor tries to restart the worker process. This value override + supervisor.worker.timeout.secs during launch because there is additional + overhead to starting and configuring the JVM on launch.</description> + <value-attributes> + <type>int</type> + <unit>seconds</unit> + </value-attributes> + </property> + <property> + <name>supervisor.worker.timeout.secs</name> + <value>30</value> + <description>How long a worker can go without heartbeating before the supervisor tries to restart the worker process.</description> + <value-attributes> + <type>int</type> + <unit>seconds</unit> + </value-attributes> + </property> + <property> + <name>supervisor.monitor.frequency.secs</name> + <value>3</value> + <description>How often the supervisor checks the worker heartbeats to see if any of them need to be restarted.</description> + <value-attributes> + <type>int</type> + <unit>seconds</unit> + </value-attributes> + </property> + <property> + <name>supervisor.heartbeat.frequency.secs</name> + <value>5</value> + <description>How often the supervisor sends a heartbeat to the master.</description> + <value-attributes> + <type>int</type> + <unit>seconds</unit> + </value-attributes> + </property> + <property> + <name>worker.childopts</name> + <value>-Xmx768m -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM</value> + <description>The jvm opts provided to workers launched by this supervisor. All \"%ID%\" substrings are replaced with an identifier for this worker.</description> + <value-attributes> + <type>multiLine</type> + </value-attributes> + </property> + <property> + <name>worker.heartbeat.frequency.secs</name> + <value>1</value> + <description>How often this worker should heartbeat to the supervisor.</description> + </property> + <property> + <name>task.heartbeat.frequency.secs</name> + <value>3</value> + <description>How often a task should heartbeat its status to the master.</description> + </property> + <property> + <name>task.refresh.poll.secs</name> + <value>10</value> + <description>How often a task should sync its connections with other tasks (if a task is + reassigned, the other tasks sending messages to it need to refresh their connections). + In general though, when a reassignment happens other tasks will be notified + almost immediately. This configuration is here just in case that notification doesn't + come through.</description> + </property> + <property> + <name>zmq.threads</name> + <value>1</value> + <description>The number of threads that should be used by the zeromq context in each worker process.</description> + </property> + <property> + <name>zmq.linger.millis</name> + <value>5000</value> + <description>How long a connection should retry sending messages to a target host when + the connection is closed. This is an advanced configuration and can almost + certainly be ignored.</description> + </property> + <property> + <name>zmq.hwm</name> + <value>0</value> + <description>The high water for the ZeroMQ push sockets used for networking. Use this config to prevent buffer explosion + on the networking layer.</description> + </property> + <property> + <name>storm.messaging.netty.server_worker_threads</name> + <value>1</value> + <description>Netty based messaging: The # of worker threads for the server.</description> + <value-attributes> + <type>int</type> + </value-attributes> + </property> + <property> + <name>storm.messaging.netty.client_worker_threads</name> + <value>1</value> + <description>Netty based messaging: The # of worker threads for the client.</description> + <value-attributes> + <type>int</type> + </value-attributes> + </property> + <property> + <name>storm.messaging.netty.buffer_size</name> + <value>5242880</value> + <description>Netty based messaging: The buffer size for send/recv buffer.</description> + <value-attributes> + <type>int</type> + <unit>bytes</unit> + </value-attributes> + </property> + <property> + <name>storm.messaging.netty.max_retries</name> + <value>30</value> + <description>Netty based messaging: The max # of retries that a peer will perform when a remote is not accessible.</description> + <value-attributes> + <type>int</type> + </value-attributes> + </property> + <property> + <name>storm.messaging.netty.max_wait_ms</name> + <value>1000</value> + <description>Netty based messaging: The max # of milliseconds that a peer will wait.</description> + <value-attributes> + <type>int</type> + <unit>ms</unit> + </value-attributes> + </property> + <property> + <name>storm.messaging.netty.min_wait_ms</name> + <value>100</value> + <description>Netty based messaging: The min # of milliseconds that a peer will wait.</description> + <value-attributes> + <type>int</type> + <unit>ms</unit> + </value-attributes> + </property> + <property> + <name>topology.enable.message.timeouts</name> + <value>true</value> + <description>True if Storm should timeout messages or not. Defaults to true. This is meant to be used + in unit tests to prevent tuples from being accidentally timed out during the test.</description> + </property> + <property> + <name>topology.debug</name> + <value>false</value> + <description>When set to true, Storm will log every message that's emitted.</description> + </property> + <property> + <name>topology.optimize</name> + <value>true</value> + <description>Whether or not the master should optimize topologies by running multiple tasks in a single thread where appropriate.</description> + </property> + <property> + <name>topology.workers</name> + <value>1</value> + <description>How many processes should be spawned around the cluster to execute this + topology. Each process will execute some number of tasks as threads within + them. This parameter should be used in conjunction with the parallelism hints + on each component in the topology to tune the performance of a topology.</description> + </property> + <property> + <name>topology.acker.executors</name> + <value>null</value> + <description>How many executors to spawn for ackers. + + If this is set to 0, then Storm will immediately ack tuples as soon + as they come off the spout, effectively disabling reliability. + </description> + </property> + <property> + <name>topology.message.timeout.secs</name> + <value>30</value> + <description>The maximum amount of time given to the topology to fully process a message + emitted by a spout. If the message is not acked within this time frame, Storm + will fail the message on the spout. Some spouts implementations will then replay + the message at a later time.</description> + </property> + <property> + <name>topology.skip.missing.kryo.registrations</name> + <value>false</value> + <description> Whether or not Storm should skip the loading of kryo registrations for which it + does not know the class or have the serializer implementation. Otherwise, the task will + fail to load and will throw an error at runtime. The use case of this is if you want to + declare your serializations on the storm.yaml files on the cluster rather than every single + time you submit a topology. Different applications may use different serializations and so + a single application may not have the code for the other serializers used by other apps. + By setting this config to true, Storm will ignore that it doesn't have those other serializations + rather than throw an error.</description> + </property> + <property> + <name>topology.max.task.parallelism</name> + <value>null</value> + <description>The maximum parallelism allowed for a component in this topology. This configuration is + typically used in testing to limit the number of threads spawned in local mode.</description> + </property> + <property> + <name>topology.max.spout.pending</name> + <value>1000</value> + <description>The maximum number of tuples that can be pending on a spout task at any given time. + This config applies to individual tasks, not to spouts or topologies as a whole. + + A pending tuple is one that has been emitted from a spout but has not been acked or failed yet. + Note that this config parameter has no effect for unreliable spouts that don't tag + their tuples with a message id.</description> + </property> + <property> + <name>topology.state.synchronization.timeout.secs</name> + <value>60</value> + <description>The maximum amount of time a component gives a source of state to synchronize before it requests + synchronization again.</description> + </property> + <property> + <name>topology.stats.sample.rate</name> + <value>0.05</value> + <description>The percentage of tuples to sample to produce stats for a task.</description> + </property> + <property> + <name>topology.builtin.metrics.bucket.size.secs</name> + <value>60</value> + <description>The time period that builtin metrics data in bucketed into.</description> + </property> + <property> + <name>topology.fall.back.on.java.serialization</name> + <value>true</value> + <description>Whether or not to use Java serialization in a topology.</description> + </property> + <property> + <name>topology.worker.childopts</name> + <value>null</value> + <description>Topology-specific options for the worker child process. This is used in addition to WORKER_CHILDOPTS.</description> + </property> + <property> + <name>topology.executor.receive.buffer.size</name> + <value>1024</value> + <description>The size of the Disruptor receive queue for each executor. Must be a power of 2.</description> + </property> + <property> + <name>topology.executor.send.buffer.size</name> + <value>1024</value> + <description>The size of the Disruptor send queue for each executor. Must be a power of 2.</description> + </property> + <property> + <name>topology.receiver.buffer.size</name> + <value>8</value> + <description>The maximum number of messages to batch from the thread receiving off the network to the + executor queues. Must be a power of 2.</description> + </property> + <property> + <name>topology.transfer.buffer.size</name> + <value>1024</value> + <description>The size of the Disruptor transfer queue for each worker.</description> + </property> + <property> + <name>topology.tick.tuple.freq.secs</name> + <value>null</value> + <description>How often a tick tuple from the "__system" component and "__tick" stream should be sent + to tasks. Meant to be used as a component-specific configuration.</description> + </property> + <property> + <name>topology.worker.shared.thread.pool.size</name> + <value>4</value> + <description>The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed + via the TopologyContext.</description> + </property> + <property> + <name>topology.disruptor.wait.strategy</name> + <value>com.lmax.disruptor.BlockingWaitStrategy</value> + <description>Configure the wait strategy used for internal queuing. Can be used to tradeoff latency + vs. throughput.</description> + </property> + <property> + <name>topology.executor.send.buffer.size</name> + <value>1024</value> + <description>The size of the Disruptor send queue for each executor. Must be a power of 2.</description> + </property> + <property> + <name>topology.receiver.buffer.size</name> + <value>8</value> + <description>The maximum number of messages to batch from the thread receiving off the network to the + executor queues. Must be a power of 2.</description> + </property> + <property> + <name>topology.transfer.buffer.size</name> + <value>1024</value> + <description>The size of the Disruptor transfer queue for each worker.</description> + </property> + <property> + <name>topology.tick.tuple.freq.secs</name> + <value>null</value> + <description>How often a tick tuple from the "__system" component and "__tick" stream should be sent + to tasks. Meant to be used as a component-specific configuration.</description> + </property> + <property> + <name>topology.worker.shared.thread.pool.size</name> + <value>4</value> + <description>The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed + via the TopologyContext.</description> + </property> + <property> + <name>topology.spout.wait.strategy</name> + <value>backtype.storm.spout.SleepSpoutWaitStrategy</value> + <description>A class that implements a strategy for what to do when a spout needs to wait. Waiting is + triggered in one of two conditions: + + 1. nextTuple emits no tuples + 2. The spout has hit maxSpoutPending and can't emit any more tuples</description> + </property> + <property> + <name>topology.sleep.spout.wait.strategy.time.ms</name> + <value>1</value> + <description>The amount of milliseconds the SleepEmptyEmitStrategy should sleep for.</description> + </property> + <property> + <name>topology.error.throttle.interval.secs</name> + <value>10</value> + <description>The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example, + an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be + reported to Zookeeper per task for every 10 second interval of time.</description> + </property> + <property> + <name>topology.max.error.report.per.interval</name> + <value>5</value> + <description>The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example, + an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be + reported to Zookeeper per task for every 10 second interval of time.</description> + </property> + <property> + <name>topology.kryo.factory</name> + <value>backtype.storm.serialization.DefaultKryoFactory</value> + <description>Class that specifies how to create a Kryo instance for serialization. Storm will then apply + topology.kryo.register and topology.kryo.decorators on top of this. The default implementation + implements topology.fall.back.on.java.serialization and turns references off.</description> + </property> + <property> + <name>topology.tuple.serializer</name> + <value>backtype.storm.serialization.types.ListDelegateSerializer</value> + <description>The serializer class for ListDelegate (tuple payload). + The default serializer will be ListDelegateSerializer</description> + </property> + <property> + <name>topology.trident.batch.emit.interval.millis</name> + <value>500</value> + <description>How often a batch can be emitted in a Trident topology.</description> + </property> + <property> + <name>dev.zookeeper.path</name> + <value>/tmp/dev-storm-zookeeper</value> + <description>The path to use as the zookeeper dir when running a zookeeper server via + "storm dev-zookeeper". This zookeeper instance is only intended for development; + it is not a production grade zookeeper setup.</description> + </property> + <property> + <name>metrics.reporter.register</name> + <description>Topology metrics reporter.</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property> + <name>atlas.cluster.name</name> + <value>{{cluster_name}}</value> + <depends-on> + <property> + <type>application-properties</type> + <name>atlas.cluster.name</name> + </property> + </depends-on> + </property> +</configuration>
