http://git-wip-us.apache.org/repos/asf/ambari/blob/d216c128/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_upgrade.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_upgrade.py new file mode 100644 index 0000000..bc245c4 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_upgrade.py @@ -0,0 +1,177 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. +import os + +from ambari_commons import yaml_utils +from resource_management.core.logger import Logger +from resource_management.core.exceptions import Fail +from resource_management.core.resources.system import Directory +from resource_management.core.resources.system import File +from resource_management.core.resources.system import Execute +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.format import format + +class StormUpgrade(Script): + """ + Applies to Rolling/Express Upgrade from HDP 2.1 or 2.2 to 2.3 or higher. + + Requirements: Needs to run from a host with ZooKeeper Client. + + This class helps perform some of the upgrade tasks needed for Storm during + a Rolling or Express upgrade. Storm writes data to disk locally and to ZooKeeper. + If any HDP 2.1 or 2.2 bits exist in these directories when an HDP 2.3 instance + starts up, it will fail to start properly. Because the upgrade framework in + Ambari doesn't yet have a mechanism to say "stop all" before starting to + upgrade each component, we need to rely on a Storm trick to bring down + running daemons. By removing the ZooKeeper data with running daemons, those + daemons will die. + """ + + def delete_storm_zookeeper_data(self, env): + """ + Deletes the Storm data from ZooKeeper, effectively bringing down all + Storm daemons. + :return: + """ + import params + + Logger.info('Clearing Storm data from ZooKeeper') + + storm_zookeeper_root_dir = params.storm_zookeeper_root_dir + if storm_zookeeper_root_dir is None: + raise Fail("The storm ZooKeeper directory specified by storm-site/storm.zookeeper.root must be specified") + + # The zookeeper client must be given a zookeeper host to contact. Guaranteed to have at least one host. + storm_zookeeper_server_list = yaml_utils.get_values_from_yaml_array(params.storm_zookeeper_servers) + if storm_zookeeper_server_list is None: + Logger.info("Unable to extract ZooKeeper hosts from '{0}', assuming localhost").format(params.storm_zookeeper_servers) + storm_zookeeper_server_list = ["localhost"] + + # For every zk server, try to remove /storm + zookeeper_data_cleared = False + for storm_zookeeper_server in storm_zookeeper_server_list: + # Determine where the zkCli.sh shell script is + zk_command_location = os.path.join(params.stack_root, "current", "zookeeper-client", "bin", "zkCli.sh") + if params.version is not None: + zk_command_location = os.path.join(params.stack_root, params.version, "zookeeper", "bin", "zkCli.sh") + + # create the ZooKeeper delete command + command = "{0} -server {1}:{2} rmr /storm".format( + zk_command_location, storm_zookeeper_server, params.storm_zookeeper_port) + + # clean out ZK + try: + # the ZK client requires Java to run; ensure it's on the path + env_map = { + 'JAVA_HOME': params.java64_home + } + + # AMBARI-12094: if security is enabled, then we need to tell zookeeper where the + # JAAS file is located since we don't use kinit directly with STORM + if params.security_enabled: + env_map['JVMFLAGS'] = "-Djava.security.auth.login.config={0}".format(params.storm_jaas_file) + + Execute(command, user=params.storm_user, environment=env_map, + logoutput=True, tries=1) + + zookeeper_data_cleared = True + break + except: + # the command failed, try a different ZK server + pass + + # fail if the ZK data could not be cleared + if not zookeeper_data_cleared: + raise Fail("Unable to clear ZooKeeper Storm data on any of the following ZooKeeper hosts: {0}".format( + storm_zookeeper_server_list)) + + + def delete_storm_local_data(self, env): + """ + Deletes Storm data from local directories. This will create a marker file + with JSON data representing the upgrade stack and request/stage ID. This + will prevent multiple Storm components on the same host from removing + the local directories more than once. + :return: + """ + import params + + Logger.info('Clearing Storm data from local directories...') + + storm_local_directory = params.local_dir + if storm_local_directory is None: + raise Fail("The storm local directory specified by storm-site/storm.local.dir must be specified") + + request_id = default("/requestId", None) + + stack_name = params.stack_name + stack_version = params.version + upgrade_direction = params.upgrade_direction + + json_map = {} + json_map["requestId"] = request_id + json_map["stackName"] = stack_name + json_map["stackVersion"] = stack_version + json_map["direction"] = upgrade_direction + + temp_directory = params.tmp_dir + marker_file = os.path.join(temp_directory, "storm-upgrade-{0}.json".format(stack_version)) + Logger.info("Marker file for upgrade/downgrade of Storm, {0}".format(marker_file)) + + if os.path.exists(marker_file): + Logger.info("The marker file exists.") + try: + with open(marker_file) as file_pointer: + existing_json_map = json.load(file_pointer) + + if cmp(json_map, existing_json_map) == 0: + Logger.info("The storm upgrade has already removed the local directories for {0}-{1} for " + "request {2} and direction {3}. Nothing else to do.".format(stack_name, stack_version, request_id, upgrade_direction)) + + # Nothing else to do here for this as it appears to have already been + # removed by another component being upgraded + return + else: + Logger.info("The marker file differs from the new value. Will proceed to delete Storm local dir, " + "and generate new file. Current marker file: {0}".format(str(existing_json_map))) + except Exception, e: + Logger.error("The marker file {0} appears to be corrupt; removing it. Error: {1}".format(marker_file, str(e))) + File(marker_file, action="delete") + else: + Logger.info('The marker file {0} does not exist; will attempt to delete local Storm directory if it exists.'.format(marker_file)) + + # Delete from local directory + if os.path.isdir(storm_local_directory): + Logger.info("Deleting storm local directory, {0}".format(storm_local_directory)) + Directory(storm_local_directory, action="delete", create_parents = True) + + # Recreate storm local directory + Logger.info("Recreating storm local directory, {0}".format(storm_local_directory)) + Directory(storm_local_directory, mode=0755, owner=params.storm_user, + group=params.user_group, create_parents = True) + + # The file doesn't exist, so create it + Logger.info("Saving marker file to {0} with contents: {1}".format(marker_file, str(json_map))) + with open(marker_file, 'w') as file_pointer: + json.dump(json_map, file_pointer, indent=2) + +if __name__ == "__main__": + StormUpgrade().execute() \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d216c128/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_yaml_utils.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_yaml_utils.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_yaml_utils.py new file mode 100644 index 0000000..9d78e71 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_yaml_utils.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import os +import resource_management + +from ambari_commons.yaml_utils import escape_yaml_property +from resource_management.core.source import InlineTemplate +from resource_management.core.resources.system import File + +def replace_jaas_placeholder(name, security_enabled, conf_dir): + if name.find('_JAAS_PLACEHOLDER') > -1: + if security_enabled: + return name.replace('_JAAS_PLACEHOLDER', '-Djava.security.auth.login.config=' + conf_dir + '/storm_jaas.conf') + else: + return name.replace('_JAAS_PLACEHOLDER', '') + else: + return name + +storm_yaml_template = """{% for key, value in configurations|dictsort if not key.startswith('_') %}{{key}} : {{ escape_yaml_property(replace_jaas_placeholder(resource_management.core.source.InlineTemplate(value).get_content().strip(), security_enabled, conf_dir)) }} +{% endfor %}""" + +def yaml_config_template(configurations): + return InlineTemplate(storm_yaml_template, configurations=configurations, + extra_imports=[escape_yaml_property, replace_jaas_placeholder, resource_management, + resource_management.core, resource_management.core.source]) + +def yaml_config(filename, configurations = None, conf_dir = None, owner = None, group = None): + import params + config_content = InlineTemplate('''{% for key, value in configurations_dict|dictsort %}{{ key }}: {{ escape_yaml_property(resource_management.core.source.InlineTemplate(value).get_content()) }} +{% endfor %}''', configurations_dict=configurations, extra_imports=[escape_yaml_property, resource_management, resource_management.core, resource_management.core.source]) + + File (os.path.join(params.conf_dir, filename), + content = config_content, + owner = owner, + mode = "f" + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/d216c128/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor.py new file mode 100644 index 0000000..da900f9 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management.libraries.functions import check_process_status +from resource_management.libraries.script import Script +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import format +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions import StackFeature +from storm import storm +from service import service +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyImpl +from resource_management.core.resources.service import Service + + +class Supervisor(Script): + def get_component_name(self): + return "storm-supervisor" + + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + storm("supervisor") + + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class SupervisorWindows(Supervisor): + def start(self, env): + import status_params + env.set_params(status_params) + self.configure(env) + Service(status_params.supervisor_win_service_name, action="start") + + def stop(self, env): + import status_params + env.set_params(status_params) + Service(status_params.supervisor_win_service_name, action="stop") + + def status(self, env): + import status_params + from resource_management.libraries.functions.windows_service_utils import check_windows_service_status + env.set_params(status_params) + check_windows_service_status(status_params.supervisor_win_service_name) + + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class SupervisorDefault(Supervisor): + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + conf_select.select(params.stack_name, "storm", params.version) + stack_select.select("storm-client", params.version) + stack_select.select("storm-supervisor", params.version) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) + + service("supervisor", action="start") + service("logviewer", action="start") + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + + service("supervisor", action="stop") + service("logviewer", action="stop") + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.pid_supervisor) + + def get_log_folder(self): + import params + return params.log_dir + + def get_user(self): + import params + return params.storm_user + +if __name__ == "__main__": + Supervisor().execute() + http://git-wip-us.apache.org/repos/asf/ambari/blob/d216c128/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor_prod.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor_prod.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor_prod.py new file mode 100644 index 0000000..d6c3545 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor_prod.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from storm import storm +from service import service +from supervisord_service import supervisord_service, supervisord_check_status +from resource_management.libraries.script import Script +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import format +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions import StackFeature + + +class Supervisor(Script): + + def get_component_name(self): + return "storm-supervisor" + + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + storm() + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + conf_select.select(params.stack_name, "storm", params.version) + stack_select.select("storm-client", params.version) + stack_select.select("storm-supervisor", params.version) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) + + supervisord_service("supervisor", action="start") + service("logviewer", action="start") + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + + supervisord_service("supervisor", action="stop") + service("logviewer", action="stop") + + def status(self, env): + supervisord_check_status("supervisor") + + def get_log_folder(self): + import params + return params.log_dir + + def get_user(self): + import params + return params.storm_user + +if __name__ == "__main__": + Supervisor().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/d216c128/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisord_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisord_service.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisord_service.py new file mode 100644 index 0000000..6ff9f9c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisord_service.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions.format import format + +def supervisord_service(component_name, action): + Execute(format("supervisorctl {action} storm-{component_name}"), + wait_for_finish=False + ) + +def supervisord_check_status(component_name): + try: + Execute(format("supervisorctl status storm-{component_name} | grep RUNNING")) + except Fail: + raise ComponentIsNotRunning() http://git-wip-us.apache.org/repos/asf/ambari/blob/d216c128/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/ui_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/ui_server.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/ui_server.py new file mode 100644 index 0000000..6551067 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/ui_server.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from storm import storm +from service import service +from service_check import ServiceCheck +from resource_management.libraries.functions import check_process_status +from resource_management.libraries.script import Script +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import format +from resource_management.core.resources.system import Link +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.security_commons import build_expectations, \ + cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ + FILE_TYPE_JAAS_CONF +from setup_ranger_storm import setup_ranger_storm +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyImpl +from resource_management.core.resources.service import Service + + +class UiServer(Script): + + def get_component_name(self): + return "storm-client" + + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + storm("ui") + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class UiServerWindows(UiServer): + def start(self, env): + import status_params + env.set_params(status_params) + self.configure(env) + Service(status_params.ui_win_service_name, action="start") + + def stop(self, env): + import status_params + env.set_params(status_params) + Service(status_params.ui_win_service_name, action="stop") + + def status(self, env): + import status_params + env.set_params(status_params) + from resource_management.libraries.functions.windows_service_utils import check_windows_service_status + check_windows_service_status(status_params.ui_win_service_name) + + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class UiServerDefault(UiServer): + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + conf_select.select(params.stack_name, "storm", params.version) + stack_select.select("storm-client", params.version) + + def link_metrics_sink_jar(self): + # Add storm metrics reporter JAR to storm-ui-server classpath. + # Remove symlinks. They can be there, if you doing upgrade from HDP < 2.2 to HDP >= 2.2 + Link(format("{storm_lib_dir}/ambari-metrics-storm-sink.jar"), + action="delete") + # On old HDP 2.1 versions, this symlink may also exist and break EU to newer versions + Link("/usr/lib/storm/lib/ambari-metrics-storm-sink.jar", action="delete") + + Execute(format("{sudo} ln -s {metric_collector_sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar"), + not_if=format("ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"), + only_if=format("ls {metric_collector_sink_jar}") + ) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) + self.link_metrics_sink_jar() + setup_ranger_storm(upgrade_type=upgrade_type) + service("ui", action="start") + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + service("ui", action="stop") + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.pid_ui) + + def security_status(self, env): + import status_params + + env.set_params(status_params) + + if status_params.security_enabled: + # Expect the following files to be available in status_params.config_dir: + # storm_jaas.conf + + try: + props_value_check = None + props_empty_check = ['storm_ui_principal_name', 'storm_ui_keytab'] + props_read_check = ['storm_ui_keytab'] + storm_env_expectations = build_expectations('storm_ui', props_value_check, props_empty_check, + props_read_check) + + storm_expectations = {} + storm_expectations.update(storm_env_expectations) + + security_params = {} + security_params['storm_ui'] = {} + security_params['storm_ui']['storm_ui_principal_name'] = status_params.storm_ui_principal + security_params['storm_ui']['storm_ui_keytab'] = status_params.storm_ui_keytab + + result_issues = validate_security_config_properties(security_params, storm_expectations) + if not result_issues: # If all validations passed successfully + # Double check the dict before calling execute + if ( 'storm_ui' not in security_params + or 'storm_ui_principal_name' not in security_params['storm_ui'] + or 'storm_ui_keytab' not in security_params['storm_ui']): + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.storm_user, + security_params['storm_ui']['storm_ui_keytab'], + security_params['storm_ui']['storm_ui_principal_name'], + status_params.hostname, + status_params.tmp_dir) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + self.put_structured_out({"securityState": "UNSECURED"}) + + def get_log_folder(self): + import params + return params.log_dir + + def get_user(self): + import params + return params.storm_user + +if __name__ == "__main__": + UiServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/d216c128/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/client_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/client_jaas.conf.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/client_jaas.conf.j2 new file mode 100644 index 0000000..e8dc122 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/client_jaas.conf.j2 @@ -0,0 +1,24 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +StormClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="{{nimbus_bare_jaas_principal}}"; +}; http://git-wip-us.apache.org/repos/asf/ambari/blob/d216c128/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/config.yaml.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/config.yaml.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/config.yaml.j2 new file mode 100644 index 0000000..a9760cb --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/config.yaml.j2 @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nimbusHost: {{nimbus_host}} +nimbusPort: {{nimbus_port}} + +# HTTP-specific options. +http: + + # The port on which the HTTP server listens for service requests. + port: {{rest_api_port}} + + # The port on which the HTTP server listens for administrative requests. + adminPort: {{rest_api_admin_port}} + +{% if ganglia_installed %} +enableGanglia: {{ganglia_installed}} + +# ganglia configuration (necessary if ganglia reporting is enabled) +ganglia: + + # how often to report to ganglia metrics (in seconds) + reportInterval: {{ganglia_report_interval}} + + # the hostname of the gmond server where storm cluster metrics will be sent + host: "{{ganglia_server}}" + + # address mode + # default is MULTICAST + addressMode: "UNICAST" + + # an <IP>:<HOSTNAME> pair to spoof + # this allows us to simulate storm cluster metrics coming from a specific host + #spoof: "192.168.1.1:storm" +{% endif %} + +{% if has_metric_collector and stack_supports_storm_ams %} +enableGanglia: False + +ganglia: + reportInterval: {{metric_collector_report_interval}} + +enableMetricsSink: True + +metrics_collector: + + reportInterval: {{metric_collector_report_interval}} + collector: "{{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}}" + appId: "{{metric_collector_app_id}}" + + # HTTPS settings + truststore.path : "{{metric_truststore_path}}" + truststore.type : "{{metric_truststore_type}}" + truststore.password : "{{metric_truststore_password}}" + +{% endif %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/d216c128/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm-metrics2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm-metrics2.properties.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm-metrics2.properties.j2 new file mode 100644 index 0000000..9acf173 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm-metrics2.properties.j2 @@ -0,0 +1,26 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}} +maxRowCacheSize=10000 +sendInterval={{metrics_report_interval}}000 + +# HTTPS properties +truststore.path = {{metric_truststore_path}} +truststore.type = {{metric_truststore_type}} +truststore.password = {{metric_truststore_password}} http://git-wip-us.apache.org/repos/asf/ambari/blob/d216c128/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm.conf.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm.conf.j2 new file mode 100644 index 0000000..82a26fe --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm.conf.j2 @@ -0,0 +1,35 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{storm_user}} - nofile {{storm_user_nofile_limit}} +{{storm_user}} - nproc {{storm_user_nproc_limit}} http://git-wip-us.apache.org/repos/asf/ambari/blob/d216c128/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm_jaas.conf.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm_jaas.conf.j2 new file mode 100644 index 0000000..8116492 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm_jaas.conf.j2 @@ -0,0 +1,57 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +{% if stack_supports_storm_kerberos %} +StormServer { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="{{nimbus_keytab_path}}" + storeKey=true + useTicketCache=false + principal="{{nimbus_jaas_principal}}"; +}; +StormClient { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="{{storm_keytab_path}}" + storeKey=true + useTicketCache=false + serviceName="{{nimbus_bare_jaas_principal}}" + principal="{{storm_jaas_principal}}"; +}; +{% endif %} +Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="{{storm_keytab_path}}" + storeKey=true + useTicketCache=false + serviceName="zookeeper" + principal="{{storm_jaas_principal}}"; +}; + +{% if kafka_bare_jaas_principal %} +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="{{storm_keytab_path}}" + storeKey=true + useTicketCache=false + serviceName="{{kafka_bare_jaas_principal}}" + principal="{{storm_jaas_principal}}"; +}; +{% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/d216c128/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/worker-launcher.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/worker-launcher.cfg.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/worker-launcher.cfg.j2 new file mode 100644 index 0000000..2228601 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/worker-launcher.cfg.j2 @@ -0,0 +1,19 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +storm.worker-launcher.group={{user_group}} +min.user.id={{min_user_ruid}} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/d216c128/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/quicklinks/quicklinks.json b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/quicklinks/quicklinks.json new file mode 100644 index 0000000..492f0a0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/quicklinks/quicklinks.json @@ -0,0 +1,28 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": + { + "type":"HTTP_ONLY" + }, + + "links": [ + { + "name": "storm_ui", + "label": "Storm UI", + "requires_user_name": "false", + "component_name": "STORM_UI_SERVER", + "url":"%@://%@:%@/", + "port":{ + "http_property": "ui.port", + "http_default_port": "8744", + "https_property": "ui.port", + "https_default_port": "8744", + "regex": "^(\\d+)$", + "site": "storm-site" + } + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/d216c128/ambari-server/src/main/resources/common-services/STORM/0.9.1/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/alerts.json b/ambari-server/src/main/resources/common-services/STORM/0.9.1/alerts.json deleted file mode 100644 index ae03617..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/alerts.json +++ /dev/null @@ -1,194 +0,0 @@ -{ - "STORM": { - "service": [ - { - "name": "storm_supervisor_process_percent", - "label": "Percent Supervisors Available", - "interval": 1, - "scope": "SERVICE", - "enabled": true, - "source": { - "type": "AGGREGATE", - "alert_name": "storm_supervisor_process", - "reporting": { - "ok": { - "text": "affected: [{1}], total: [{0}]" - }, - "warning": { - "text": "affected: [{1}], total: [{0}]", - "value": 10 - }, - "critical": { - "text": "affected: [{1}], total: [{0}]", - "value": 30 - }, - "units" : "%", - "type": "PERCENT" - } - } - } - ], - "STORM_UI_SERVER": [ - { - "name": "storm_server_process", - "label": "Storm Server Process", - "interval": 1, - "scope": "ANY", - "enabled": true, - "source": { - "type": "PORT", - "uri": "{{storm-site/ui.port}}", - "default_port": 8744, - "reporting": { - "ok": { - "text": "TCP OK - {0:.3f}s response on port {1}" - }, - "warning": { - "text": "TCP OK - {0:.3f}s response on port {1}", - "value": 1.5 - }, - "critical": { - "text": "Connection failed: {0} to {1}:{2}", - "value": 5.0 - } - } - } - }, - { - "name": "storm_webui", - "label": "Storm Web UI", - "interval": 1, - "scope": "ANY", - "enabled": true, - "source": { - "type": "WEB", - "uri": { - "http": "{{storm-site/ui.port}}", - "kerberos_keytab": "{{storm-env/storm_ui_keytab}}", - "kerberos_principal": "{{storm-env/storm_ui_principal_name}}", - "connection_timeout": 5.0 - }, - "reporting": { - "ok": { - "text": "HTTP {0} response in {2:.3f}s" - }, - "warning":{ - "text": "HTTP {0} response from {1} in {2:.3f}s ({3})" - }, - "critical": { - "text": "Connection failed to {1} ({3})" - } - } - } - } - ], - "NIMBUS": [ - { - "name": "storm_nimbus_process", - "label": "Nimbus Process", - "interval": 1, - "scope": "ANY", - "enabled": true, - "source": { - "type": "PORT", - "uri": "{{storm-site/nimbus.thrift.port}}", - "default_port": 6627, - "reporting": { - "ok": { - "text": "TCP OK - {0:.3f}s response on port {1}" - }, - "warning": { - "text": "TCP OK - {0:.3f}s response on port {1}", - "value": 1.5 - }, - "critical": { - "text": "Connection failed: {0} to {1}:{2}", - "value": 5.0 - } - } - } - } - ], - "DRPC_SERVER": [ - { - "name": "storm_drpc_server", - "label": "DRPC Server Process", - "interval": 1, - "scope": "ANY", - "enabled": true, - "source": { - "type": "PORT", - "uri": "{{storm-site/drpc.port}}", - "default_port": 3772, - "reporting": { - "ok": { - "text": "TCP OK - {0:.3f}s response on port {1}" - }, - "warning": { - "text": "TCP OK - {0:.3f}s response on port {1}", - "value": 1.5 - }, - "critical": { - "text": "Connection failed: {0} to {1}:{2}", - "value": 5.0 - } - } - } - } - ], - "STORM_REST_API": [ - { - "name": "storm_rest_api", - "label": "Storm REST API", - "interval": 1, - "scope": "ANY", - "enabled": true, - "source": { - "type": "PORT", - "uri": "8745", - "default_port": 8745, - "reporting": { - "ok": { - "text": "TCP OK - {0:.3f}s response on port {1}" - }, - "warning": { - "text": "TCP OK - {0:.3f}s response on port {1}", - "value": 1.5 - }, - "critical": { - "text": "Connection failed: {0} to {1}:{2}", - "value": 5.0 - } - } - } - } - ], - "SUPERVISOR": [ - { - "name": "storm_supervisor_process", - "label": "Supervisor Process", - "interval": 1, - "scope": "HOST", - "enabled": true, - "source": { - "type": "PORT", - "uri": "{{storm-env/jmxremote_port}}", - "default_port": 56431, - "reporting": { - "ok": { - "text": "TCP OK - {0:.3f}s response on port {1}" - }, - "warning": { - "text": "TCP OK - {0:.3f}s response on port {1}", - "value": 1.5 - }, - "critical": { - "text": "Connection failed: {0} to {1}:{2}", - "value": 5.0 - } - } - } - } - ] - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/d216c128/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-env.xml b/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-env.xml deleted file mode 100644 index 75b080a..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-env.xml +++ /dev/null @@ -1,139 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> - -<configuration supports_adding_forbidden="true"> - <property> - <name>storm_user</name> - <display-name>Storm User</display-name> - <value>storm</value> - <property-type>USER</property-type> - <description></description> - <value-attributes> - <type>user</type> - <overridable>false</overridable> - </value-attributes> - </property> - <property> - <name>storm_log_dir</name> - <value>/var/log/storm</value> - <description></description> - <value-attributes> - <type>directory</type> - </value-attributes> - </property> - <property> - <name>storm_pid_dir</name> - <value>/var/run/storm</value> - <description></description> - <value-attributes> - <type>directory</type> - </value-attributes> - </property> - <property> - <name>jmxremote_port</name> - <value>56431</value> - <description></description> - </property> - - - <property> - <name>storm_principal_name</name> - <description>Storm principal name</description> - </property> - - <property> - <name>storm_principal_name</name> - <description>Storm principal name</description> - </property> - - <property> - <name>storm_keytab</name> - <description>Storm keytab path</description> - </property> - - <property> - <name>storm_ui_principal_name</name> - <description>Storm UI principal name</description> - </property> - - <property> - <name>storm_ui_keytab</name> - <description>Storm UI keytab path</description> - </property> - - <property> - <name>nimbus_keytab</name> - <description>Nimbus keytab path</description> - </property> - - <property> - <name>nimbus_principal_name</name> - <description>Nimbus principal name</description> - </property> - - <property> - <name>storm_user_nofile_limit</name> - <value>128000</value> - <description>Max open files limit setting for STORM user.</description> - </property> - <property> - <name>storm_user_nproc_limit</name> - <value>65536</value> - <description>Max number of processes limit setting for STORM user.</description> - </property> - - <!-- storm-env.sh --> - <property> - <name>content</name> - <description>This is the jinja template for storm-env.sh file</description> - <value> -#!/bin/bash - -# Set Storm specific environment variables here. - -# The java implementation to use. -export JAVA_HOME={{java64_home}} - -# Storm log folder -export STORM_LOG_DIR={{log_dir}} - -export STORM_CONF_DIR={{conf_dir}} -export STORM_HOME={{storm_component_home_dir}} - </value> - <value-attributes> - <type>content</type> - </value-attributes> - </property> - - <property> - <name>nimbus_seeds_supported</name> - <value>false</value> - <description></description> - </property> - <property> - <name>storm_logs_supported</name> - <value>false</value> - <description></description> - </property> - - -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/d216c128/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-site.xml b/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-site.xml deleted file mode 100644 index 01b3f4e..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-site.xml +++ /dev/null @@ -1,740 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> - -<configuration supports_final="true"> - <property> - <name>java.library.path</name> - <value>/usr/local/lib:/opt/local/lib:/usr/lib</value> - <description>This value is passed to spawned JVMs (e.g., Nimbus, Supervisor, and Workers) - for the java.library.path value. java.library.path tells the JVM where - to look for native libraries. It is necessary to set this config correctly since - Storm uses the ZeroMQ and JZMQ native libs. </description> - </property> - <property> - <name>storm.local.dir</name> - <value>/hadoop/storm</value> - <description>A directory on the local filesystem used by Storm for any local - filesystem usage it needs. The directory must exist and the Storm daemons must - have permission to read/write from this location.</description> - <value-attributes> - <type>directory</type> - </value-attributes> - </property> - <property> - <name>storm.zookeeper.servers</name> - <value>['localhost']</value> - <property-type>DONT_ADD_ON_UPGRADE</property-type> - <description>A list of hosts of ZooKeeper servers used to manage the cluster.</description> - <value-attributes> - <type>multiLine</type> - <overridable>false</overridable> - </value-attributes> - </property> - <property> - <name>storm.zookeeper.port</name> - <value>2181</value> - <description>The port Storm will use to connect to each of the ZooKeeper servers.</description> - <value-attributes> - <type>int</type> - </value-attributes> - </property> - <property> - <name>storm.zookeeper.root</name> - <value>/storm</value> - <description>The root location at which Storm stores data in ZooKeeper.</description> - <value-attributes> - <type>directory</type> - </value-attributes> - </property> - <property> - <name>storm.zookeeper.session.timeout</name> - <value>20000</value> - <description>The session timeout for clients to ZooKeeper.</description> - <value-attributes> - <type>int</type> - </value-attributes> - </property> - <property> - <name>storm.zookeeper.connection.timeout</name> - <value>15000</value> - <description>The connection timeout for clients to ZooKeeper.</description> - <value-attributes> - <type>int</type> - </value-attributes> - </property> - <property> - <name>storm.zookeeper.retry.times</name> - <value>5</value> - <description>The number of times to retry a Zookeeper operation.</description> - <value-attributes> - <type>int</type> - </value-attributes> - </property> - <property> - <name>storm.zookeeper.retry.interval</name> - <value>1000</value> - <description>The interval between retries of a Zookeeper operation.</description> - <value-attributes> - <unit>ms</unit> - <type>int</type> - </value-attributes> - </property> - <property> - <name>storm.zookeeper.retry.intervalceiling.millis</name> - <value>30000</value> - <description>The ceiling of the interval between retries of a Zookeeper operation.</description> - <value-attributes> - <type>int</type> - <unit>ms</unit> - </value-attributes> - </property> - <property> - <name>storm.cluster.mode</name> - <value>distributed</value> - <description>The mode this Storm cluster is running in. Either "distributed" or "local".</description> - </property> - <property> - <name>storm.local.mode.zmq</name> - <value>false</value> - <description>Whether or not to use ZeroMQ for messaging in local mode. If this is set - to false, then Storm will use a pure-Java messaging system. The purpose - of this flag is to make it easy to run Storm in local mode by eliminating - the need for native dependencies, which can be difficult to install. - </description> - <value-attributes> - <type>boolean</type> - </value-attributes> - </property> - <property> - <name>storm.thrift.transport</name> - <value>backtype.storm.security.auth.SimpleTransportPlugin</value> - <description>The transport plug-in for Thrift client/server communication.</description> - </property> - <property> - <name>storm.messaging.transport</name> - <value>backtype.storm.messaging.netty.Context</value> - <description>The transporter for communication among Storm tasks.</description> - </property> - <property> - <name>nimbus.host</name> - <value>localhost</value> - <description>The host that the master server is running on.</description> - <value-attributes> - <type>componentHost</type> - <editable-only-at-install>true</editable-only-at-install> - <overridable>false</overridable> - </value-attributes> - </property> - <property> - <name>nimbus.thrift.port</name> - <value>6627</value> - <description> Which port the Thrift interface of Nimbus should run on. Clients should - connect to this port to upload jars and submit topologies.</description> - <value-attributes> - <type>int</type> - </value-attributes> - </property> - <property> - <name>nimbus.thrift.max_buffer_size</name> - <value>1048576</value> - <description>The maximum buffer size thrift should use when reading messages.</description> - <value-attributes> - <type>int</type> - <unit>bytes</unit> - </value-attributes> - </property> - <property> - <name>nimbus.childopts</name> - <value>-Xmx1024m -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8649,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM</value> - <description>This parameter is used by the storm-deploy project to configure the jvm options for the nimbus daemon.</description> - <value-attributes> - <type>multiLine</type> - <overridable>false</overridable> - </value-attributes> - </property> - <property> - <name>nimbus.task.timeout.secs</name> - <value>30</value> - <description>How long without heartbeating a task can go before nimbus will consider the task dead and reassign it to another location.</description> - <value-attributes> - <type>int</type> - <unit>seconds</unit> - </value-attributes> - </property> - <property> - <name>nimbus.supervisor.timeout.secs</name> - <value>60</value> - <description>How long before a supervisor can go without heartbeating before nimbus considers it dead and stops assigning new work to it.</description> - <value-attributes> - <type>int</type> - </value-attributes> - </property> - <property> - <name>nimbus.monitor.freq.secs</name> - <value>120</value> - <description> - How often nimbus should wake up to check heartbeats and do reassignments. Note - that if a machine ever goes down Nimbus will immediately wake up and take action. - This parameter is for checking for failures when there's no explicit event like that occuring. - </description> - <value-attributes> - <type>int</type> - <unit>seconds</unit> - </value-attributes> - </property> - <property> - <name>nimbus.cleanup.inbox.freq.secs</name> - <value>600</value> - <description>How often nimbus should wake the cleanup thread to clean the inbox.</description> - <value-attributes> - <type>int</type> - <unit>seconds</unit> - </value-attributes> - </property> - <property> - <name>nimbus.inbox.jar.expiration.secs</name> - <value>3600</value> - <description> - The length of time a jar file lives in the inbox before being deleted by the cleanup thread. - - Probably keep this value greater than or equal to NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS. - Note that the time it takes to delete an inbox jar file is going to be somewhat more than - NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS (depending on how often NIMBUS_CLEANUP_FREQ_SECS is set to). - </description> - <value-attributes> - <type>int</type> - <unit>seconds</unit> - </value-attributes> - </property> - <property> - <name>nimbus.task.launch.secs</name> - <value>120</value> - <description>A special timeout used when a task is initially launched. During launch, this is the timeout - used until the first heartbeat, overriding nimbus.task.timeout.secs.</description> - <value-attributes> - <type>int</type> - <unit>seconds</unit> - </value-attributes> - </property> - <property> - <name>nimbus.reassign</name> - <value>true</value> - <description>Whether or not nimbus should reassign tasks if it detects that a task goes down. - Defaults to true, and it's not recommended to change this value.</description> - <value-attributes> - <type>boolean</type> - </value-attributes> - </property> - <property> - <name>nimbus.file.copy.expiration.secs</name> - <value>600</value> - <description>During upload/download with the master, how long an upload or download connection is idle - before nimbus considers it dead and drops the connection.</description> - <value-attributes> - <type>int</type> - <unit>seconds</unit> - </value-attributes> - </property> - <property> - <name>nimbus.topology.validator</name> - <value>backtype.storm.nimbus.DefaultTopologyValidator</value> - <description>A custom class that implements ITopologyValidator that is run whenever a - topology is submitted. Can be used to provide business-specific logic for - whether topologies are allowed to run or not.</description> - </property> - <property> - <name>ui.port</name> - <value>8744</value> - <description>Storm UI binds to this port.</description> - <value-attributes> - <type>int</type> - </value-attributes> - </property> - <property> - <name>ui.childopts</name> - <value>-Xmx768m</value> - <description>Childopts for Storm UI Java process.</description> - </property> - <property> - <name>logviewer.port</name> - <value>8000</value> - <description>HTTP UI port for log viewer.</description> - </property> - <property> - <name>logviewer.childopts</name> - <value>-Xmx128m</value> - <description>Childopts for log viewer java process.</description> - </property> - <property> - <name>logviewer.appender.name</name> - <value>A1</value> - <description>Appender name used by log viewer to determine log directory.</description> - </property> - <property> - <name>drpc.port</name> - <value>3772</value> - <description>This port is used by Storm DRPC for receiving DPRC requests from clients.</description> - <value-attributes> - <type>int</type> - </value-attributes> - </property> - <property> - <name>drpc.worker.threads</name> - <value>64</value> - <description>DRPC thrift server worker threads.</description> - <value-attributes> - <type>int</type> - </value-attributes> - </property> - <property> - <name>drpc.queue.size</name> - <value>128</value> - <description>DRPC thrift server queue size.</description> - <value-attributes> - <type>int</type> - </value-attributes> - </property> - <property> - <name>drpc.invocations.port</name> - <value>3773</value> - <description>This port on Storm DRPC is used by DRPC topologies to receive function invocations and send results back.</description> - <value-attributes> - <type>int</type> - </value-attributes> - </property> - <property> - <name>drpc.request.timeout.secs</name> - <value>600</value> - <description>The timeout on DRPC requests within the DRPC server. Defaults to 10 minutes. Note that requests can also - timeout based on the socket timeout on the DRPC client, and separately based on the topology message - timeout for the topology implementing the DRPC function.</description> - <value-attributes> - <type>int</type> - <unit>seconds</unit> - </value-attributes> - </property> - <property> - <name>drpc.childopts</name> - <value>-Xmx768m</value> - <description>Childopts for Storm DRPC Java process.</description> - </property> - <property> - <name>transactional.zookeeper.root</name> - <value>/transactional</value> - <description>The root directory in ZooKeeper for metadata about TransactionalSpouts.</description> - </property> - <property> - <name>transactional.zookeeper.servers</name> - <value>null</value> - <description>The list of zookeeper servers in which to keep the transactional state. If null (which is default), - will use storm.zookeeper.servers</description> - </property> - <property> - <name>transactional.zookeeper.port</name> - <value>null</value> - <description>The port to use to connect to the transactional zookeeper servers. If null (which is default), - will use storm.zookeeper.port</description> - </property> - <property> - <name>supervisor.slots.ports</name> - <value>[6700, 6701]</value> - <description>A list of ports that can run workers on this supervisor. Each worker uses one port, and - the supervisor will only run one worker per port. Use this configuration to tune - how many workers run on each machine.</description> - </property> - <property> - <name>supervisor.childopts</name> - <value>-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port={{jmxremote_port}} -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM</value> - <description>This parameter is used by the storm-deploy project to configure the jvm options for the supervisor daemon.</description> - <value-attributes> - <type>multiLine</type> - <overridable>false</overridable> - </value-attributes> - </property> - <property> - <name>supervisor.worker.start.timeout.secs</name> - <value>120</value> - <description>How long a worker can go without heartbeating during the initial launch before - the supervisor tries to restart the worker process. This value override - supervisor.worker.timeout.secs during launch because there is additional - overhead to starting and configuring the JVM on launch.</description> - <value-attributes> - <type>int</type> - <unit>seconds</unit> - </value-attributes> - </property> - <property> - <name>supervisor.worker.timeout.secs</name> - <value>30</value> - <description>How long a worker can go without heartbeating before the supervisor tries to restart the worker process.</description> - <value-attributes> - <type>int</type> - <unit>seconds</unit> - </value-attributes> - </property> - <property> - <name>supervisor.monitor.frequency.secs</name> - <value>3</value> - <description>How often the supervisor checks the worker heartbeats to see if any of them need to be restarted.</description> - <value-attributes> - <type>int</type> - <unit>seconds</unit> - </value-attributes> - </property> - <property> - <name>supervisor.heartbeat.frequency.secs</name> - <value>5</value> - <description>How often the supervisor sends a heartbeat to the master.</description> - <value-attributes> - <type>int</type> - <unit>seconds</unit> - </value-attributes> - </property> - <property> - <name>worker.childopts</name> - <value>-Xmx768m -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM</value> - <description>The jvm opts provided to workers launched by this supervisor. All \"%ID%\" substrings are replaced with an identifier for this worker.</description> - <value-attributes> - <type>multiLine</type> - </value-attributes> - </property> - <property> - <name>worker.heartbeat.frequency.secs</name> - <value>1</value> - <description>How often this worker should heartbeat to the supervisor.</description> - </property> - <property> - <name>task.heartbeat.frequency.secs</name> - <value>3</value> - <description>How often a task should heartbeat its status to the master.</description> - </property> - <property> - <name>task.refresh.poll.secs</name> - <value>10</value> - <description>How often a task should sync its connections with other tasks (if a task is - reassigned, the other tasks sending messages to it need to refresh their connections). - In general though, when a reassignment happens other tasks will be notified - almost immediately. This configuration is here just in case that notification doesn't - come through.</description> - </property> - <property> - <name>zmq.threads</name> - <value>1</value> - <description>The number of threads that should be used by the zeromq context in each worker process.</description> - </property> - <property> - <name>zmq.linger.millis</name> - <value>5000</value> - <description>How long a connection should retry sending messages to a target host when - the connection is closed. This is an advanced configuration and can almost - certainly be ignored.</description> - </property> - <property> - <name>zmq.hwm</name> - <value>0</value> - <description>The high water for the ZeroMQ push sockets used for networking. Use this config to prevent buffer explosion - on the networking layer.</description> - </property> - <property> - <name>storm.messaging.netty.server_worker_threads</name> - <value>1</value> - <description>Netty based messaging: The # of worker threads for the server.</description> - <value-attributes> - <type>int</type> - </value-attributes> - </property> - <property> - <name>storm.messaging.netty.client_worker_threads</name> - <value>1</value> - <description>Netty based messaging: The # of worker threads for the client.</description> - <value-attributes> - <type>int</type> - </value-attributes> - </property> - <property> - <name>storm.messaging.netty.buffer_size</name> - <value>5242880</value> - <description>Netty based messaging: The buffer size for send/recv buffer.</description> - <value-attributes> - <type>int</type> - <unit>bytes</unit> - </value-attributes> - </property> - <property> - <name>storm.messaging.netty.max_retries</name> - <value>30</value> - <description>Netty based messaging: The max # of retries that a peer will perform when a remote is not accessible.</description> - <value-attributes> - <type>int</type> - </value-attributes> - </property> - <property> - <name>storm.messaging.netty.max_wait_ms</name> - <value>1000</value> - <description>Netty based messaging: The max # of milliseconds that a peer will wait.</description> - <value-attributes> - <type>int</type> - <unit>ms</unit> - </value-attributes> - </property> - <property> - <name>storm.messaging.netty.min_wait_ms</name> - <value>100</value> - <description>Netty based messaging: The min # of milliseconds that a peer will wait.</description> - <value-attributes> - <type>int</type> - <unit>ms</unit> - </value-attributes> - </property> - <property> - <name>topology.enable.message.timeouts</name> - <value>true</value> - <description>True if Storm should timeout messages or not. Defaults to true. This is meant to be used - in unit tests to prevent tuples from being accidentally timed out during the test.</description> - </property> - <property> - <name>topology.debug</name> - <value>false</value> - <description>When set to true, Storm will log every message that's emitted.</description> - </property> - <property> - <name>topology.optimize</name> - <value>true</value> - <description>Whether or not the master should optimize topologies by running multiple tasks in a single thread where appropriate.</description> - </property> - <property> - <name>topology.workers</name> - <value>1</value> - <description>How many processes should be spawned around the cluster to execute this - topology. Each process will execute some number of tasks as threads within - them. This parameter should be used in conjunction with the parallelism hints - on each component in the topology to tune the performance of a topology.</description> - </property> - <property> - <name>topology.acker.executors</name> - <value>null</value> - <description>How many executors to spawn for ackers. - - If this is set to 0, then Storm will immediately ack tuples as soon - as they come off the spout, effectively disabling reliability. - </description> - </property> - <property> - <name>topology.message.timeout.secs</name> - <value>30</value> - <description>The maximum amount of time given to the topology to fully process a message - emitted by a spout. If the message is not acked within this time frame, Storm - will fail the message on the spout. Some spouts implementations will then replay - the message at a later time.</description> - </property> - <property> - <name>topology.skip.missing.kryo.registrations</name> - <value>false</value> - <description> Whether or not Storm should skip the loading of kryo registrations for which it - does not know the class or have the serializer implementation. Otherwise, the task will - fail to load and will throw an error at runtime. The use case of this is if you want to - declare your serializations on the storm.yaml files on the cluster rather than every single - time you submit a topology. Different applications may use different serializations and so - a single application may not have the code for the other serializers used by other apps. - By setting this config to true, Storm will ignore that it doesn't have those other serializations - rather than throw an error.</description> - </property> - <property> - <name>topology.max.task.parallelism</name> - <value>null</value> - <description>The maximum parallelism allowed for a component in this topology. This configuration is - typically used in testing to limit the number of threads spawned in local mode.</description> - </property> - <property> - <name>topology.max.spout.pending</name> - <value>1000</value> - <description>The maximum number of tuples that can be pending on a spout task at any given time. - This config applies to individual tasks, not to spouts or topologies as a whole. - - A pending tuple is one that has been emitted from a spout but has not been acked or failed yet. - Note that this config parameter has no effect for unreliable spouts that don't tag - their tuples with a message id.</description> - </property> - <property> - <name>topology.state.synchronization.timeout.secs</name> - <value>60</value> - <description>The maximum amount of time a component gives a source of state to synchronize before it requests - synchronization again.</description> - </property> - <property> - <name>topology.stats.sample.rate</name> - <value>0.05</value> - <description>The percentage of tuples to sample to produce stats for a task.</description> - </property> - <property> - <name>topology.builtin.metrics.bucket.size.secs</name> - <value>60</value> - <description>The time period that builtin metrics data in bucketed into.</description> - </property> - <property> - <name>topology.fall.back.on.java.serialization</name> - <value>true</value> - <description>Whether or not to use Java serialization in a topology.</description> - </property> - <property> - <name>topology.worker.childopts</name> - <value>null</value> - <description>Topology-specific options for the worker child process. This is used in addition to WORKER_CHILDOPTS.</description> - </property> - <property> - <name>topology.executor.receive.buffer.size</name> - <value>1024</value> - <description>The size of the Disruptor receive queue for each executor. Must be a power of 2.</description> - </property> - <property> - <name>topology.executor.send.buffer.size</name> - <value>1024</value> - <description>The size of the Disruptor send queue for each executor. Must be a power of 2.</description> - </property> - <property> - <name>topology.receiver.buffer.size</name> - <value>8</value> - <description>The maximum number of messages to batch from the thread receiving off the network to the - executor queues. Must be a power of 2.</description> - </property> - <property> - <name>topology.transfer.buffer.size</name> - <value>1024</value> - <description>The size of the Disruptor transfer queue for each worker.</description> - </property> - <property> - <name>topology.tick.tuple.freq.secs</name> - <value>null</value> - <description>How often a tick tuple from the "__system" component and "__tick" stream should be sent - to tasks. Meant to be used as a component-specific configuration.</description> - </property> - <property> - <name>topology.worker.shared.thread.pool.size</name> - <value>4</value> - <description>The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed - via the TopologyContext.</description> - </property> - <property> - <name>topology.disruptor.wait.strategy</name> - <value>com.lmax.disruptor.BlockingWaitStrategy</value> - <description>Configure the wait strategy used for internal queuing. Can be used to tradeoff latency - vs. throughput.</description> - </property> - <property> - <name>topology.executor.send.buffer.size</name> - <value>1024</value> - <description>The size of the Disruptor send queue for each executor. Must be a power of 2.</description> - </property> - <property> - <name>topology.receiver.buffer.size</name> - <value>8</value> - <description>The maximum number of messages to batch from the thread receiving off the network to the - executor queues. Must be a power of 2.</description> - </property> - <property> - <name>topology.transfer.buffer.size</name> - <value>1024</value> - <description>The size of the Disruptor transfer queue for each worker.</description> - </property> - <property> - <name>topology.tick.tuple.freq.secs</name> - <value>null</value> - <description>How often a tick tuple from the "__system" component and "__tick" stream should be sent - to tasks. Meant to be used as a component-specific configuration.</description> - </property> - <property> - <name>topology.worker.shared.thread.pool.size</name> - <value>4</value> - <description>The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed - via the TopologyContext.</description> - </property> - <property> - <name>topology.spout.wait.strategy</name> - <value>backtype.storm.spout.SleepSpoutWaitStrategy</value> - <description>A class that implements a strategy for what to do when a spout needs to wait. Waiting is - triggered in one of two conditions: - - 1. nextTuple emits no tuples - 2. The spout has hit maxSpoutPending and can't emit any more tuples</description> - </property> - <property> - <name>topology.sleep.spout.wait.strategy.time.ms</name> - <value>1</value> - <description>The amount of milliseconds the SleepEmptyEmitStrategy should sleep for.</description> - </property> - <property> - <name>topology.error.throttle.interval.secs</name> - <value>10</value> - <description>The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example, - an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be - reported to Zookeeper per task for every 10 second interval of time.</description> - </property> - <property> - <name>topology.max.error.report.per.interval</name> - <value>5</value> - <description>The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example, - an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be - reported to Zookeeper per task for every 10 second interval of time.</description> - </property> - <property> - <name>topology.kryo.factory</name> - <value>backtype.storm.serialization.DefaultKryoFactory</value> - <description>Class that specifies how to create a Kryo instance for serialization. Storm will then apply - topology.kryo.register and topology.kryo.decorators on top of this. The default implementation - implements topology.fall.back.on.java.serialization and turns references off.</description> - </property> - <property> - <name>topology.tuple.serializer</name> - <value>backtype.storm.serialization.types.ListDelegateSerializer</value> - <description>The serializer class for ListDelegate (tuple payload). - The default serializer will be ListDelegateSerializer</description> - </property> - <property> - <name>topology.trident.batch.emit.interval.millis</name> - <value>500</value> - <description>How often a batch can be emitted in a Trident topology.</description> - </property> - <property> - <name>dev.zookeeper.path</name> - <value>/tmp/dev-storm-zookeeper</value> - <description>The path to use as the zookeeper dir when running a zookeeper server via - "storm dev-zookeeper". This zookeeper instance is only intended for development; - it is not a production grade zookeeper setup.</description> - </property> - <property> - <name>metrics.reporter.register</name> - <description>Topology metrics reporter.</description> - <value-attributes> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - </property> - <property> - <name>atlas.cluster.name</name> - <value>{{cluster_name}}</value> - <depends-on> - <property> - <type>application-properties</type> - <name>atlas.cluster.name</name> - </property> - </depends-on> - </property> -</configuration>
