Repository: ambari Updated Branches: refs/heads/trunk b0fa86ce8 -> 661d143b9
AMBARI-11009. Storm service check fails with multiple nimbus hosts, remove messages from Storm RU now that it is rolling, allow nimbus.seeds to be edited (alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/661d143b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/661d143b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/661d143b Branch: refs/heads/trunk Commit: 661d143b9010711a55bb3f7f903aa148425733e2 Parents: b0fa86c Author: Alejandro Fernandez <[email protected]> Authored: Thu May 7 16:03:43 2015 -0700 Committer: Alejandro Fernandez <[email protected]> Committed: Mon May 11 11:09:50 2015 -0700 ---------------------------------------------------------------------- .../0.9.1.2.1/package/scripts/params_linux.py | 8 +- .../0.9.1.2.1/package/scripts/service_check.py | 8 +- .../0.9.1.2.1/package/scripts/yaml_utils.py | 10 +- .../STORM/package/scripts/yaml_config.py | 6 +- .../services/STORM/configuration/storm-site.xml | 2 +- .../stacks/HDP/2.3/upgrades/upgrade-2.3.xml | 10 - .../stacks/2.3/STORM/test_service_check.py | 50 ++++ .../python/stacks/2.3/STORM/test_storm_base.py | 124 +++++++++ .../stacks/2.3/configs/storm_default.json | 260 +++++++++++++++++++ ambari-web/app/data/HDP2/site_properties.js | 1 - 10 files changed, 456 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/661d143b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py index 6c96d73..729e383 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py @@ -81,8 +81,12 @@ user_group = config['configurations']['cluster-env']['user_group'] java64_home = config['hostLevelParams']['java_home'] jps_binary = format("{java64_home}/bin/jps") nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port'] -nimbus_seeds_supported = config['configurations']['storm-env']['nimbus_seeds_supported'] -nimbus_host = config['configurations']['storm-site']['nimbus.seeds'] if nimbus_seeds_supported else config['configurations']['storm-site']['nimbus.host'] + +# nimbus.seeds is supported in HDP 2.3.0.0 and higher +nimbus_seeds_supported = default('/configurations/storm-env/nimbus_seeds_supported', False) +nimbus_host = default('/configurations/storm-site/nimbus.host', None) +nimbus_seeds = default('/configurations/storm-site/nimbus.seeds', None) + rest_api_port = "8745" rest_api_admin_port = "8746" rest_api_conf_file = format("{conf_dir}/config.yaml") http://git-wip-us.apache.org/repos/asf/ambari/blob/661d143b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/service_check.py index 483c144..4484501 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/service_check.py @@ -53,7 +53,13 @@ class ServiceCheckDefault(ServiceCheck): content=StaticFile("wordCount.jar") ) - cmd = format("storm jar /tmp/wordCount.jar storm.starter.WordCountTopology WordCount{unique} -c nimbus.host={nimbus_host}") + cmd = "" + if params.nimbus_seeds_supported: + # Because this command is guaranteed to run on one of the hosts with storm client, there is no need + # to specify "-c nimbus.seeds={nimbus_seeds}" + cmd = format("storm jar /tmp/wordCount.jar storm.starter.WordCountTopology WordCount{unique}") + elif params.nimbus_host is not None: + cmd = format("storm jar /tmp/wordCount.jar storm.starter.WordCountTopology WordCount{unique} -c nimbus.host={nimbus_host}") Execute(cmd, logoutput=True, http://git-wip-us.apache.org/repos/asf/ambari/blob/661d143b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/yaml_utils.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/yaml_utils.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/yaml_utils.py index aeda3b3..6e99e8b 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/yaml_utils.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/yaml_utils.py @@ -24,7 +24,7 @@ import resource_management from resource_management.core.source import InlineTemplate from resource_management.core.resources.system import File -def escape_yaml_propetry(value): +def escape_yaml_property(value): unquouted = False unquouted_values = ["null","Null","NULL","true","True","TRUE","false","False","FALSE","YES","Yes","yes","NO","No","no","ON","On","on","OFF","Off","off"] if value in unquouted_values: @@ -61,18 +61,18 @@ def replace_jaas_placeholder(name, security_enabled, conf_dir): else: return name -storm_yaml_template = """{% for key, value in configurations|dictsort if not key.startswith('_') %}{{key}} : {{ escape_yaml_propetry(replace_jaas_placeholder(resource_management.core.source.InlineTemplate(value).get_content().strip(), security_enabled, conf_dir)) }} +storm_yaml_template = """{% for key, value in configurations|dictsort if not key.startswith('_') %}{{key}} : {{ escape_yaml_property(replace_jaas_placeholder(resource_management.core.source.InlineTemplate(value).get_content().strip(), security_enabled, conf_dir)) }} {% endfor %}""" def yaml_config_template(configurations): return InlineTemplate(storm_yaml_template, configurations=configurations, - extra_imports=[escape_yaml_propetry, replace_jaas_placeholder, resource_management, + extra_imports=[escape_yaml_property, replace_jaas_placeholder, resource_management, resource_management.core, resource_management.core.source]) def yaml_config(filename, configurations = None, conf_dir = None, owner = None, group = None): import params - config_content = InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}: {{ escape_yaml_propetry(value) }} -{% endfor %}''', configurations_dict=configurations, extra_imports=[escape_yaml_propetry]) + config_content = InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}: {{ escape_yaml_property(value) }} +{% endfor %}''', configurations_dict=configurations, extra_imports=[escape_yaml_property]) File (os.path.join(params.conf_dir, filename), content = config_content, http://git-wip-us.apache.org/repos/asf/ambari/blob/661d143b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/STORM/package/scripts/yaml_config.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/STORM/package/scripts/yaml_config.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/STORM/package/scripts/yaml_config.py index 39261be..62c954d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/STORM/package/scripts/yaml_config.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/STORM/package/scripts/yaml_config.py @@ -21,7 +21,7 @@ limitations under the License. import re from resource_management import * -def escape_yaml_propetry(value): +def escape_yaml_property(value): unquouted = False unquouted_values = ["null","Null","NULL","true","True","TRUE","false","False","FALSE","YES","Yes","yes","NO","No","no","ON","On","on","OFF","Off","off"] @@ -58,8 +58,8 @@ def yaml_config( owner = None, group = None ): - config_content = source.InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}: {{ escape_yaml_propetry(value) }} -{% endfor %}''', configurations_dict=configurations, extra_imports=[escape_yaml_propetry]) + config_content = source.InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}: {{ escape_yaml_property(value) }} +{% endfor %}''', configurations_dict=configurations, extra_imports=[escape_yaml_property]) File (format("{conf_dir}/{filename}"), content = config_content, http://git-wip-us.apache.org/repos/asf/ambari/blob/661d143b/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-site.xml index 3ecc014..373f408 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-site.xml @@ -22,7 +22,7 @@ <property> <name>nimbus.seeds</name> <value>localhost</value> - <description>The host that the nimbus server is running on.</description> + <description>Comma-delimited list of the hosts running nimbus server.</description> </property> <property> <name>topology.min.replication.count</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/661d143b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml index 65549dd..1937890 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml @@ -569,11 +569,6 @@ <service name="STORM"> <component name="NIMBUS"> - <pre-upgrade> - <task xsi:type="manual"> - <message>Before continuing, please deactivate and kill any currently running topologies.</message> - </task> - </pre-upgrade> <upgrade> <task xsi:type="restart" /> </upgrade> @@ -597,11 +592,6 @@ <upgrade> <task xsi:type="restart" /> </upgrade> - <post-upgrade> - <task xsi:type="manual"> - <message>Please rebuild your topology using the new Storm version dependencies and resubmit it using the newly created jar.</message> - </task> - </post-upgrade> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/661d143b/ambari-server/src/test/python/stacks/2.3/STORM/test_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/STORM/test_service_check.py b/ambari-server/src/test/python/stacks/2.3/STORM/test_service_check.py new file mode 100644 index 0000000..1aa3cee --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.3/STORM/test_service_check.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +from mock.mock import MagicMock, patch +from stacks.utils.RMFTestCase import * +import resource_management.libraries.functions +from test_storm_base import TestStormBase + [email protected](resource_management.libraries.functions, "get_unique_id_and_date", new = MagicMock(return_value='')) + +class TestStormServiceCheck(TestStormBase): + + def test_service_check(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/service_check.py", + classname="ServiceCheck", + command="service_check", + config_file="storm_default.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.assertResourceCalled('File', '/tmp/wordCount.jar', + content = StaticFile('wordCount.jar'), + ) + self.assertResourceCalled('Execute', 'storm jar /tmp/wordCount.jar storm.starter.WordCountTopology WordCount', + logoutput = True, + path = ['/usr/hdp/current/storm-client/bin'], + user = 'storm' + ) + self.assertResourceCalled('Execute', 'storm kill WordCount', + path = ['/usr/hdp/current/storm-client/bin'], + user = 'storm' + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/661d143b/ambari-server/src/test/python/stacks/2.3/STORM/test_storm_base.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/STORM/test_storm_base.py b/ambari-server/src/test/python/stacks/2.3/STORM/test_storm_base.py new file mode 100644 index 0000000..e0a050f --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.3/STORM/test_storm_base.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +from stacks.utils.RMFTestCase import * + + +class TestStormBase(RMFTestCase): + COMMON_SERVICES_PACKAGE_DIR = "STORM/0.9.1.2.1/package" + STACK_VERSION = "2.3" + + def assert_configure_default(self, confDir="/etc/storm/conf"): + import params + self.assertResourceCalled('Directory', '/var/log/storm', + owner = 'storm', + group = 'hadoop', + mode = 0777, + recursive = True, + ) + self.assertResourceCalled('Directory', '/var/run/storm', + owner = 'storm', + group = 'hadoop', + recursive = True, + cd_access='a' + ) + self.assertResourceCalled('Directory', '/hadoop/storm', + owner = 'storm', + group = 'hadoop', + recursive = True, + cd_access='a' + ) + self.assertResourceCalled('Directory', confDir, + group = 'hadoop', + recursive = True, + cd_access='a' + ) + self.assertResourceCalled('File', confDir + '/config.yaml', + owner = 'storm', + content = Template('config.yaml.j2'), + group = 'hadoop', + ) + + storm_yarn_content = self.call_storm_template_and_assert(confDir=confDir) + + self.assertTrue(storm_yarn_content.find('_JAAS_PLACEHOLDER') == -1, 'Placeholder have to be substituted') + + self.assertResourceCalled('File', confDir + '/storm-env.sh', + owner = 'storm', + content = InlineTemplate(self.getConfig()['configurations']['storm-env']['content']) + ) + return storm_yarn_content + + def assert_configure_secured(self, confDir='/etc/storm/conf'): + import params + self.assertResourceCalled('Directory', '/var/log/storm', + owner = 'storm', + group = 'hadoop', + mode = 0777, + recursive = True, + ) + self.assertResourceCalled('Directory', '/var/run/storm', + owner = 'storm', + group = 'hadoop', + recursive = True, + cd_access='a' + ) + self.assertResourceCalled('Directory', '/hadoop/storm', + owner = 'storm', + group = 'hadoop', + recursive = True, + cd_access='a' + ) + self.assertResourceCalled('Directory', confDir, + group = 'hadoop', + recursive = True, + cd_access='a' + ) + self.assertResourceCalled('File', confDir + '/config.yaml', + owner = 'storm', + content = Template('config.yaml.j2'), + group = 'hadoop', + ) + storm_yarn_content = self.call_storm_template_and_assert(confDir=confDir) + + self.assertTrue(storm_yarn_content.find('_JAAS_PLACEHOLDER') == -1, 'Placeholder have to be substituted') + + self.assertResourceCalled('File', confDir + '/storm-env.sh', + owner = 'storm', + content = InlineTemplate(self.getConfig()['configurations']['storm-env']['content']) + ) + self.assertResourceCalled('TemplateConfig', confDir + '/storm_jaas.conf', + owner = 'storm', + ) + return storm_yarn_content + + def call_storm_template_and_assert(self, confDir="/etc/storm/conf"): + import yaml_utils + + with RMFTestCase.env as env: + storm_yarn_temlate = yaml_utils.yaml_config_template(self.getConfig()['configurations']['storm-site']) + + self.assertResourceCalled('File', confDir + '/storm.yaml', + owner = 'storm', + content= storm_yarn_temlate, + group = 'hadoop' + ) + + return storm_yarn_temlate.get_content() http://git-wip-us.apache.org/repos/asf/ambari/blob/661d143b/ambari-server/src/test/python/stacks/2.3/configs/storm_default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/storm_default.json b/ambari-server/src/test/python/stacks/2.3/configs/storm_default.json new file mode 100644 index 0000000..b8b06eb --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.3/configs/storm_default.json @@ -0,0 +1,260 @@ +{ + "configuration_attributes": { + "storm-site": {}, + "storm-env": {}, + "zoo.cfg": {}, + "zookeeper-env": {}, + "zookeeper-log4j": {}, + "cluster-env": {} + }, + "commandParams": { + "service_package_folder": "common-services/STORM/0.9.1.2.1/package", + "script": "scripts/service_check.py", + "hooks_folder": "HDP/2.0.6/hooks", + "command_retry_max_attempt_count": "3", + "command_retry_enabled": "false", + "command_timeout": "300", + "script_type": "PYTHON" + }, + "roleCommand": "SERVICE_CHECK", + "kerberosCommandParams": [], + "clusterName": "c1", + "hostname": "c6401.ambari.apache.org", + "hostLevelParams": { + "jdk_location": "http://c6401.ambari.apache.org:8080/resources/";, + "ambari_db_rca_password": "mapred", + "java_home": "/usr/jdk64/jdk1.8.0_40", + "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca", + "jce_name": "jce_policy-8.zip", + "java_version": "8", + "oracle_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//ojdbc6.jar";, + "stack_name": "HDP", + "stack_version": "2.3", + "host_sys_prepped": "false", + "db_name": "ambari", + "jdk_name": "jdk-8u40-linux-x64.tar.gz", + "ambari_db_rca_driver": "org.postgresql.Driver", + "current_version": "2.3.0.0-1889", + "ambari_db_rca_username": "mapred", + "db_driver_filename": "mysql-connector-java.jar", + "agentCacheDir": "/var/lib/ambari-agent/cache", + "mysql_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//mysql-connector-java.jar"; + }, + "commandType": "EXECUTION_COMMAND", + "roleParams": {}, + "serviceName": "STORM", + "role": "STORM_SERVICE_CHECK", + "forceRefreshConfigTags": [], + "taskId": 31, + "public_hostname": "c6401.ambari.apache.org", + "configurations": { + "storm-site": { + "topology.tuple.serializer": "backtype.storm.serialization.types.ListDelegateSerializer", + "topology.workers": "1", + "drpc.worker.threads": "64", + "storm.messaging.netty.client_worker_threads": "1", + "supervisor.heartbeat.frequency.secs": "5", + "topology.executor.send.buffer.size": "1024", + "drpc.childopts": "-Xmx768m _JAAS_PLACEHOLDER", + "nimbus.thrift.port": "6627", + "storm.zookeeper.retry.intervalceiling.millis": "30000", + "storm.local.dir": "/hadoop/storm", + "topology.receiver.buffer.size": "8", + "storm.zookeeper.servers": "['c6401.ambari.apache.org','c6403.ambari.apache.org','c6402.ambari.apache.org']", + "transactional.zookeeper.root": "/transactional", + "_storm.min.ruid": "null", + "nimbus.seeds": "[c6402.ambari.apache.org, c6401.ambari.apache.org]", + "topology.min.replication.count": "1", + "topology.skip.missing.kryo.registrations": "false", + "worker.heartbeat.frequency.secs": "1", + "zmq.hwm": "0", + "storm.zookeeper.connection.timeout": "15000", + "_storm.thrift.secure.transport": "backtype.storm.security.auth.kerberos.KerberosSaslTransportPlugin", + "storm.messaging.netty.server_worker_threads": "1", + "supervisor.worker.start.timeout.secs": "120", + "zmq.threads": "1", + "topology.acker.executors": "null", + "storm.local.mode.zmq": "false", + "topology.max.task.parallelism": "null", + "topology.max.error.report.per.interval": "5", + "topology.debug": "false", + "drpc.queue.size": "128", + "worker.childopts": "-Xmx768m _JAAS_PLACEHOLDER -javaagent:/usr/hdp/current/storm-client/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/hdp/current/storm-client/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM", + "nimbus.childopts": "-Xmx1024m _JAAS_PLACEHOLDER -javaagent:/usr/hdp/current/storm-nimbus/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8649,wireformat31x=true,mode=multicast,config=/usr/hdp/current/storm-nimbus/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM", + "storm.zookeeper.retry.times": "5", + "nimbus.monitor.freq.secs": "10", + "storm.cluster.mode": "distributed", + "dev.zookeeper.path": "/tmp/dev-storm-zookeeper", + "drpc.invocations.port": "3773", + "_storm.thrift.nonsecure.transport": "backtype.storm.security.auth.SimpleTransportPlugin", + "storm.zookeeper.root": "/storm", + "logviewer.childopts": "-Xmx128m _JAAS_PLACEHOLDER", + "topology.max.replication.wait.time.sec": "60", + "transactional.zookeeper.port": "null", + "topology.worker.childopts": "null", + "topology.max.spout.pending": "null", + "nimbus.cleanup.inbox.freq.secs": "600", + "storm.messaging.netty.min_wait_ms": "100", + "nimbus.task.timeout.secs": "30", + "nimbus.thrift.max_buffer_size": "1048576", + "topology.sleep.spout.wait.strategy.time.ms": "1", + "topology.optimize": "true", + "nimbus.reassign": "true", + "storm.messaging.transport": "backtype.storm.messaging.netty.Context", + "logviewer.appender.name": "A1", + "ui.port": "8744", + "supervisor.slots.ports": "[6700, 6701]", + "nimbus.file.copy.expiration.secs": "600", + "supervisor.monitor.frequency.secs": "3", + "ui.childopts": "-Xmx768m _JAAS_PLACEHOLDER", + "transactional.zookeeper.servers": "null", + "zmq.linger.millis": "5000", + "topology.error.throttle.interval.secs": "10", + "topology.worker.shared.thread.pool.size": "4", + "java.library.path": "/usr/local/lib:/opt/local/lib:/usr/lib:/usr/hdp/current/storm-client/lib", + "topology.spout.wait.strategy": "backtype.storm.spout.SleepSpoutWaitStrategy", + "task.heartbeat.frequency.secs": "3", + "topology.transfer.buffer.size": "1024", + "topology.builtin.metrics.bucket.size.secs": "60", + "topology.executor.receive.buffer.size": "1024", + "topology.stats.sample.rate": "0.05", + "topology.fall.back.on.java.serialization": "true", + "supervisor.childopts": "-Xmx256m _JAAS_PLACEHOLDER -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port={{jmxremote_port}} -javaagent:/usr/hdp/current/storm-supervisor/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/hdp/current/storm-supervisor/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", + "topology.enable.message.timeouts": "true", + "storm.messaging.netty.max_wait_ms": "1000", + "nimbus.topology.validator": "backtype.storm.nimbus.DefaultTopologyValidator", + "nimbus.supervisor.timeout.secs": "60", + "topology.disruptor.wait.strategy": "com.lmax.disruptor.BlockingWaitStrategy", + "storm.messaging.netty.buffer_size": "5242880", + "drpc.port": "3772", + "topology.kryo.factory": "backtype.storm.serialization.DefaultKryoFactory", + "storm.zookeeper.retry.interval": "1000", + "storm.messaging.netty.max_retries": "30", + "topology.tick.tuple.freq.secs": "null", + "drpc.request.timeout.secs": "600", + "nimbus.task.launch.secs": "120", + "task.refresh.poll.secs": "10", + "topology.message.timeout.secs": "30", + "nimbus.inbox.jar.expiration.secs": "3600", + "topology.state.synchronization.timeout.secs": "60", + "supervisor.worker.timeout.secs": "30", + "ui.filter": "null", + "topology.trident.batch.emit.interval.millis": "500", + "storm.zookeeper.session.timeout": "20000", + "storm.thrift.transport": "{{storm_thrift_transport}}", + "logviewer.port": "8000", + "storm.log.dir": "{{log_dir}}", + "storm.zookeeper.port": "2181" + }, + "storm-env": { + "jmxremote_port": "56431", + "storm_log_dir": "/var/log/storm", + "storm_pid_dir": "/var/run/storm", + "content": "\n#!/bin/bash\n\n# Set Storm specific environment variables here.\n\n# The java implementation to use.\nexport JAVA_HOME={{java64_home}}\n\n# export STORM_CONF_DIR=\"\"\nexport STORM_HOME=/usr/hdp/current/storm-client", + "nimbus_seeds_supported": "true", + "storm_user": "storm" + }, + "zoo.cfg": { + "clientPort": "2181", + "autopurge.purgeInterval": "24", + "syncLimit": "5", + "dataDir": "/hadoop/zookeeper", + "initLimit": "10", + "tickTime": "2000", + "autopurge.snapRetainCount": "30" + }, + "zookeeper-env": { + "zk_log_dir": "/var/log/zookeeper", + "content": "\nexport JAVA_HOME={{java64_home}}\nexport ZOOKEEPER_HOME={{zk_home}}\nexport ZOO_LOG_DIR={{zk_log_dir}}\nexport ZOOPIDFILE={{zk_pid_file}}\nexport SERVER_JVMFLAGS={{zk_server_heapsize}}\nexport JAVA=$JAVA_HOME/bin/java\nexport CLASSPATH=$CLASSPATH:/usr/share/zookeeper/*\n\n{% if security_enabled %}\nexport SERVER_JVMFLAGS=\"$SERVER_JVMFLAGS -Djava.security.auth.login.config={{zk_server_jaas_file}}\"\nexport CLIENT_JVMFLAGS=\"$CLIENT_JVMFLAGS -Djava.security.auth.login.config={{zk_client_jaas_file}}\"\n{% endif %}", + "zk_pid_dir": "/var/run/zookeeper", + "zk_user": "zookeeper" + }, + "zookeeper-log4j": { + "content": "\n#\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n#\n\n#\n# ZooKeeper Logging Configuration\n#\n\n# DEFAULT: console appender only\nlog4j.rootLogger=INFO, CONSOLE\n\n# Example with rolling log file\n#log4 j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE\n\n# Example with rolling log file and tracing\n#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE\n\n#\n# Log INFO level and above messages to the console\n#\nlog4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender\nlog4j.appender.CONSOLE.Threshold=INFO\nlog4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n#\n# Add ROLLINGFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender\nlog4j.appender.ROLLINGFILE.Threshold=DEBUG\nlog4j.appender.ROLLINGFILE.File=zookeeper.log\n\n# Max log file size of 10MB\nlog4j.appender.ROLLINGFILE.MaxFileSize=10MB\n# uncomment the next line to limit number of backup files\n#log4j.appender.ROLLINGFILE.MaxBackupIndex=10\n\nlog4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.ROLLIN GFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n\n#\n# Add TRACEFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.TRACEFILE=org.apache.log4j.FileAppender\nlog4j.appender.TRACEFILE.Threshold=TRACE\nlog4j.appender.TRACEFILE.File=zookeeper_trace.log\n\nlog4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout\n### Notice we are including log4j's NDC here (%x)\nlog4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n" + }, + "cluster-env": { + "security_enabled": "false", + "hive_tar_source": "/usr/hdp/current/hive-client/hive.tar.gz", + "hadoop-streaming_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", + "pig_tar_source": "/usr/hdp/current/pig-client/pig.tar.gz", + "hive_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/", + "ignore_groupsusers_create": "false", + "smokeuser_keytab": "/etc/security/keytabs/smokeuser.headless.keytab", + "hadoop-streaming_tar_source": "/usr/hdp/current/hadoop-mapreduce-client/hadoop-streaming.jar", + "kerberos_domain": "EXAMPLE.COM", + "tez_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/", + "mapreduce_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", + "tez_tar_source": "/usr/hdp/current/tez-client/lib/tez.tar.gz", + "pig_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/", + "user_group": "hadoop", + "mapreduce_tar_source": "/usr/hdp/current/hadoop-client/mapreduce.tar.gz", + "sqoop_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/", + "smokeuser": "ambari-qa", + "sqoop_tar_source": "/usr/hdp/current/sqoop-client/sqoop.tar.gz" + } + }, + "configurationTags": { + "storm-site": { + "tag": "version1" + }, + "storm-env": { + "tag": "version1" + }, + "zoo.cfg": { + "tag": "version1" + }, + "zookeeper-env": { + "tag": "version1" + }, + "zookeeper-log4j": { + "tag": "version1" + }, + "cluster-env": { + "tag": "version1" + } + }, + "commandId": "7-0", + "clusterHostInfo": { + "drpc_server_hosts": [ + "c6402.ambari.apache.org" + ], + "nimbus_hosts": [ + "c6402.ambari.apache.org", + "c6401.ambari.apache.org" + ], + "all_ping_ports": [ + "8670", + "8670", + "8670" + ], + "all_hosts": [ + "c6402.ambari.apache.org", + "c6403.ambari.apache.org", + "c6401.ambari.apache.org" + ], + "storm_ui_server_hosts": [ + "c6402.ambari.apache.org" + ], + "all_racks": [ + "/default-rack", + "/default-rack", + "/default-rack" + ], + "all_ipv4_ips": [ + "192.168.64.102", + "192.168.64.103", + "192.168.64.101" + ], + "ambari_server_host": [ + "c6401.ambari.apache.org" + ], + "zookeeper_hosts": [ + "c6402.ambari.apache.org", + "c6403.ambari.apache.org", + "c6401.ambari.apache.org" + ], + "supervisor_hosts": [ + "c6401.ambari.apache.org" + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/661d143b/ambari-web/app/data/HDP2/site_properties.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/data/HDP2/site_properties.js b/ambari-web/app/data/HDP2/site_properties.js index 2c50819..7fd2c26 100644 --- a/ambari-web/app/data/HDP2/site_properties.js +++ b/ambari-web/app/data/HDP2/site_properties.js @@ -1035,7 +1035,6 @@ var hdp2properties = [ "name": "nimbus.seeds", "displayName": "nimbus.seeds", "displayType": "masterHosts", - "isOverridable": false, "serviceName": "STORM", "filename": "storm-site.xml", "category": "NIMBUS",
