This is an automated email from the ASF dual-hosted git repository. echekanskiy pushed a commit to branch branch-2.6 in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.6 by this push: new f4a4dca AMBARI-22819. Issues with storm jaas files and recursive ch_(mod/own) calls (echekanskiy) f4a4dca is described below commit f4a4dca591d18076c0afb26807d2961bc4ac1d46 Author: Eugene Chekanskiy <echekans...@gmail.com> AuthorDate: Fri Feb 16 13:57:05 2018 +0200 AMBARI-22819. Issues with storm jaas files and recursive ch_(mod/own) calls (echekanskiy) --- .../python/ambari_commons/unicode_tolerant_fs.py | 63 ++++++++++++++++++++++ .../main/python/resource_management/core/sudo.py | 5 +- .../STORM/0.9.1/package/scripts/storm.py | 21 ++++++-- .../python/stacks/2.1/STORM/test_storm_base.py | 32 ++++++++++- .../2.1/STORM/test_storm_jaas_configuration.py | 1 + .../python/stacks/2.1/STORM/test_storm_nimbus.py | 34 +----------- .../stacks/2.1/STORM/test_storm_supervisor_prod.py | 1 - .../python/stacks/2.3/STORM/test_storm_base.py | 1 + 8 files changed, 117 insertions(+), 41 deletions(-) diff --git a/ambari-common/src/main/python/ambari_commons/unicode_tolerant_fs.py b/ambari-common/src/main/python/ambari_commons/unicode_tolerant_fs.py new file mode 100644 index 0000000..a2b8e6b --- /dev/null +++ b/ambari-common/src/main/python/ambari_commons/unicode_tolerant_fs.py @@ -0,0 +1,63 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + + +def unicode_walk(top, topdown=True, onerror=None, followlinks=False): + """ + Unicode tolerant version of os.walk. Can(and must) be used environments with messed locales(and other encoding-related + problems) to traverse directories trees with unicode names in files and directories. All others function seems like + to accept utf-8 encoded strings, so result of `unicode_walk` can be used without a problems. + """ + import os.path + import os + + islink, join, isdir = os.path.islink, os.path.join, os.path.isdir + + top = top.encode("utf8") + + try: + # Note that listdir and error are globals in this module due + # to earlier import-*. + names = os.listdir(top) + except os.error, err: + if onerror is not None: + onerror(err) + return + + dirs, nondirs = [], [] + for name in names: + name = name.encode("utf8") + if isdir(join(top, name)): + dirs.append(name) + else: + nondirs.append(name) + + if topdown: + yield top, dirs, nondirs + for name in dirs: + name = name.encode("utf8") + new_path = join(top, name) + if followlinks or not islink(new_path): + for x in unicode_walk(new_path, topdown, onerror, followlinks): + yield x + if not topdown: + yield top, dirs, nondirs + diff --git a/ambari-common/src/main/python/resource_management/core/sudo.py b/ambari-common/src/main/python/resource_management/core/sudo.py index 2989367..a0a8e45 100644 --- a/ambari-common/src/main/python/resource_management/core/sudo.py +++ b/ambari-common/src/main/python/resource_management/core/sudo.py @@ -28,6 +28,7 @@ import errno import random from resource_management.core import shell from resource_management.core.exceptions import Fail +from ambari_commons.unicode_tolerant_fs import unicode_walk from ambari_commons import subprocess32 from resource_management.core.utils import attr_to_bitmask @@ -46,7 +47,7 @@ if os.geteuid() == 0: if uid == -1 and gid == -1: return - for root, dirs, files in os.walk(path, followlinks=follow_links): + for root, dirs, files in unicode_walk(path, followlinks=True): for name in files + dirs: if follow_links: os.chown(os.path.join(root, name), uid, gid) @@ -83,7 +84,7 @@ if os.geteuid() == 0: dir_attrib = recursive_mode_flags["d"] if "d" in recursive_mode_flags else None files_attrib = recursive_mode_flags["f"] if "d" in recursive_mode_flags else None - for root, dirs, files in os.walk(path, followlinks=recursion_follow_links): + for root, dirs, files in unicode_walk(path, followlinks=recursion_follow_links): if dir_attrib is not None: for dir_name in dirs: full_dir_path = os.path.join(root, dir_name) diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py index 99579d2..1b003d6 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py @@ -154,11 +154,13 @@ def storm(name=None): if params.security_enabled: TemplateConfig(format("{conf_dir}/storm_jaas.conf"), - owner=params.storm_user + owner=params.storm_user, + mode=0644 ) if params.stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version_formatted): TemplateConfig(format("{conf_dir}/client_jaas.conf"), - owner=params.storm_user + owner=params.storm_user, + mode=0644 ) minRuid = configurations['_storm.min.ruid'] if configurations.has_key('_storm.min.ruid') else '' @@ -169,12 +171,21 @@ def storm(name=None): owner='root', group=params.user_group ) + else: + File( + format("{conf_dir}/storm_jaas.conf"), + action="delete" + ) + File( + format("{conf_dir}/client_jaas.conf"), + action="delete" + ) -''' -Finds minimal real user UID -''' def _find_real_user_min_uid(): + """ + Finds minimal real user UID + """ with open('/etc/login.defs') as f: for line in f: if line.strip().startswith('UID_MIN') and len(line.split()) == 2 and line.split()[1].isdigit(): diff --git a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py index f2e05be..dc615ae 100644 --- a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py +++ b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py @@ -28,7 +28,7 @@ class TestStormBase(RMFTestCase): COMMON_SERVICES_PACKAGE_DIR = "STORM/0.9.1/package" STACK_VERSION = "2.1" - def assert_configure_default(self, confDir="/etc/storm/conf"): + def assert_configure_default(self, confDir="/etc/storm/conf", has_metrics=False, legacy=True): import params self.assertResourceCalled('Directory', '/var/log/storm', owner = 'storm', @@ -76,6 +76,35 @@ class TestStormBase(RMFTestCase): owner = 'storm', content = InlineTemplate(self.getConfig()['configurations']['storm-env']['content']) ) + if has_metrics: + self.assertResourceCalled('File', confDir + '/storm-metrics2.properties', + content = Template('storm-metrics2.properties.j2'), + owner = 'storm', + group = 'hadoop', + ) + self.assertResourceCalled('Link', '/usr/lib/storm/lib//ambari-metrics-storm-sink.jar', + action = ['delete'], + ) + self.assertResourceCalled('Link', '/usr/lib/storm/lib/ambari-metrics-storm-sink.jar', + action = ['delete'], + ) + if legacy: + self.assertResourceCalled('Execute', 'ambari-sudo.sh ln -s /usr/lib/storm/lib/ambari-metrics-storm-sink-legacy-with-common-*.jar /usr/lib/storm/lib//ambari-metrics-storm-sink.jar', + not_if = 'ls /usr/lib/storm/lib//ambari-metrics-storm-sink.jar', + only_if = 'ls /usr/lib/storm/lib/ambari-metrics-storm-sink-legacy-with-common-*.jar', + ) + else: + self.assertResourceCalled('Execute', 'ambari-sudo.sh ln -s /usr/lib/storm/lib/ambari-metrics-storm-sink-with-common-*.jar /usr/lib/storm/lib//ambari-metrics-storm-sink.jar', + not_if = 'ls /usr/lib/storm/lib//ambari-metrics-storm-sink.jar', + only_if = 'ls /usr/lib/storm/lib/ambari-metrics-storm-sink-with-common-*.jar', + ) + + self.assertResourceCalled('File', confDir + '/storm_jaas.conf', + action=['delete'], + ) + self.assertResourceCalled('File', confDir + '/client_jaas.conf', + action=['delete'], + ) return storm_yarn_content def assert_configure_secured(self, confDir='/etc/storm/conf'): @@ -127,6 +156,7 @@ class TestStormBase(RMFTestCase): ) self.assertResourceCalled('TemplateConfig', confDir + '/storm_jaas.conf', owner = 'storm', + mode = 0644 ) return storm_yarn_content diff --git a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_jaas_configuration.py b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_jaas_configuration.py index ddb111b..9b13427 100644 --- a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_jaas_configuration.py +++ b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_jaas_configuration.py @@ -83,6 +83,7 @@ class TestStormJaasConfiguration(TestStormBase): storm_yarn_content = super(TestStormJaasConfiguration, self).assert_configure_secured(confDir="/usr/hdp/current/storm-nimbus/conf") self.assertResourceCalled('TemplateConfig', '/usr/hdp/current/storm-nimbus/conf/client_jaas.conf', owner = 'storm', + mode = 0644 ) self.assertResourceCalled('File', '/usr/hdp/current/storm-nimbus/conf/worker-launcher.cfg', owner = 'root', diff --git a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py index a2f2e5e..9cece09 100644 --- a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py +++ b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py @@ -83,23 +83,8 @@ class TestStormNimbus(TestStormBase): stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) - self.assert_configure_default() + self.assert_configure_default(has_metrics=True) - self.assertResourceCalled('File', '/etc/storm/conf/storm-metrics2.properties', - content = Template('storm-metrics2.properties.j2'), - owner = 'storm', - group = 'hadoop', - ) - self.assertResourceCalled('Link', '/usr/lib/storm/lib//ambari-metrics-storm-sink.jar', - action = ['delete'], - ) - self.assertResourceCalled('Link', '/usr/lib/storm/lib/ambari-metrics-storm-sink.jar', - action = ['delete'], - ) - self.assertResourceCalled('Execute', 'ambari-sudo.sh ln -s /usr/lib/storm/lib/ambari-metrics-storm-sink-legacy-with-common-*.jar /usr/lib/storm/lib//ambari-metrics-storm-sink.jar', - not_if = 'ls /usr/lib/storm/lib//ambari-metrics-storm-sink.jar', - only_if = 'ls /usr/lib/storm/lib/ambari-metrics-storm-sink-legacy-with-common-*.jar', - ) self.assertResourceCalled('Execute', 'source /etc/storm/conf/storm-env.sh ; export PATH=$JAVA_HOME/bin:$PATH ; storm nimbus > /var/log/storm/nimbus.out 2>&1 &\n echo $! > /var/run/storm/nimbus.pid', path = ['/usr/bin'], user = 'storm', @@ -134,23 +119,8 @@ class TestStormNimbus(TestStormBase): stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) - self.assert_configure_default() + self.assert_configure_default(has_metrics=True, legacy=False) - self.assertResourceCalled('File', '/etc/storm/conf/storm-metrics2.properties', - content = Template('storm-metrics2.properties.j2'), - owner = 'storm', - group = 'hadoop', - ) - self.assertResourceCalled('Link', '/usr/lib/storm/lib//ambari-metrics-storm-sink.jar', - action = ['delete'], - ) - self.assertResourceCalled('Link', '/usr/lib/storm/lib/ambari-metrics-storm-sink.jar', - action = ['delete'], - ) - self.assertResourceCalled('Execute', 'ambari-sudo.sh ln -s /usr/lib/storm/lib/ambari-metrics-storm-sink-with-common-*.jar /usr/lib/storm/lib//ambari-metrics-storm-sink.jar', - not_if = 'ls /usr/lib/storm/lib//ambari-metrics-storm-sink.jar', - only_if = 'ls /usr/lib/storm/lib/ambari-metrics-storm-sink-with-common-*.jar', - ) self.assertResourceCalled('Execute', 'source /etc/storm/conf/storm-env.sh ; export PATH=$JAVA_HOME/bin:$PATH ; storm nimbus > /var/log/storm/nimbus.out 2>&1 &\n echo $! > /var/run/storm/nimbus.pid', path = ['/usr/bin'], user = 'storm', diff --git a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor_prod.py b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor_prod.py index f995d89..ac756af 100644 --- a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor_prod.py +++ b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor_prod.py @@ -50,7 +50,6 @@ class TestStormSupervisor(TestStormBase): ) self.assert_configure_default() - self.assertResourceCalled('Execute', 'supervisorctl start storm-supervisor', wait_for_finish = False, ) diff --git a/ambari-server/src/test/python/stacks/2.3/STORM/test_storm_base.py b/ambari-server/src/test/python/stacks/2.3/STORM/test_storm_base.py index c95adb6..721d115 100644 --- a/ambari-server/src/test/python/stacks/2.3/STORM/test_storm_base.py +++ b/ambari-server/src/test/python/stacks/2.3/STORM/test_storm_base.py @@ -108,6 +108,7 @@ class TestStormBase(RMFTestCase): ) self.assertResourceCalled('TemplateConfig', confDir + '/storm_jaas.conf', owner = 'storm', + mode = 0644, ) return storm_yarn_content -- To stop receiving notification emails like this one, please contact echekans...@apache.org.