This is an automated email from the ASF dual-hosted git repository.
echekanskiy pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.6 by this push:
new f4a4dca AMBARI-22819. Issues with storm jaas files and recursive
ch_(mod/own) calls (echekanskiy)
f4a4dca is described below
commit f4a4dca591d18076c0afb26807d2961bc4ac1d46
Author: Eugene Chekanskiy <echekans...@gmail.com>
AuthorDate: Fri Feb 16 13:57:05 2018 +0200
AMBARI-22819. Issues with storm jaas files and recursive ch_(mod/own) calls
(echekanskiy)
---
.../python/ambari_commons/unicode_tolerant_fs.py | 63 ++++++++++++++++++++++
.../main/python/resource_management/core/sudo.py | 5 +-
.../STORM/0.9.1/package/scripts/storm.py | 21 ++++++--
.../python/stacks/2.1/STORM/test_storm_base.py | 32 ++++++++++-
.../2.1/STORM/test_storm_jaas_configuration.py | 1 +
.../python/stacks/2.1/STORM/test_storm_nimbus.py | 34 +-----------
.../stacks/2.1/STORM/test_storm_supervisor_prod.py | 1 -
.../python/stacks/2.3/STORM/test_storm_base.py | 1 +
8 files changed, 117 insertions(+), 41 deletions(-)
diff --git
a/ambari-common/src/main/python/ambari_commons/unicode_tolerant_fs.py
b/ambari-common/src/main/python/ambari_commons/unicode_tolerant_fs.py
new file mode 100644
index 0000000..a2b8e6b
--- /dev/null
+++ b/ambari-common/src/main/python/ambari_commons/unicode_tolerant_fs.py
@@ -0,0 +1,63 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+
+def unicode_walk(top, topdown=True, onerror=None, followlinks=False):
+ """
+ Unicode tolerant version of os.walk. Can(and must) be used environments with
messed locales(and other encoding-related
+ problems) to traverse directories trees with unicode names in files and
directories. All others function seems like
+ to accept utf-8 encoded strings, so result of `unicode_walk` can be used
without a problems.
+ """
+ import os.path
+ import os
+
+ islink, join, isdir = os.path.islink, os.path.join, os.path.isdir
+
+ top = top.encode("utf8")
+
+ try:
+ # Note that listdir and error are globals in this module due
+ # to earlier import-*.
+ names = os.listdir(top)
+ except os.error, err:
+ if onerror is not None:
+ onerror(err)
+ return
+
+ dirs, nondirs = [], []
+ for name in names:
+ name = name.encode("utf8")
+ if isdir(join(top, name)):
+ dirs.append(name)
+ else:
+ nondirs.append(name)
+
+ if topdown:
+ yield top, dirs, nondirs
+ for name in dirs:
+ name = name.encode("utf8")
+ new_path = join(top, name)
+ if followlinks or not islink(new_path):
+ for x in unicode_walk(new_path, topdown, onerror, followlinks):
+ yield x
+ if not topdown:
+ yield top, dirs, nondirs
+
diff --git a/ambari-common/src/main/python/resource_management/core/sudo.py
b/ambari-common/src/main/python/resource_management/core/sudo.py
index 2989367..a0a8e45 100644
--- a/ambari-common/src/main/python/resource_management/core/sudo.py
+++ b/ambari-common/src/main/python/resource_management/core/sudo.py
@@ -28,6 +28,7 @@ import errno
import random
from resource_management.core import shell
from resource_management.core.exceptions import Fail
+from ambari_commons.unicode_tolerant_fs import unicode_walk
from ambari_commons import subprocess32
from resource_management.core.utils import attr_to_bitmask
@@ -46,7 +47,7 @@ if os.geteuid() == 0:
if uid == -1 and gid == -1:
return
- for root, dirs, files in os.walk(path, followlinks=follow_links):
+ for root, dirs, files in unicode_walk(path, followlinks=True):
for name in files + dirs:
if follow_links:
os.chown(os.path.join(root, name), uid, gid)
@@ -83,7 +84,7 @@ if os.geteuid() == 0:
dir_attrib = recursive_mode_flags["d"] if "d" in recursive_mode_flags else
None
files_attrib = recursive_mode_flags["f"] if "d" in recursive_mode_flags
else None
- for root, dirs, files in os.walk(path, followlinks=recursion_follow_links):
+ for root, dirs, files in unicode_walk(path,
followlinks=recursion_follow_links):
if dir_attrib is not None:
for dir_name in dirs:
full_dir_path = os.path.join(root, dir_name)
diff --git
a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py
b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py
index 99579d2..1b003d6 100644
---
a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py
+++
b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py
@@ -154,11 +154,13 @@ def storm(name=None):
if params.security_enabled:
TemplateConfig(format("{conf_dir}/storm_jaas.conf"),
- owner=params.storm_user
+ owner=params.storm_user,
+ mode=0644
)
if params.stack_version_formatted and
check_stack_feature(StackFeature.ROLLING_UPGRADE,
params.stack_version_formatted):
TemplateConfig(format("{conf_dir}/client_jaas.conf"),
- owner=params.storm_user
+ owner=params.storm_user,
+ mode=0644
)
minRuid = configurations['_storm.min.ruid'] if
configurations.has_key('_storm.min.ruid') else ''
@@ -169,12 +171,21 @@ def storm(name=None):
owner='root',
group=params.user_group
)
+ else:
+ File(
+ format("{conf_dir}/storm_jaas.conf"),
+ action="delete"
+ )
+ File(
+ format("{conf_dir}/client_jaas.conf"),
+ action="delete"
+ )
-'''
-Finds minimal real user UID
-'''
def _find_real_user_min_uid():
+ """
+ Finds minimal real user UID
+ """
with open('/etc/login.defs') as f:
for line in f:
if line.strip().startswith('UID_MIN') and len(line.split()) == 2 and
line.split()[1].isdigit():
diff --git a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py
b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py
index f2e05be..dc615ae 100644
--- a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py
+++ b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py
@@ -28,7 +28,7 @@ class TestStormBase(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "STORM/0.9.1/package"
STACK_VERSION = "2.1"
- def assert_configure_default(self, confDir="/etc/storm/conf"):
+ def assert_configure_default(self, confDir="/etc/storm/conf",
has_metrics=False, legacy=True):
import params
self.assertResourceCalled('Directory', '/var/log/storm',
owner = 'storm',
@@ -76,6 +76,35 @@ class TestStormBase(RMFTestCase):
owner = 'storm',
content =
InlineTemplate(self.getConfig()['configurations']['storm-env']['content'])
)
+ if has_metrics:
+ self.assertResourceCalled('File', confDir + '/storm-metrics2.properties',
+ content =
Template('storm-metrics2.properties.j2'),
+ owner = 'storm',
+ group = 'hadoop',
+ )
+ self.assertResourceCalled('Link',
'/usr/lib/storm/lib//ambari-metrics-storm-sink.jar',
+ action = ['delete'],
+ )
+ self.assertResourceCalled('Link',
'/usr/lib/storm/lib/ambari-metrics-storm-sink.jar',
+ action = ['delete'],
+ )
+ if legacy:
+ self.assertResourceCalled('Execute', 'ambari-sudo.sh ln -s
/usr/lib/storm/lib/ambari-metrics-storm-sink-legacy-with-common-*.jar
/usr/lib/storm/lib//ambari-metrics-storm-sink.jar',
+ not_if = 'ls
/usr/lib/storm/lib//ambari-metrics-storm-sink.jar',
+ only_if = 'ls
/usr/lib/storm/lib/ambari-metrics-storm-sink-legacy-with-common-*.jar',
+ )
+ else:
+ self.assertResourceCalled('Execute', 'ambari-sudo.sh ln -s
/usr/lib/storm/lib/ambari-metrics-storm-sink-with-common-*.jar
/usr/lib/storm/lib//ambari-metrics-storm-sink.jar',
+ not_if = 'ls
/usr/lib/storm/lib//ambari-metrics-storm-sink.jar',
+ only_if = 'ls
/usr/lib/storm/lib/ambari-metrics-storm-sink-with-common-*.jar',
+ )
+
+ self.assertResourceCalled('File', confDir + '/storm_jaas.conf',
+ action=['delete'],
+ )
+ self.assertResourceCalled('File', confDir + '/client_jaas.conf',
+ action=['delete'],
+ )
return storm_yarn_content
def assert_configure_secured(self, confDir='/etc/storm/conf'):
@@ -127,6 +156,7 @@ class TestStormBase(RMFTestCase):
)
self.assertResourceCalled('TemplateConfig', confDir + '/storm_jaas.conf',
owner = 'storm',
+ mode = 0644
)
return storm_yarn_content
diff --git
a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_jaas_configuration.py
b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_jaas_configuration.py
index ddb111b..9b13427 100644
---
a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_jaas_configuration.py
+++
b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_jaas_configuration.py
@@ -83,6 +83,7 @@ class TestStormJaasConfiguration(TestStormBase):
storm_yarn_content = super(TestStormJaasConfiguration,
self).assert_configure_secured(confDir="/usr/hdp/current/storm-nimbus/conf")
self.assertResourceCalled('TemplateConfig',
'/usr/hdp/current/storm-nimbus/conf/client_jaas.conf',
owner = 'storm',
+ mode = 0644
)
self.assertResourceCalled('File',
'/usr/hdp/current/storm-nimbus/conf/worker-launcher.cfg',
owner = 'root',
diff --git
a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py
b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py
index a2f2e5e..9cece09 100644
--- a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py
+++ b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py
@@ -83,23 +83,8 @@ class TestStormNimbus(TestStormBase):
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
- self.assert_configure_default()
+ self.assert_configure_default(has_metrics=True)
- self.assertResourceCalled('File',
'/etc/storm/conf/storm-metrics2.properties',
- content =
Template('storm-metrics2.properties.j2'),
- owner = 'storm',
- group = 'hadoop',
- )
- self.assertResourceCalled('Link',
'/usr/lib/storm/lib//ambari-metrics-storm-sink.jar',
- action = ['delete'],
- )
- self.assertResourceCalled('Link',
'/usr/lib/storm/lib/ambari-metrics-storm-sink.jar',
- action = ['delete'],
- )
- self.assertResourceCalled('Execute', 'ambari-sudo.sh ln -s
/usr/lib/storm/lib/ambari-metrics-storm-sink-legacy-with-common-*.jar
/usr/lib/storm/lib//ambari-metrics-storm-sink.jar',
- not_if = 'ls
/usr/lib/storm/lib//ambari-metrics-storm-sink.jar',
- only_if = 'ls
/usr/lib/storm/lib/ambari-metrics-storm-sink-legacy-with-common-*.jar',
- )
self.assertResourceCalled('Execute', 'source /etc/storm/conf/storm-env.sh
; export PATH=$JAVA_HOME/bin:$PATH ; storm nimbus > /var/log/storm/nimbus.out
2>&1 &\n echo $! > /var/run/storm/nimbus.pid',
path = ['/usr/bin'],
user = 'storm',
@@ -134,23 +119,8 @@ class TestStormNimbus(TestStormBase):
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
- self.assert_configure_default()
+ self.assert_configure_default(has_metrics=True, legacy=False)
- self.assertResourceCalled('File',
'/etc/storm/conf/storm-metrics2.properties',
- content =
Template('storm-metrics2.properties.j2'),
- owner = 'storm',
- group = 'hadoop',
- )
- self.assertResourceCalled('Link',
'/usr/lib/storm/lib//ambari-metrics-storm-sink.jar',
- action = ['delete'],
- )
- self.assertResourceCalled('Link',
'/usr/lib/storm/lib/ambari-metrics-storm-sink.jar',
- action = ['delete'],
- )
- self.assertResourceCalled('Execute', 'ambari-sudo.sh ln -s
/usr/lib/storm/lib/ambari-metrics-storm-sink-with-common-*.jar
/usr/lib/storm/lib//ambari-metrics-storm-sink.jar',
- not_if = 'ls
/usr/lib/storm/lib//ambari-metrics-storm-sink.jar',
- only_if = 'ls
/usr/lib/storm/lib/ambari-metrics-storm-sink-with-common-*.jar',
- )
self.assertResourceCalled('Execute', 'source /etc/storm/conf/storm-env.sh
; export PATH=$JAVA_HOME/bin:$PATH ; storm nimbus > /var/log/storm/nimbus.out
2>&1 &\n echo $! > /var/run/storm/nimbus.pid',
path = ['/usr/bin'],
user = 'storm',
diff --git
a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor_prod.py
b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor_prod.py
index f995d89..ac756af 100644
---
a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor_prod.py
+++
b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor_prod.py
@@ -50,7 +50,6 @@ class TestStormSupervisor(TestStormBase):
)
self.assert_configure_default()
-
self.assertResourceCalled('Execute', 'supervisorctl start
storm-supervisor',
wait_for_finish = False,
)
diff --git a/ambari-server/src/test/python/stacks/2.3/STORM/test_storm_base.py
b/ambari-server/src/test/python/stacks/2.3/STORM/test_storm_base.py
index c95adb6..721d115 100644
--- a/ambari-server/src/test/python/stacks/2.3/STORM/test_storm_base.py
+++ b/ambari-server/src/test/python/stacks/2.3/STORM/test_storm_base.py
@@ -108,6 +108,7 @@ class TestStormBase(RMFTestCase):
)
self.assertResourceCalled('TemplateConfig', confDir + '/storm_jaas.conf',
owner = 'storm',
+ mode = 0644,
)
return storm_yarn_content
--
To stop receiving notification emails like this one, please contact
echekans...@apache.org.
