Verify if restricting acls on /var/lib/ambari-agent/data will be OK (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b8388e12 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b8388e12 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b8388e12 Branch: refs/heads/trunk Commit: b8388e12b348e3989490e812dd2fe993b2ca603e Parents: e837d4f Author: Andrew Onishuk <[email protected]> Authored: Mon Sep 14 15:01:01 2015 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Mon Sep 14 15:01:01 2015 +0300 ---------------------------------------------------------------------- ambari-agent/conf/unix/ambari-agent.ini | 1 - ambari-agent/conf/unix/install-helper.sh | 5 +- ambari-agent/pom.xml | 4 +- .../src/main/python/ambari_agent/Constants.py | 21 + .../ambari_agent/CustomServiceOrchestrator.py | 3 +- .../python/ambari_agent/alerts/metric_alert.py | 3 +- .../python/ambari_agent/alerts/script_alert.py | 3 +- .../python/ambari_agent/alerts/web_alert.py | 3 +- .../src/main/python/ambari_agent/security.py | 11 +- .../python/ambari_agent/TestCertGeneration.py | 6 +- .../ambari_agent/TestCertGeneration.py.orig | 50 ++ .../TestCustomServiceOrchestrator.py | 1 - .../TestCustomServiceOrchestrator.py.orig | 643 ++++++++++++++ .../test/python/ambari_agent/TestSecurity.py | 6 +- .../python/ambari_agent/TestSecurity.py.orig | 389 ++++++++ .../libraries/script/script.py | 2 +- ambari-server/src/main/python/bootstrap.py | 2 +- .../HDFS/2.1.0.2.0/package/scripts/namenode.py | 2 +- ambari-server/src/test/python/TestBootstrap.py | 12 +- .../src/test/python/TestBootstrap.py.orig | 879 +++++++++++++++++++ 20 files changed, 2020 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/conf/unix/ambari-agent.ini ---------------------------------------------------------------------- diff --git a/ambari-agent/conf/unix/ambari-agent.ini b/ambari-agent/conf/unix/ambari-agent.ini index abfde62..3b7631c 100644 --- a/ambari-agent/conf/unix/ambari-agent.ini +++ b/ambari-agent/conf/unix/ambari-agent.ini @@ -19,7 +19,6 @@ secured_url_port=8441 [agent] prefix=/var/lib/ambari-agent/data -tmp_dir=/var/lib/ambari-agent/data/tmp ;loglevel=(DEBUG/INFO) loglevel=INFO data_cleanup_interval=86400 http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/conf/unix/install-helper.sh ---------------------------------------------------------------------- diff --git a/ambari-agent/conf/unix/install-helper.sh b/ambari-agent/conf/unix/install-helper.sh index 48391d5..35c67fb 100644 --- a/ambari-agent/conf/unix/install-helper.sh +++ b/ambari-agent/conf/unix/install-helper.sh @@ -58,8 +58,9 @@ do_install(){ # on nano Ubuntu, when umask=027 those folders are created without 'x' bit for 'others'. # which causes failures when hadoop users try to access tmp_dir chmod a+x /var/lib/ambari-agent - chmod a+x /var/lib/ambari-agent/data - chmod 777 /var/lib/ambari-agent/data/tmp + + chmod 777 /var/lib/ambari-agent/tmp + chmod 700 /var/lib/ambari-agent/data } do_remove(){ http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-agent/pom.xml b/ambari-agent/pom.xml index 24927e2..0096162 100644 --- a/ambari-agent/pom.xml +++ b/ambari-agent/pom.xml @@ -391,7 +391,7 @@ <groupname>root</groupname> </mapping> <mapping> - <directory>/var/lib/${project.artifactId}/data/tmp</directory> + <directory>/var/lib/${project.artifactId}/tmp</directory> <filemode>777</filemode> <username>root</username> <groupname>root</groupname> @@ -596,7 +596,7 @@ <paths> <path>${package.pid.dir}</path> <path>/var/lib/${project.artifactId}/data</path> - <path>/var/lib/${project.artifactId}/data/tmp</path> + <path>/var/lib/${project.artifactId}/tmp</path> <path>/var/lib/${project.artifactId}/keys</path> <path>${package.log.dir}</path> <path>/var/lib/${project.artifactId}/lib</path> http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/Constants.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/Constants.py b/ambari-agent/src/main/python/ambari_agent/Constants.py new file mode 100644 index 0000000..6b80f00 --- /dev/null +++ b/ambari-agent/src/main/python/ambari_agent/Constants.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +AGENT_TMP_DIR = "/var/lib/ambari-agent/tmp" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py index 6ee929c..6c1a161 100644 --- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py +++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py @@ -29,6 +29,7 @@ from FileCache import FileCache from AgentException import AgentException from PythonExecutor import PythonExecutor from PythonReflectiveExecutor import PythonReflectiveExecutor +import Constants import hostname @@ -61,7 +62,7 @@ class CustomServiceOrchestrator(): def __init__(self, config, controller): self.config = config self.tmp_dir = config.get('agent', 'prefix') - self.exec_tmp_dir = config.get('agent', 'tmp_dir') + self.exec_tmp_dir = Constants.AGENT_TMP_DIR self.file_cache = FileCache(config) self.status_commands_stdout = os.path.join(self.tmp_dir, 'status_command_stdout.txt') http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py index aa4ad75..8de49cd 100644 --- a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py +++ b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py @@ -30,6 +30,7 @@ from alerts.base_alert import BaseAlert from ambari_commons.urllib_handlers import RefreshHeaderProcessor from resource_management.libraries.functions.get_port_from_url import get_port_from_url from resource_management.libraries.functions.curl_krb_request import curl_krb_request +from ambari_agent import Constants logger = logging.getLogger() @@ -199,7 +200,7 @@ class MetricAlert(BaseAlert): content = '' try: if kerberos_principal is not None and kerberos_keytab is not None and security_enabled: - tmp_dir = self.config.get('agent', 'tmp_dir') + tmp_dir = Constants.AGENT_TMP_DIR if tmp_dir is None: tmp_dir = gettempdir() http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py index 76afbc9..e70dc22 100644 --- a/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py +++ b/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py @@ -25,6 +25,7 @@ import re from alerts.base_alert import BaseAlert from resource_management.core.environment import Environment from resource_management.core.logger import Logger +from ambari_agent import Constants logger = logging.getLogger(__name__) @@ -99,7 +100,7 @@ class ScriptAlert(BaseAlert): matchObj = re.match( r'((.*)services(.*)package)', self.path_to_script) if matchObj: basedir = matchObj.group(1) - with Environment(basedir, tmp_dir=self.config.get('agent', 'tmp_dir')) as env: + with Environment(basedir, tmp_dir=Constants.AGENT_TMP_DIR) as env: return cmd_module.execute(configurations, self.parameters, self.host_name) else: return cmd_module.execute(configurations, self.parameters, self.host_name) http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py index b76d5e0..0d627c3 100644 --- a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py +++ b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py @@ -31,6 +31,7 @@ from resource_management.libraries.functions.get_port_from_url import get_port_f from resource_management.libraries.functions.curl_krb_request import curl_krb_request from ambari_commons import OSCheck from ambari_commons.inet_utils import resolve_address +from ambari_agent import Constants # hashlib is supplied as of Python 2.5 as the replacement interface for md5 # and other secure hashes. In 2.6, md5 is deprecated. Import hashlib if @@ -171,7 +172,7 @@ class WebAlert(BaseAlert): # Create the kerberos credentials cache (ccache) file and set it in the environment to use # when executing curl. Use the md5 hash of the combination of the principal and keytab file # to generate a (relatively) unique cache filename so that we can use it as needed. - tmp_dir = self.config.get('agent', 'tmp_dir') + tmp_dir = Constants.AGENT_TMP_DIR if tmp_dir is None: tmp_dir = gettempdir() http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/main/python/ambari_agent/security.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/security.py b/ambari-agent/src/main/python/ambari_agent/security.py index bfaf134..dc1f141 100644 --- a/ambari-agent/src/main/python/ambari_agent/security.py +++ b/ambari-agent/src/main/python/ambari_agent/security.py @@ -35,6 +35,7 @@ logger = logging.getLogger(__name__) GEN_AGENT_KEY = 'openssl req -new -newkey rsa:1024 -nodes -keyout "%(keysdir)s' \ + os.sep + '%(hostname)s.key" -subj /OU=%(hostname)s/ ' \ '-out "%(keysdir)s' + os.sep + '%(hostname)s.csr"' +KEY_FILENAME = '%(hostname)s.key' class VerifiedHTTPSConnection(httplib.HTTPSConnection): @@ -190,7 +191,7 @@ class CertificateManager(): if not agent_key_exists: logger.info("Agent key not exists, generating request") - self.genAgentCrtReq() + self.genAgentCrtReq(self.getAgentKeyName()) else: logger.info("Agent key exists, ok") @@ -252,10 +253,12 @@ class CertificateManager(): '\nExiting..') raise ssl.SSLError - def genAgentCrtReq(self): + def genAgentCrtReq(self, keyname): + keysdir = os.path.abspath(self.config.get('security', 'keysdir')) generate_script = GEN_AGENT_KEY % { 'hostname': hostname.hostname(self.config), - 'keysdir': os.path.abspath(self.config.get('security', 'keysdir'))} + 'keysdir': keysdir} + logger.info(generate_script) if platform.system() == 'Windows': p = subprocess.Popen(generate_script, stdout=subprocess.PIPE) @@ -264,6 +267,8 @@ class CertificateManager(): p = subprocess.Popen([generate_script], shell=True, stdout=subprocess.PIPE) p.communicate() + # this is required to be 600 for security concerns. + os.chmod(keyname, 0600) def initSecurity(self): self.checkCertExists() http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py index 2461281..1cf7866 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py +++ b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py @@ -41,8 +41,10 @@ class TestCertGeneration(TestCase): config.set('security', 'server_crt', 'ca.crt') self.certMan = CertificateManager(config) - def test_generation(self): - self.certMan.genAgentCrtReq() + @patch.object(os, "chmod") + def test_generation(self, chmod_mock): + self.certMan.genAgentCrtReq('/dummy_dir/hostname.key') + self.assertTrue(chmod_mock.called) self.assertTrue(os.path.exists(self.certMan.getAgentKeyName())) self.assertTrue(os.path.exists(self.certMan.getAgentCrtReqName())) def tearDown(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py.orig ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py.orig b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py.orig new file mode 100644 index 0000000..2461281 --- /dev/null +++ b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py.orig @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +import os +import tempfile +import shutil +from unittest import TestCase + +from ambari_agent.security import CertificateManager +from ambari_agent import AmbariConfig +from mock.mock import patch, MagicMock +from ambari_commons import OSCheck +from only_for_platform import os_distro_value + +class TestCertGeneration(TestCase): + @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + config = AmbariConfig.AmbariConfig() + #config.add_section('server') + config.set('server', 'hostname', 'example.com') + config.set('server', 'url_port', '777') + #config.add_section('security') + config.set('security', 'keysdir', self.tmpdir) + config.set('security', 'server_crt', 'ca.crt') + self.certMan = CertificateManager(config) + + def test_generation(self): + self.certMan.genAgentCrtReq() + self.assertTrue(os.path.exists(self.certMan.getAgentKeyName())) + self.assertTrue(os.path.exists(self.certMan.getAgentCrtReqName())) + def tearDown(self): + shutil.rmtree(self.tmpdir) + http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py index 8957b44..a4025eb 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py +++ b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py @@ -56,7 +56,6 @@ class TestCustomServiceOrchestrator(TestCase): self.config = ConfigParser.RawConfigParser() self.config.add_section('agent') self.config.set('agent', 'prefix', tmpdir) - self.config.set('agent', 'tmp_dir', exec_tmp_dir) self.config.set('agent', 'cache_dir', "/cachedir") self.config.add_section('python') self.config.set('python', 'custom_actions_dir', tmpdir) http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py.orig ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py.orig b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py.orig new file mode 100644 index 0000000..8957b44 --- /dev/null +++ b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py.orig @@ -0,0 +1,643 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +import ConfigParser +from multiprocessing.pool import ThreadPool +import os + +import pprint +from ambari_commons import shell + +from unittest import TestCase +import threading +import tempfile +import time +from threading import Thread + +from mock.mock import MagicMock, patch +import StringIO +import sys + +from ambari_agent.ActionQueue import ActionQueue +from ambari_agent.AgentException import AgentException +from ambari_agent.AmbariConfig import AmbariConfig +from ambari_agent.BackgroundCommandExecutionHandle import BackgroundCommandExecutionHandle +from ambari_agent.CustomServiceOrchestrator import CustomServiceOrchestrator +from ambari_agent.FileCache import FileCache +from ambari_agent.PythonExecutor import PythonExecutor +from ambari_commons import OSCheck +from only_for_platform import get_platform, os_distro_value, PLATFORM_WINDOWS + +class TestCustomServiceOrchestrator(TestCase): + + def setUp(self): + # disable stdout + out = StringIO.StringIO() + sys.stdout = out + # generate sample config + tmpdir = tempfile.gettempdir() + exec_tmp_dir = os.path.join(tmpdir, 'tmp') + self.config = ConfigParser.RawConfigParser() + self.config.add_section('agent') + self.config.set('agent', 'prefix', tmpdir) + self.config.set('agent', 'tmp_dir', exec_tmp_dir) + self.config.set('agent', 'cache_dir', "/cachedir") + self.config.add_section('python') + self.config.set('python', 'custom_actions_dir', tmpdir) + + + @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) + @patch.object(FileCache, "__init__") + def test_add_reg_listener_to_controller(self, FileCache_mock): + FileCache_mock.return_value = None + dummy_controller = MagicMock() + config = AmbariConfig().getConfig() + tempdir = tempfile.gettempdir() + config.set('agent', 'prefix', tempdir) + CustomServiceOrchestrator(config, dummy_controller) + self.assertTrue(dummy_controller.registration_listeners.append.called) + + + @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) + @patch.object(CustomServiceOrchestrator, 'decompressClusterHostInfo') + @patch("ambari_agent.hostname.public_hostname") + @patch("os.path.isfile") + @patch("os.unlink") + @patch.object(FileCache, "__init__") + def test_dump_command_to_json(self, FileCache_mock, unlink_mock, + isfile_mock, hostname_mock, + decompress_cluster_host_info_mock): + FileCache_mock.return_value = None + hostname_mock.return_value = "test.hst" + command = { + 'commandType': 'EXECUTION_COMMAND', + 'role': u'DATANODE', + 'roleCommand': u'INSTALL', + 'commandId': '1-1', + 'taskId': 3, + 'clusterName': u'cc', + 'serviceName': u'HDFS', + 'configurations':{'global' : {}}, + 'configurationTags':{'global' : { 'tag': 'v1' }}, + 'clusterHostInfo':{'namenode_host' : ['1'], + 'slave_hosts' : ['0', '1'], + 'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'], + 'all_ping_ports': ['8670:0,1']}, + 'hostLevelParams':{} + } + + decompress_cluster_host_info_mock.return_value = {'namenode_host' : ['h2.hortonworks.com'], + 'slave_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'], + 'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'], + 'all_ping_ports': ['8670', '8670']} + + config = AmbariConfig().getConfig() + tempdir = tempfile.gettempdir() + config.set('agent', 'prefix', tempdir) + dummy_controller = MagicMock() + orchestrator = CustomServiceOrchestrator(config, dummy_controller) + isfile_mock.return_value = True + # Test dumping EXECUTION_COMMAND + json_file = orchestrator.dump_command_to_json(command) + self.assertTrue(os.path.exists(json_file)) + self.assertTrue(os.path.getsize(json_file) > 0) + if get_platform() != PLATFORM_WINDOWS: + self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600') + self.assertTrue(json_file.endswith("command-3.json")) + self.assertTrue(decompress_cluster_host_info_mock.called) + os.unlink(json_file) + # Test dumping STATUS_COMMAND + command['commandType']='STATUS_COMMAND' + decompress_cluster_host_info_mock.reset_mock() + json_file = orchestrator.dump_command_to_json(command) + self.assertTrue(os.path.exists(json_file)) + self.assertTrue(os.path.getsize(json_file) > 0) + if get_platform() != PLATFORM_WINDOWS: + self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600') + self.assertTrue(json_file.endswith("status_command.json")) + self.assertFalse(decompress_cluster_host_info_mock.called) + os.unlink(json_file) + # Testing side effect of dump_command_to_json + self.assertEquals(command['public_hostname'], "test.hst") + self.assertTrue(unlink_mock.called) + + + @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) + @patch("ambari_agent.hostname.public_hostname") + @patch("os.path.isfile") + @patch("os.unlink") + @patch.object(FileCache, "__init__") + def test_dump_command_to_json_with_retry(self, FileCache_mock, unlink_mock, + isfile_mock, hostname_mock): + FileCache_mock.return_value = None + hostname_mock.return_value = "test.hst" + command = { + 'commandType': 'EXECUTION_COMMAND', + 'role': u'DATANODE', + 'roleCommand': u'INSTALL', + 'commandId': '1-1', + 'taskId': 3, + 'clusterName': u'cc', + 'serviceName': u'HDFS', + 'configurations':{'global' : {}}, + 'configurationTags':{'global' : { 'tag': 'v1' }}, + 'clusterHostInfo':{'namenode_host' : ['1'], + 'slave_hosts' : ['0', '1'], + 'all_racks' : [u'/default-rack:0'], + 'ambari_server_host' : 'a.b.c', + 'all_ipv4_ips' : [u'192.168.12.101:0'], + 'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'], + 'all_ping_ports': ['8670:0,1']}, + 'hostLevelParams':{} + } + + config = AmbariConfig().getConfig() + tempdir = tempfile.gettempdir() + config.set('agent', 'prefix', tempdir) + dummy_controller = MagicMock() + orchestrator = CustomServiceOrchestrator(config, dummy_controller) + isfile_mock.return_value = True + # Test dumping EXECUTION_COMMAND + json_file = orchestrator.dump_command_to_json(command) + self.assertTrue(os.path.exists(json_file)) + self.assertTrue(os.path.getsize(json_file) > 0) + if get_platform() != PLATFORM_WINDOWS: + self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600') + self.assertTrue(json_file.endswith("command-3.json")) + os.unlink(json_file) + # Test dumping STATUS_COMMAND + json_file = orchestrator.dump_command_to_json(command, True) + self.assertTrue(os.path.exists(json_file)) + self.assertTrue(os.path.getsize(json_file) > 0) + if get_platform() != PLATFORM_WINDOWS: + self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600') + self.assertTrue(json_file.endswith("command-3.json")) + os.unlink(json_file) + # Testing side effect of dump_command_to_json + self.assertEquals(command['public_hostname'], "test.hst") + self.assertTrue(unlink_mock.called) + + @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) + @patch("os.path.exists") + @patch.object(FileCache, "__init__") + def test_resolve_script_path(self, FileCache_mock, exists_mock): + FileCache_mock.return_value = None + dummy_controller = MagicMock() + config = AmbariConfig().getConfig() + orchestrator = CustomServiceOrchestrator(config, dummy_controller) + # Testing existing path + exists_mock.return_value = True + path = orchestrator.\ + resolve_script_path(os.path.join("HBASE", "package"), os.path.join("scripts", "hbase_master.py")) + self.assertEqual(os.path.join("HBASE", "package", "scripts", "hbase_master.py"), path) + # Testing not existing path + exists_mock.return_value = False + try: + orchestrator.resolve_script_path("/HBASE", + os.path.join("scripts", "hbase_master.py")) + self.fail('ExpectedException not thrown') + except AgentException: + pass # Expected + + + @patch.object(CustomServiceOrchestrator, "resolve_script_path") + @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path") + @patch.object(FileCache, "get_host_scripts_base_dir") + @patch.object(FileCache, "get_service_base_dir") + @patch.object(FileCache, "get_hook_base_dir") + @patch.object(CustomServiceOrchestrator, "dump_command_to_json") + @patch.object(PythonExecutor, "run_file") + @patch.object(FileCache, "__init__") + def test_runCommand(self, FileCache_mock, + run_file_mock, dump_command_to_json_mock, + get_hook_base_dir_mock, get_service_base_dir_mock, + get_host_scripts_base_dir_mock, + resolve_hook_script_path_mock, + resolve_script_path_mock): + + FileCache_mock.return_value = None + command = { + 'role' : 'REGION_SERVER', + 'hostLevelParams' : { + 'stack_name' : 'HDP', + 'stack_version' : '2.0.7', + 'jdk_location' : 'some_location' + }, + 'commandParams': { + 'script_type': 'PYTHON', + 'script': 'scripts/hbase_regionserver.py', + 'command_timeout': '600', + 'service_package_folder' : 'HBASE' + }, + 'taskId' : '3', + 'roleCommand': 'INSTALL' + } + + get_host_scripts_base_dir_mock.return_value = "/host_scripts" + get_service_base_dir_mock.return_value = "/basedir/" + resolve_script_path_mock.return_value = "/basedir/scriptpath" + resolve_hook_script_path_mock.return_value = \ + ('/hooks_dir/prefix-command/scripts/hook.py', + '/hooks_dir/prefix-command') + dummy_controller = MagicMock() + orchestrator = CustomServiceOrchestrator(self.config, dummy_controller) + unix_process_id = 111 + orchestrator.commands_in_progress = {command['taskId']: unix_process_id} + get_hook_base_dir_mock.return_value = "/hooks/" + # normal run case + run_file_mock.return_value = { + 'stdout' : 'sss', + 'stderr' : 'eee', + 'exitcode': 0, + } + ret = orchestrator.runCommand(command, "out.txt", "err.txt") + self.assertEqual(ret['exitcode'], 0) + self.assertTrue(run_file_mock.called) + self.assertEqual(run_file_mock.call_count, 3) + + run_file_mock.reset_mock() + + # Case when we force another command + run_file_mock.return_value = { + 'stdout' : 'sss', + 'stderr' : 'eee', + 'exitcode': 0, + } + ret = orchestrator.runCommand(command, "out.txt", "err.txt", + forced_command_name=CustomServiceOrchestrator.SCRIPT_TYPE_PYTHON) + ## Check that override_output_files was true only during first call + print run_file_mock + self.assertEquals(run_file_mock.call_args_list[0][0][8], True) + self.assertEquals(run_file_mock.call_args_list[1][0][8], False) + self.assertEquals(run_file_mock.call_args_list[2][0][8], False) + ## Check that forced_command_name was taken into account + self.assertEqual(run_file_mock.call_args_list[0][0][1][0], + CustomServiceOrchestrator.SCRIPT_TYPE_PYTHON) + + run_file_mock.reset_mock() + + # unknown script type case + command['commandParams']['script_type'] = "SOME_TYPE" + ret = orchestrator.runCommand(command, "out.txt", "err.txt") + self.assertEqual(ret['exitcode'], 1) + self.assertFalse(run_file_mock.called) + self.assertTrue("Unknown script type" in ret['stdout']) + + #By default returns empty dictionary + self.assertEqual(ret['structuredOut'], '{}') + + pass + + @patch("ambari_commons.shell.kill_process_with_children") + @patch.object(CustomServiceOrchestrator, "resolve_script_path") + @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path") + @patch.object(FileCache, "get_host_scripts_base_dir") + @patch.object(FileCache, "get_service_base_dir") + @patch.object(FileCache, "get_hook_base_dir") + @patch.object(CustomServiceOrchestrator, "dump_command_to_json") + @patch.object(PythonExecutor, "run_file") + @patch.object(FileCache, "__init__") + def test_cancel_command(self, FileCache_mock, + run_file_mock, dump_command_to_json_mock, + get_hook_base_dir_mock, get_service_base_dir_mock, + get_host_scripts_base_dir_mock, + resolve_hook_script_path_mock, resolve_script_path_mock, + kill_process_with_children_mock): + FileCache_mock.return_value = None + command = { + 'role' : 'REGION_SERVER', + 'hostLevelParams' : { + 'stack_name' : 'HDP', + 'stack_version' : '2.0.7', + 'jdk_location' : 'some_location' + }, + 'commandParams': { + 'script_type': 'PYTHON', + 'script': 'scripts/hbase_regionserver.py', + 'command_timeout': '600', + 'service_package_folder' : 'HBASE' + }, + 'taskId' : '3', + 'roleCommand': 'INSTALL' + } + + get_host_scripts_base_dir_mock.return_value = "/host_scripts" + get_service_base_dir_mock.return_value = "/basedir/" + resolve_script_path_mock.return_value = "/basedir/scriptpath" + resolve_hook_script_path_mock.return_value = \ + ('/hooks_dir/prefix-command/scripts/hook.py', + '/hooks_dir/prefix-command') + dummy_controller = MagicMock() + orchestrator = CustomServiceOrchestrator(self.config, dummy_controller) + unix_process_id = 111 + orchestrator.commands_in_progress = {command['taskId']: unix_process_id} + get_hook_base_dir_mock.return_value = "/hooks/" + run_file_mock_return_value = { + 'stdout' : 'killed', + 'stderr' : 'killed', + 'exitcode': 1, + } + def side_effect(*args, **kwargs): + time.sleep(0.2) + return run_file_mock_return_value + run_file_mock.side_effect = side_effect + + _, out = tempfile.mkstemp() + _, err = tempfile.mkstemp() + pool = ThreadPool(processes=1) + async_result = pool.apply_async(orchestrator.runCommand, (command, out, err)) + + time.sleep(0.1) + orchestrator.cancel_command(command['taskId'], 'reason') + + ret = async_result.get() + + self.assertEqual(ret['exitcode'], 1) + self.assertEquals(ret['stdout'], 'killed\nCommand aborted. reason') + self.assertEquals(ret['stderr'], 'killed\nCommand aborted. reason') + + self.assertTrue(kill_process_with_children_mock.called) + self.assertFalse(command['taskId'] in orchestrator.commands_in_progress.keys()) + self.assertTrue(os.path.exists(out)) + self.assertTrue(os.path.exists(err)) + try: + os.remove(out) + os.remove(err) + except: + pass + + from ambari_agent.StackVersionsFileHandler import StackVersionsFileHandler + + @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) + @patch.object(CustomServiceOrchestrator, "get_py_executor") + @patch("ambari_commons.shell.kill_process_with_children") + @patch.object(FileCache, "__init__") + @patch.object(CustomServiceOrchestrator, "resolve_script_path") + @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path") + @patch.object(StackVersionsFileHandler, "read_stack_version") + def test_cancel_backgound_command(self, read_stack_version_mock, resolve_hook_script_path_mock, + resolve_script_path_mock, FileCache_mock, kill_process_with_children_mock, + get_py_executor_mock): + FileCache_mock.return_value = None + FileCache_mock.cache_dir = MagicMock() + resolve_hook_script_path_mock.return_value = None +# shell.kill_process_with_children = MagicMock() + dummy_controller = MagicMock() + cfg = AmbariConfig() + cfg.set('agent', 'tolerate_download_failures', 'true') + cfg.set('agent', 'prefix', '.') + cfg.set('agent', 'cache_dir', 'background_tasks') + + actionQueue = ActionQueue(cfg, dummy_controller) + + dummy_controller.actionQueue = actionQueue + orchestrator = CustomServiceOrchestrator(cfg, dummy_controller) + orchestrator.file_cache = MagicMock() + def f (a, b): + return "" + orchestrator.file_cache.get_service_base_dir = f + actionQueue.customServiceOrchestrator = orchestrator + + import TestActionQueue + import copy + + pyex = PythonExecutor(actionQueue.customServiceOrchestrator.tmp_dir, actionQueue.customServiceOrchestrator.config) + TestActionQueue.patch_output_file(pyex) + pyex.prepare_process_result = MagicMock() + get_py_executor_mock.return_value = pyex + orchestrator.dump_command_to_json = MagicMock() + + lock = threading.RLock() + complete_done = threading.Condition(lock) + + complete_was_called = {} + def command_complete_w(process_condenced_result, handle): + with lock: + complete_was_called['visited']= '' + complete_done.wait(3) + + actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(actionQueue.on_background_command_complete_callback, command_complete_w, None) + execute_command = copy.deepcopy(TestActionQueue.TestActionQueue.background_command) + actionQueue.put([execute_command]) + actionQueue.processBackgroundQueueSafeEmpty() + + time.sleep(.1) + + orchestrator.cancel_command(19,'') + self.assertTrue(kill_process_with_children_mock.called) + kill_process_with_children_mock.assert_called_with(33) + + with lock: + complete_done.notifyAll() + + with lock: + self.assertTrue(complete_was_called.has_key('visited')) + + time.sleep(.1) + + runningCommand = actionQueue.commandStatuses.get_command_status(19) + self.assertTrue(runningCommand is not None) + self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS) + + + @patch.object(CustomServiceOrchestrator, "dump_command_to_json") + @patch.object(PythonExecutor, "run_file") + @patch.object(FileCache, "__init__") + @patch.object(FileCache, "get_custom_actions_base_dir") + def test_runCommand_custom_action(self, get_custom_actions_base_dir_mock, + FileCache_mock, + run_file_mock, dump_command_to_json_mock): + FileCache_mock.return_value = None + get_custom_actions_base_dir_mock.return_value = "some path" + _, script = tempfile.mkstemp() + command = { + 'role' : 'any', + 'commandParams': { + 'script_type': 'PYTHON', + 'script': 'some_custom_action.py', + 'command_timeout': '600', + 'jdk_location' : 'some_location' + }, + 'taskId' : '3', + 'roleCommand': 'ACTIONEXECUTE' + } + dummy_controller = MagicMock() + orchestrator = CustomServiceOrchestrator(self.config, dummy_controller) + unix_process_id = 111 + orchestrator.commands_in_progress = {command['taskId']: unix_process_id} + # normal run case + run_file_mock.return_value = { + 'stdout' : 'sss', + 'stderr' : 'eee', + 'exitcode': 0, + } + ret = orchestrator.runCommand(command, "out.txt", "err.txt") + self.assertEqual(ret['exitcode'], 0) + self.assertTrue(run_file_mock.called) + # Hoooks are not supported for custom actions, + # that's why run_file() should be called only once + self.assertEqual(run_file_mock.call_count, 1) + + + @patch("os.path.isfile") + @patch.object(FileCache, "__init__") + def test_resolve_hook_script_path(self, FileCache_mock, isfile_mock): + FileCache_mock.return_value = None + dummy_controller = MagicMock() + orchestrator = CustomServiceOrchestrator(self.config, dummy_controller) + # Testing None param + res1 = orchestrator.resolve_hook_script_path(None, "prefix", "command", + "script_type") + self.assertEqual(res1, None) + # Testing existing hook script + isfile_mock.return_value = True + res2 = orchestrator.resolve_hook_script_path("hooks_dir", "prefix", "command", + "script_type") + self.assertEqual(res2, (os.path.join('hooks_dir', 'prefix-command', 'scripts', 'hook.py'), + os.path.join('hooks_dir', 'prefix-command'))) + # Testing not existing hook script + isfile_mock.return_value = False + res3 = orchestrator.resolve_hook_script_path("hooks_dir", "prefix", "command", + "script_type") + self.assertEqual(res3, None) + + + @patch.object(CustomServiceOrchestrator, "runCommand") + @patch.object(FileCache, "__init__") + def test_requestComponentStatus(self, FileCache_mock, runCommand_mock): + FileCache_mock.return_value = None + status_command = { + "serviceName" : 'HDFS', + "commandType" : "STATUS_COMMAND", + "clusterName" : "", + "componentName" : "DATANODE", + 'configurations':{} + } + dummy_controller = MagicMock() + orchestrator = CustomServiceOrchestrator(self.config, dummy_controller) + # Test alive case + runCommand_mock.return_value = { + "exitcode" : 0 + } + + status = orchestrator.requestComponentStatus(status_command) + self.assertEqual(runCommand_mock.return_value, status) + + # Test dead case + runCommand_mock.return_value = { + "exitcode" : 1 + } + status = orchestrator.requestComponentStatus(status_command) + self.assertEqual(runCommand_mock.return_value, status) + + @patch.object(CustomServiceOrchestrator, "runCommand") + @patch.object(FileCache, "__init__") + def test_requestComponentSecurityState(self, FileCache_mock, runCommand_mock): + FileCache_mock.return_value = None + status_command = { + "serviceName" : 'HDFS', + "commandType" : "STATUS_COMMAND", + "clusterName" : "", + "componentName" : "DATANODE", + 'configurations':{} + } + dummy_controller = MagicMock() + orchestrator = CustomServiceOrchestrator(self.config, dummy_controller) + # Test securityState + runCommand_mock.return_value = { + 'exitcode' : 0, + 'structuredOut' : {'securityState': 'UNSECURED'} + } + + status = orchestrator.requestComponentSecurityState(status_command) + self.assertEqual('UNSECURED', status) + + # Test case where exit code indicates failure + runCommand_mock.return_value = { + "exitcode" : 1 + } + status = orchestrator.requestComponentSecurityState(status_command) + self.assertEqual('UNKNOWN', status) + + @patch.object(FileCache, "__init__") + def test_requestComponentSecurityState_realFailure(self, FileCache_mock): + ''' + Tests the case where the CustomServiceOrchestrator attempts to call a service's security_status + method, but fails to do so because the script or method was not found. + :param FileCache_mock: + :return: + ''' + FileCache_mock.return_value = None + status_command = { + "serviceName" : 'BOGUS_SERVICE', + "commandType" : "STATUS_COMMAND", + "clusterName" : "", + "componentName" : "DATANODE", + 'configurations':{} + } + dummy_controller = MagicMock() + orchestrator = CustomServiceOrchestrator(self.config, dummy_controller) + + status = orchestrator.requestComponentSecurityState(status_command) + self.assertEqual('UNKNOWN', status) + + + @patch.object(CustomServiceOrchestrator, "get_py_executor") + @patch.object(CustomServiceOrchestrator, "dump_command_to_json") + @patch.object(FileCache, "__init__") + @patch.object(FileCache, "get_custom_actions_base_dir") + def test_runCommand_background_action(self, get_custom_actions_base_dir_mock, + FileCache_mock, + dump_command_to_json_mock, + get_py_executor_mock): + FileCache_mock.return_value = None + get_custom_actions_base_dir_mock.return_value = "some path" + _, script = tempfile.mkstemp() + command = { + 'role' : 'any', + 'commandParams': { + 'script_type': 'PYTHON', + 'script': 'some_custom_action.py', + 'command_timeout': '600', + 'jdk_location' : 'some_location' + }, + 'taskId' : '13', + 'roleCommand': 'ACTIONEXECUTE', + 'commandType': 'BACKGROUND_EXECUTION_COMMAND', + '__handle': BackgroundCommandExecutionHandle({'taskId': '13'}, 13, + MagicMock(), MagicMock()) + } + dummy_controller = MagicMock() + orchestrator = CustomServiceOrchestrator(self.config, dummy_controller) + + import TestActionQueue + pyex = PythonExecutor(orchestrator.tmp_dir, orchestrator.config) + TestActionQueue.patch_output_file(pyex) + pyex.condenseOutput = MagicMock() + get_py_executor_mock.return_value = pyex + orchestrator.dump_command_to_json = MagicMock() + + ret = orchestrator.runCommand(command, "out.txt", "err.txt") + self.assertEqual(ret['exitcode'], 777) + + def tearDown(self): + # enable stdout + sys.stdout = sys.__stdout__ + + http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestSecurity.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestSecurity.py b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py index 0e1e4ee..c1b7812 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestSecurity.py +++ b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py @@ -343,12 +343,14 @@ class TestSecurity(unittest.TestCase): @patch("subprocess.Popen") @patch("subprocess.Popen.communicate") - def test_genAgentCrtReq(self, communicate_mock, popen_mock): + @patch.object(os, "chmod") + def test_genAgentCrtReq(self, chmod_mock, communicate_mock, popen_mock): man = CertificateManager(self.config) p = MagicMock(spec=subprocess.Popen) p.communicate = communicate_mock popen_mock.return_value = p - man.genAgentCrtReq() + man.genAgentCrtReq('/dummy-keysdir/hostname.key') + self.assertTrue(chmod_mock.called) self.assertTrue(popen_mock.called) self.assertTrue(communicate_mock.called) http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-agent/src/test/python/ambari_agent/TestSecurity.py.orig ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestSecurity.py.orig b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py.orig new file mode 100644 index 0000000..0e1e4ee --- /dev/null +++ b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py.orig @@ -0,0 +1,389 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +import StringIO +import sys, subprocess +from mock.mock import MagicMock, patch, ANY +import mock.mock +import unittest +import logging +import signal +import ConfigParser +import ssl +import os +import tempfile + +from ambari_commons import OSCheck +from only_for_platform import os_distro_value + +with patch("platform.linux_distribution", return_value = ('Suse','11','Final')): + from ambari_agent import NetUtil + from ambari_agent.security import CertificateManager + from ambari_agent import ProcessHelper, main + from ambari_agent.AmbariConfig import AmbariConfig + from ambari_agent.Controller import Controller + from ambari_agent import security + +aa = mock.mock.mock_open() +class TestSecurity(unittest.TestCase): + + @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) + def setUp(self): + # disable stdout + out = StringIO.StringIO() + sys.stdout = out + # Create config + self.config = AmbariConfig() + # Instantiate CachedHTTPSConnection (skip connect() call) + with patch.object(security.VerifiedHTTPSConnection, "connect"): + self.cachedHTTPSConnection = security.CachedHTTPSConnection(self.config) + + + def tearDown(self): + # enable stdout + sys.stdout = sys.__stdout__ + + + ### VerifiedHTTPSConnection ### + + @patch.object(security.CertificateManager, "initSecurity") + @patch("socket.create_connection") + @patch("ssl.wrap_socket") + def test_VerifiedHTTPSConnection_connect(self, wrap_socket_mock, + create_connection_mock, + init_security_mock): + init_security_mock.return_value = None + self.config.set('security', 'keysdir', '/dummy-keysdir') + connection = security.VerifiedHTTPSConnection("example.com", + self.config.get('server', 'secured_url_port'), self.config) + connection._tunnel_host = False + connection.sock = None + connection.connect() + self.assertTrue(wrap_socket_mock.called) + + ### VerifiedHTTPSConnection with no certificates creation + @patch.object(security.CertificateManager, "initSecurity") + @patch("socket.create_connection") + @patch("ssl.wrap_socket") + def test_Verified_HTTPSConnection_non_secure_connect(self, wrap_socket_mock, + create_connection_mock, + init_security_mock): + connection = security.VerifiedHTTPSConnection("example.com", + self.config.get('server', 'secured_url_port'), self.config) + connection._tunnel_host = False + connection.sock = None + connection.connect() + self.assertFalse(init_security_mock.called) + + ### VerifiedHTTPSConnection with two-way SSL authentication enabled + @patch.object(security.CertificateManager, "initSecurity") + @patch("socket.create_connection") + @patch("ssl.wrap_socket") + def test_Verified_HTTPSConnection_two_way_ssl_connect(self, wrap_socket_mock, + create_connection_mock, + init_security_mock): + wrap_socket_mock.side_effect=ssl.SSLError() + connection = security.VerifiedHTTPSConnection("example.com", + self.config.get('server', 'secured_url_port'), self.config) + connection._tunnel_host = False + connection.sock = None + try: + connection.connect() + except ssl.SSLError: + pass + self.assertTrue(init_security_mock.called) + + ### CachedHTTPSConnection ### + + @patch.object(security.VerifiedHTTPSConnection, "connect") + def test_CachedHTTPSConnection_connect(self, vhc_connect_mock): + self.config.set('server', 'hostname', 'dummy.server.hostname') + self.config.set('server', 'secured_url_port', '443') + # Testing not connected case + self.cachedHTTPSConnection.connected = False + self.cachedHTTPSConnection.connect() + self.assertTrue(vhc_connect_mock.called) + vhc_connect_mock.reset_mock() + # Testing already connected case + self.cachedHTTPSConnection.connect() + self.assertFalse(vhc_connect_mock.called) + + + @patch.object(security.CachedHTTPSConnection, "connect") + def test_forceClear(self, connect_mock): + # Testing if httpsconn instance changed + old = self.cachedHTTPSConnection.httpsconn + self.cachedHTTPSConnection.forceClear() + self.assertNotEqual(old, self.cachedHTTPSConnection.httpsconn) + + + @patch.object(security.CachedHTTPSConnection, "connect") + def test_request(self, connect_mock): + httpsconn_mock = MagicMock(create = True) + self.cachedHTTPSConnection.httpsconn = httpsconn_mock + + dummy_request = MagicMock(create = True) + dummy_request.get_method.return_value = "dummy_get_method" + dummy_request.get_full_url.return_value = "dummy_full_url" + dummy_request.get_data.return_value = "dummy_get_data" + dummy_request.headers = "dummy_headers" + + responce_mock = MagicMock(create = True) + responce_mock.read.return_value = "dummy responce" + httpsconn_mock.getresponse.return_value = responce_mock + + # Testing normal case + responce = self.cachedHTTPSConnection.request(dummy_request) + + self.assertEqual(responce, responce_mock.read.return_value) + httpsconn_mock.request.assert_called_once_with( + dummy_request.get_method.return_value, + dummy_request.get_full_url.return_value, + dummy_request.get_data.return_value, + dummy_request.headers) + + # Testing case of exception + try: + def side_eff(): + raise Exception("Dummy exception") + httpsconn_mock.read.side_effect = side_eff + responce = self.cachedHTTPSConnection.request(dummy_request) + self.fail("Should raise IOError") + except Exception, err: + # Expected + pass + + + ### CertificateManager ### + + + @patch("ambari_agent.hostname.hostname") + def test_getAgentKeyName(self, hostname_mock): + hostname_mock.return_value = "dummy.hostname" + self.config.set('security', 'keysdir', '/dummy-keysdir') + man = CertificateManager(self.config) + res = man.getAgentKeyName() + self.assertEquals(res, os.path.abspath("/dummy-keysdir/dummy.hostname.key")) + + + @patch("ambari_agent.hostname.hostname") + def test_getAgentCrtName(self, hostname_mock): + hostname_mock.return_value = "dummy.hostname" + self.config.set('security', 'keysdir', '/dummy-keysdir') + man = CertificateManager(self.config) + res = man.getAgentCrtName() + self.assertEquals(res, os.path.abspath("/dummy-keysdir/dummy.hostname.crt")) + + + @patch("ambari_agent.hostname.hostname") + def test_getAgentCrtReqName(self, hostname_mock): + hostname_mock.return_value = "dummy.hostname" + self.config.set('security', 'keysdir', '/dummy-keysdir') + man = CertificateManager(self.config) + res = man.getAgentCrtReqName() + self.assertEquals(res, os.path.abspath("/dummy-keysdir/dummy.hostname.csr")) + + + def test_getSrvrCrtName(self): + self.config.set('security', 'keysdir', '/dummy-keysdir') + man = CertificateManager(self.config) + res = man.getSrvrCrtName() + self.assertEquals(res, os.path.abspath("/dummy-keysdir/ca.crt")) + + + @patch("os.path.exists") + @patch.object(security.CertificateManager, "loadSrvrCrt") + @patch.object(security.CertificateManager, "getAgentKeyName") + @patch.object(security.CertificateManager, "genAgentCrtReq") + @patch.object(security.CertificateManager, "getAgentCrtName") + @patch.object(security.CertificateManager, "reqSignCrt") + def test_checkCertExists(self, reqSignCrt_mock, getAgentCrtName_mock, + genAgentCrtReq_mock, getAgentKeyName_mock, + loadSrvrCrt_mock, exists_mock): + self.config.set('security', 'keysdir', '/dummy-keysdir') + getAgentKeyName_mock.return_value = "dummy AgentKeyName" + getAgentCrtName_mock.return_value = "dummy AgentCrtName" + man = CertificateManager(self.config) + + # Case when all files exist + exists_mock.side_effect = [True, True, True] + man.checkCertExists() + self.assertFalse(loadSrvrCrt_mock.called) + self.assertFalse(genAgentCrtReq_mock.called) + self.assertFalse(reqSignCrt_mock.called) + + # Absent server cert + exists_mock.side_effect = [False, True, True] + man.checkCertExists() + self.assertTrue(loadSrvrCrt_mock.called) + self.assertFalse(genAgentCrtReq_mock.called) + self.assertFalse(reqSignCrt_mock.called) + loadSrvrCrt_mock.reset_mock() + + # Absent agent key + exists_mock.side_effect = [True, False, True] + man.checkCertExists() + self.assertFalse(loadSrvrCrt_mock.called) + self.assertTrue(genAgentCrtReq_mock.called) + self.assertFalse(reqSignCrt_mock.called) + genAgentCrtReq_mock.reset_mock() + + # Absent agent cert + exists_mock.side_effect = [True, True, False] + man.checkCertExists() + self.assertFalse(loadSrvrCrt_mock.called) + self.assertFalse(genAgentCrtReq_mock.called) + self.assertTrue(reqSignCrt_mock.called) + reqSignCrt_mock.reset_mock() + + + + @patch("urllib2.OpenerDirector.open") + @patch.object(security.CertificateManager, "getSrvrCrtName") + def test_loadSrvrCrt(self, getSrvrCrtName_mock, urlopen_mock): + read_mock = MagicMock(create=True) + read_mock.read.return_value = "dummy_cert" + urlopen_mock.return_value = read_mock + _, tmpoutfile = tempfile.mkstemp() + getSrvrCrtName_mock.return_value = tmpoutfile + + man = CertificateManager(self.config) + man.loadSrvrCrt() + + # Checking file contents + saved = open(tmpoutfile, 'r').read() + self.assertEqual(saved, read_mock.read.return_value) + try: + os.unlink(tmpoutfile) + except: + pass + + + @patch("ambari_agent.hostname.hostname") + @patch('__builtin__.open', create=True, autospec=True) + @patch.dict('os.environ', {'DUMMY_PASSPHRASE': 'dummy-passphrase'}) + @patch('ambari_simplejson.dumps') + @patch('urllib2.Request') + @patch("urllib2.OpenerDirector.open") + @patch('ambari_simplejson.loads') + def test_reqSignCrt(self, loads_mock, urlopen_mock, request_mock, dumps_mock, open_mock, hostname_mock): + self.config.set('security', 'keysdir', '/dummy-keysdir') + self.config.set('security', 'passphrase_env_var_name', 'DUMMY_PASSPHRASE') + man = CertificateManager(self.config) + hostname_mock.return_value = "dummy-hostname" + + open_mock.return_value.read.return_value = "dummy_request" + urlopen_mock.return_value.read.return_value = "dummy_server_request" + loads_mock.return_value = { + 'result': 'OK', + 'signedCa': 'dummy-crt' + } + + # Test normal server interaction + man.reqSignCrt() + + self.assertEqual(dumps_mock.call_args[0][0], { + 'csr' : 'dummy_request', + 'passphrase' : 'dummy-passphrase' + }) + self.assertEqual(open_mock.return_value.write.call_args[0][0], 'dummy-crt') + + # Test negative server reply + dumps_mock.reset_mock() + open_mock.return_value.write.reset_mock() + loads_mock.return_value = { + 'result': 'FAIL', + 'signedCa': 'fail-crt' + } + + # If certificate signing failed, then exception must be raised + try: + man.reqSignCrt() + self.fail() + except ssl.SSLError: + pass + self.assertFalse(open_mock.return_value.write.called) + + # Test connection fail + dumps_mock.reset_mock() + open_mock.return_value.write.reset_mock() + + try: + man.reqSignCrt() + self.fail("Expected exception here") + except Exception, err: + # expected + pass + + # Test malformed JSON response + open_mock.return_value.write.reset_mock() + loads_mock.side_effect = Exception() + try: + man.reqSignCrt() + self.fail("Expected exception here") + except ssl.SSLError: + pass + self.assertFalse(open_mock.return_value.write.called) + + @patch("subprocess.Popen") + @patch("subprocess.Popen.communicate") + def test_genAgentCrtReq(self, communicate_mock, popen_mock): + man = CertificateManager(self.config) + p = MagicMock(spec=subprocess.Popen) + p.communicate = communicate_mock + popen_mock.return_value = p + man.genAgentCrtReq() + self.assertTrue(popen_mock.called) + self.assertTrue(communicate_mock.called) + + @patch("ambari_agent.hostname.hostname") + @patch('__builtin__.open', create=True, autospec=True) + @patch("urllib2.OpenerDirector.open") + @patch.dict('os.environ', {'DUMMY_PASSPHRASE': 'dummy-passphrase'}) + def test_reqSignCrt_malformedJson(self, urlopen_mock, open_mock, hostname_mock): + hostname_mock.return_value = "dummy-hostname" + open_mock.return_value.read.return_value = "dummy_request" + self.config.set('security', 'keysdir', '/dummy-keysdir') + self.config.set('security', 'passphrase_env_var_name', 'DUMMY_PASSPHRASE') + man = CertificateManager(self.config) + + # test valid JSON response + urlopen_mock.return_value.read.return_value = '{"result": "OK", "signedCa":"dummy"}' + try: + man.reqSignCrt() + except ssl.SSLError: + self.fail("Unexpected exception!") + open_mock.return_value.write.assert_called_with(u'dummy') + + # test malformed JSON response + open_mock.return_value.write.reset_mock() + urlopen_mock.return_value.read.return_value = '{malformed_object}' + try: + man.reqSignCrt() + self.fail("Expected exception!") + except ssl.SSLError: + pass + self.assertFalse(open_mock.return_value.write.called) + + @patch.object(security.CertificateManager, "checkCertExists") + def test_initSecurity(self, checkCertExists_method): + man = CertificateManager(self.config) + man.initSecurity() + self.assertTrue(checkCertExists_method.called) + http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 1ee5b70..e0a7877 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -61,7 +61,7 @@ USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEV <BASEDIR> path to service metadata dir. Ex: /var/lib/ambari-agent/cache/common-services/HDFS/2.1.0.2.0/package <STROUTPUT> path to file with structured command output (file will be created). Ex:/tmp/my.txt <LOGGING_LEVEL> log level for stdout. Ex:DEBUG,INFO -<TMP_DIR> temporary directory for executable scripts. Ex: /var/lib/ambari-agent/data/tmp +<TMP_DIR> temporary directory for executable scripts. Ex: /var/lib/ambari-agent/tmp """ _PASSWORD_MAP = {"/configurations/cluster-env/hadoop.user.name":"/configurations/cluster-env/hadoop.user.password"} http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-server/src/main/python/bootstrap.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/bootstrap.py b/ambari-server/src/main/python/bootstrap.py index faf873b..3eba75c 100755 --- a/ambari-server/src/main/python/bootstrap.py +++ b/ambari-server/src/main/python/bootstrap.py @@ -47,7 +47,7 @@ MAX_PARALLEL_BOOTSTRAPS = 20 # How many seconds to wait between polling parallel bootstraps POLL_INTERVAL_SEC = 1 DEBUG = False -DEFAULT_AGENT_TEMP_FOLDER = "/var/lib/ambari-agent/data/tmp" +DEFAULT_AGENT_TEMP_FOLDER = "/var/lib/ambari-agent/tmp" DEFAULT_AGENT_DATA_FOLDER = "/var/lib/ambari-agent/data" DEFAULT_AGENT_LIB_FOLDER = "/var/lib/ambari-agent" PYTHON_ENV="env PYTHONPATH=$PYTHONPATH:" + DEFAULT_AGENT_TEMP_FOLDER http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py index 1415367..a3c02a6 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py @@ -202,7 +202,7 @@ class NameNodeDefault(NameNode): # Create the kerberos credentials cache (ccache) file and set it in the environment to use # when executing HDFS rebalance command. Use the md5 hash of the combination of the principal and keytab file # to generate a (relatively) unique cache filename so that we can use it as needed. - # TODO: params.tmp_dir=/var/lib/ambari-agent/data/tmp. However hdfs user doesn't have access to this path. + # TODO: params.tmp_dir=/var/lib/ambari-agent/tmp. However hdfs user doesn't have access to this path. # TODO: Hence using /tmp ccache_file_name = "hdfs_rebalance_cc_" + _md5(format("{hdfs_principal_name}|{hdfs_user_keytab}")).hexdigest() ccache_file_path = os.path.join(tempfile.gettempdir(), ccache_file_name) http://git-wip-us.apache.org/repos/asf/ambari/blob/b8388e12/ambari-server/src/test/python/TestBootstrap.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestBootstrap.py b/ambari-server/src/test/python/TestBootstrap.py index 792d99d..e143a68 100644 --- a/ambari-server/src/test/python/TestBootstrap.py +++ b/ambari-server/src/test/python/TestBootstrap.py @@ -112,8 +112,8 @@ class TestBootstrap(TestCase): utime = 1234 bootstrap_obj.getUtime = MagicMock(return_value=utime) ret = bootstrap_obj.getRunSetupWithPasswordCommand("hostname") - expected = "sudo -S python /var/lib/ambari-agent/data/tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \ - "ambariServer root 8440 < /var/lib/ambari-agent/data/tmp/host_pass{0}".format(utime) + expected = "sudo -S python /var/lib/ambari-agent/tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \ + "ambariServer root 8440 < /var/lib/ambari-agent/tmp/host_pass{0}".format(utime) self.assertEquals(ret, expected) @@ -353,11 +353,11 @@ class TestBootstrap(TestCase): self.assertEquals(res, expected) command = str(init_mock.call_args[0][3]) self.assertEqual(command, - "sudo mkdir -p /var/lib/ambari-agent/data/tmp ; " - "sudo chown -R root /var/lib/ambari-agent/data/tmp ; " + "sudo mkdir -p /var/lib/ambari-agent/tmp ; " + "sudo chown -R root /var/lib/ambari-agent/tmp ; " "sudo chmod 755 /var/lib/ambari-agent ; " "sudo chmod 755 /var/lib/ambari-agent/data ; " - "sudo chmod 777 /var/lib/ambari-agent/data/tmp") + "sudo chmod 777 /var/lib/ambari-agent/tmp") @patch.object(BootstrapDefault, "getOsCheckScript") @patch.object(BootstrapDefault, "getOsCheckScriptRemoteLocation") @@ -520,7 +520,7 @@ class TestBootstrap(TestCase): command = str(init_mock.call_args[0][3]) self.assertEqual(command, "chmod a+x OsCheckScriptRemoteLocation && " - "env PYTHONPATH=$PYTHONPATH:/var/lib/ambari-agent/data/tmp OsCheckScriptRemoteLocation centos6") + "env PYTHONPATH=$PYTHONPATH:/var/lib/ambari-agent/tmp OsCheckScriptRemoteLocation centos6") @patch.object(SSH, "__init__")
