Repository: ambari Updated Branches: refs/heads/branch-2.1 cba000d2d -> ccc200e33
Verify if restricting acls on /var/lib/ambari-agent/data will be OK (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ccc200e3 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ccc200e3 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ccc200e3 Branch: refs/heads/branch-2.1 Commit: ccc200e3345e06b23644e20de74b51476b4cac8b Parents: cba000d Author: Andrew Onishuk <[email protected]> Authored: Mon Sep 14 14:59:53 2015 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Mon Sep 14 15:00:23 2015 +0300 ---------------------------------------------------------------------- ambari-agent/conf/unix/ambari-agent.ini | 1 - ambari-agent/conf/unix/install-helper.sh | 5 +++-- ambari-agent/pom.xml | 4 ++-- .../src/main/python/ambari_agent/Constants.py | 21 ++++++++++++++++++++ .../ambari_agent/CustomServiceOrchestrator.py | 3 ++- .../python/ambari_agent/alerts/metric_alert.py | 3 ++- .../python/ambari_agent/alerts/script_alert.py | 3 ++- .../python/ambari_agent/alerts/web_alert.py | 3 ++- .../src/main/python/ambari_agent/security.py | 11 +++++++--- .../python/ambari_agent/TestCertGeneration.py | 6 ++++-- .../TestCustomServiceOrchestrator.py | 1 - .../test/python/ambari_agent/TestSecurity.py | 6 ++++-- .../libraries/script/script.py | 2 +- ambari-server/src/main/python/bootstrap.py | 2 +- .../HDFS/2.1.0.2.0/package/scripts/namenode.py | 2 +- ambari-server/src/test/python/TestBootstrap.py | 12 +++++------ 16 files changed, 59 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/conf/unix/ambari-agent.ini ---------------------------------------------------------------------- diff --git a/ambari-agent/conf/unix/ambari-agent.ini b/ambari-agent/conf/unix/ambari-agent.ini index abfde62..3b7631c 100644 --- a/ambari-agent/conf/unix/ambari-agent.ini +++ b/ambari-agent/conf/unix/ambari-agent.ini @@ -19,7 +19,6 @@ secured_url_port=8441 [agent] prefix=/var/lib/ambari-agent/data -tmp_dir=/var/lib/ambari-agent/data/tmp ;loglevel=(DEBUG/INFO) loglevel=INFO data_cleanup_interval=86400 http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/conf/unix/install-helper.sh ---------------------------------------------------------------------- diff --git a/ambari-agent/conf/unix/install-helper.sh b/ambari-agent/conf/unix/install-helper.sh index 48391d5..35c67fb 100644 --- a/ambari-agent/conf/unix/install-helper.sh +++ b/ambari-agent/conf/unix/install-helper.sh @@ -58,8 +58,9 @@ do_install(){ # on nano Ubuntu, when umask=027 those folders are created without 'x' bit for 'others'. # which causes failures when hadoop users try to access tmp_dir chmod a+x /var/lib/ambari-agent - chmod a+x /var/lib/ambari-agent/data - chmod 777 /var/lib/ambari-agent/data/tmp + + chmod 777 /var/lib/ambari-agent/tmp + chmod 700 /var/lib/ambari-agent/data } do_remove(){ http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-agent/pom.xml b/ambari-agent/pom.xml index c2bee4a..b544d75 100644 --- a/ambari-agent/pom.xml +++ b/ambari-agent/pom.xml @@ -391,7 +391,7 @@ <groupname>root</groupname> </mapping> <mapping> - <directory>/var/lib/${project.artifactId}/data/tmp</directory> + <directory>/var/lib/${project.artifactId}/tmp</directory> <filemode>777</filemode> <username>root</username> <groupname>root</groupname> @@ -596,7 +596,7 @@ <paths> <path>${package.pid.dir}</path> <path>/var/lib/${project.artifactId}/data</path> - <path>/var/lib/${project.artifactId}/data/tmp</path> + <path>/var/lib/${project.artifactId}/tmp</path> <path>/var/lib/${project.artifactId}/keys</path> <path>${package.log.dir}</path> <path>/var/lib/${project.artifactId}/lib</path> http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/main/python/ambari_agent/Constants.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/Constants.py b/ambari-agent/src/main/python/ambari_agent/Constants.py new file mode 100644 index 0000000..6b80f00 --- /dev/null +++ b/ambari-agent/src/main/python/ambari_agent/Constants.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +AGENT_TMP_DIR = "/var/lib/ambari-agent/tmp" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py index 6ee929c..6c1a161 100644 --- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py +++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py @@ -29,6 +29,7 @@ from FileCache import FileCache from AgentException import AgentException from PythonExecutor import PythonExecutor from PythonReflectiveExecutor import PythonReflectiveExecutor +import Constants import hostname @@ -61,7 +62,7 @@ class CustomServiceOrchestrator(): def __init__(self, config, controller): self.config = config self.tmp_dir = config.get('agent', 'prefix') - self.exec_tmp_dir = config.get('agent', 'tmp_dir') + self.exec_tmp_dir = Constants.AGENT_TMP_DIR self.file_cache = FileCache(config) self.status_commands_stdout = os.path.join(self.tmp_dir, 'status_command_stdout.txt') http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py index aa4ad75..8de49cd 100644 --- a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py +++ b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py @@ -30,6 +30,7 @@ from alerts.base_alert import BaseAlert from ambari_commons.urllib_handlers import RefreshHeaderProcessor from resource_management.libraries.functions.get_port_from_url import get_port_from_url from resource_management.libraries.functions.curl_krb_request import curl_krb_request +from ambari_agent import Constants logger = logging.getLogger() @@ -199,7 +200,7 @@ class MetricAlert(BaseAlert): content = '' try: if kerberos_principal is not None and kerberos_keytab is not None and security_enabled: - tmp_dir = self.config.get('agent', 'tmp_dir') + tmp_dir = Constants.AGENT_TMP_DIR if tmp_dir is None: tmp_dir = gettempdir() http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py index 76afbc9..e70dc22 100644 --- a/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py +++ b/ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py @@ -25,6 +25,7 @@ import re from alerts.base_alert import BaseAlert from resource_management.core.environment import Environment from resource_management.core.logger import Logger +from ambari_agent import Constants logger = logging.getLogger(__name__) @@ -99,7 +100,7 @@ class ScriptAlert(BaseAlert): matchObj = re.match( r'((.*)services(.*)package)', self.path_to_script) if matchObj: basedir = matchObj.group(1) - with Environment(basedir, tmp_dir=self.config.get('agent', 'tmp_dir')) as env: + with Environment(basedir, tmp_dir=Constants.AGENT_TMP_DIR) as env: return cmd_module.execute(configurations, self.parameters, self.host_name) else: return cmd_module.execute(configurations, self.parameters, self.host_name) http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py index b76d5e0..0d627c3 100644 --- a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py +++ b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py @@ -31,6 +31,7 @@ from resource_management.libraries.functions.get_port_from_url import get_port_f from resource_management.libraries.functions.curl_krb_request import curl_krb_request from ambari_commons import OSCheck from ambari_commons.inet_utils import resolve_address +from ambari_agent import Constants # hashlib is supplied as of Python 2.5 as the replacement interface for md5 # and other secure hashes. In 2.6, md5 is deprecated. Import hashlib if @@ -171,7 +172,7 @@ class WebAlert(BaseAlert): # Create the kerberos credentials cache (ccache) file and set it in the environment to use # when executing curl. Use the md5 hash of the combination of the principal and keytab file # to generate a (relatively) unique cache filename so that we can use it as needed. - tmp_dir = self.config.get('agent', 'tmp_dir') + tmp_dir = Constants.AGENT_TMP_DIR if tmp_dir is None: tmp_dir = gettempdir() http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/main/python/ambari_agent/security.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/security.py b/ambari-agent/src/main/python/ambari_agent/security.py index bfaf134..dc1f141 100644 --- a/ambari-agent/src/main/python/ambari_agent/security.py +++ b/ambari-agent/src/main/python/ambari_agent/security.py @@ -35,6 +35,7 @@ logger = logging.getLogger(__name__) GEN_AGENT_KEY = 'openssl req -new -newkey rsa:1024 -nodes -keyout "%(keysdir)s' \ + os.sep + '%(hostname)s.key" -subj /OU=%(hostname)s/ ' \ '-out "%(keysdir)s' + os.sep + '%(hostname)s.csr"' +KEY_FILENAME = '%(hostname)s.key' class VerifiedHTTPSConnection(httplib.HTTPSConnection): @@ -190,7 +191,7 @@ class CertificateManager(): if not agent_key_exists: logger.info("Agent key not exists, generating request") - self.genAgentCrtReq() + self.genAgentCrtReq(self.getAgentKeyName()) else: logger.info("Agent key exists, ok") @@ -252,10 +253,12 @@ class CertificateManager(): '\nExiting..') raise ssl.SSLError - def genAgentCrtReq(self): + def genAgentCrtReq(self, keyname): + keysdir = os.path.abspath(self.config.get('security', 'keysdir')) generate_script = GEN_AGENT_KEY % { 'hostname': hostname.hostname(self.config), - 'keysdir': os.path.abspath(self.config.get('security', 'keysdir'))} + 'keysdir': keysdir} + logger.info(generate_script) if platform.system() == 'Windows': p = subprocess.Popen(generate_script, stdout=subprocess.PIPE) @@ -264,6 +267,8 @@ class CertificateManager(): p = subprocess.Popen([generate_script], shell=True, stdout=subprocess.PIPE) p.communicate() + # this is required to be 600 for security concerns. + os.chmod(keyname, 0600) def initSecurity(self): self.checkCertExists() http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py index d188dbd..be72588 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py +++ b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py @@ -47,8 +47,10 @@ class TestCertGeneration(TestCase): config.set('security', 'server_crt', 'ca.crt') self.certMan = CertificateManager(config) - def test_generation(self): - self.certMan.genAgentCrtReq() + @patch.object(os, "chmod") + def test_generation(self, chmod_mock): + self.certMan.genAgentCrtReq('/dummy_dir/hostname.key') + self.assertTrue(chmod_mock.called) self.assertTrue(os.path.exists(self.certMan.getAgentKeyName())) self.assertTrue(os.path.exists(self.certMan.getAgentCrtReqName())) def tearDown(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py index 831ecce..e08e2f7 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py +++ b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py @@ -61,7 +61,6 @@ class TestCustomServiceOrchestrator(TestCase): self.config = ConfigParser.RawConfigParser() self.config.add_section('agent') self.config.set('agent', 'prefix', tmpdir) - self.config.set('agent', 'tmp_dir', exec_tmp_dir) self.config.set('agent', 'cache_dir', "/cachedir") self.config.add_section('python') self.config.set('python', 'custom_actions_dir', tmpdir) http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-agent/src/test/python/ambari_agent/TestSecurity.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestSecurity.py b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py index c47172a..a1d04e1 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestSecurity.py +++ b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py @@ -348,12 +348,14 @@ class TestSecurity(unittest.TestCase): @patch("subprocess.Popen") @patch("subprocess.Popen.communicate") - def test_genAgentCrtReq(self, communicate_mock, popen_mock): + @patch.object(os, "chmod") + def test_genAgentCrtReq(self, chmod_mock, communicate_mock, popen_mock): man = CertificateManager(self.config) p = MagicMock(spec=subprocess.Popen) p.communicate = communicate_mock popen_mock.return_value = p - man.genAgentCrtReq() + man.genAgentCrtReq('/dummy-keysdir/hostname.key') + self.assertTrue(chmod_mock.called) self.assertTrue(popen_mock.called) self.assertTrue(communicate_mock.called) http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index a2c0c45..b4731a0 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -61,7 +61,7 @@ USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEV <BASEDIR> path to service metadata dir. Ex: /var/lib/ambari-agent/cache/common-services/HDFS/2.1.0.2.0/package <STROUTPUT> path to file with structured command output (file will be created). Ex:/tmp/my.txt <LOGGING_LEVEL> log level for stdout. Ex:DEBUG,INFO -<TMP_DIR> temporary directory for executable scripts. Ex: /var/lib/ambari-agent/data/tmp +<TMP_DIR> temporary directory for executable scripts. Ex: /var/lib/ambari-agent/tmp """ _PASSWORD_MAP = {"/configurations/cluster-env/hadoop.user.name":"/configurations/cluster-env/hadoop.user.password"} http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-server/src/main/python/bootstrap.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/bootstrap.py b/ambari-server/src/main/python/bootstrap.py index 98a3a93..adde438 100755 --- a/ambari-server/src/main/python/bootstrap.py +++ b/ambari-server/src/main/python/bootstrap.py @@ -47,7 +47,7 @@ MAX_PARALLEL_BOOTSTRAPS = 20 # How many seconds to wait between polling parallel bootstraps POLL_INTERVAL_SEC = 1 DEBUG = False -DEFAULT_AGENT_TEMP_FOLDER = "/var/lib/ambari-agent/data/tmp" +DEFAULT_AGENT_TEMP_FOLDER = "/var/lib/ambari-agent/tmp" DEFAULT_AGENT_DATA_FOLDER = "/var/lib/ambari-agent/data" DEFAULT_AGENT_LIB_FOLDER = "/var/lib/ambari-agent" PYTHON_ENV="env PYTHONPATH=$PYTHONPATH:" + DEFAULT_AGENT_TEMP_FOLDER http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py index 1415367..a3c02a6 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py @@ -202,7 +202,7 @@ class NameNodeDefault(NameNode): # Create the kerberos credentials cache (ccache) file and set it in the environment to use # when executing HDFS rebalance command. Use the md5 hash of the combination of the principal and keytab file # to generate a (relatively) unique cache filename so that we can use it as needed. - # TODO: params.tmp_dir=/var/lib/ambari-agent/data/tmp. However hdfs user doesn't have access to this path. + # TODO: params.tmp_dir=/var/lib/ambari-agent/tmp. However hdfs user doesn't have access to this path. # TODO: Hence using /tmp ccache_file_name = "hdfs_rebalance_cc_" + _md5(format("{hdfs_principal_name}|{hdfs_user_keytab}")).hexdigest() ccache_file_path = os.path.join(tempfile.gettempdir(), ccache_file_name) http://git-wip-us.apache.org/repos/asf/ambari/blob/ccc200e3/ambari-server/src/test/python/TestBootstrap.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestBootstrap.py b/ambari-server/src/test/python/TestBootstrap.py index 1fcb3ad..633ee39 100644 --- a/ambari-server/src/test/python/TestBootstrap.py +++ b/ambari-server/src/test/python/TestBootstrap.py @@ -112,8 +112,8 @@ class TestBootstrap(TestCase): utime = 1234 bootstrap_obj.getUtime = MagicMock(return_value=utime) ret = bootstrap_obj.getRunSetupWithPasswordCommand("hostname") - expected = "sudo -S python /var/lib/ambari-agent/data/tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \ - "ambariServer root 8440 < /var/lib/ambari-agent/data/tmp/host_pass{0}".format(utime) + expected = "sudo -S python /var/lib/ambari-agent/tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \ + "ambariServer root 8440 < /var/lib/ambari-agent/tmp/host_pass{0}".format(utime) self.assertEquals(ret, expected) @@ -353,11 +353,11 @@ class TestBootstrap(TestCase): self.assertEquals(res, expected) command = str(init_mock.call_args[0][3]) self.assertEqual(command, - "sudo mkdir -p /var/lib/ambari-agent/data/tmp ; " - "sudo chown -R root /var/lib/ambari-agent/data/tmp ; " + "sudo mkdir -p /var/lib/ambari-agent/tmp ; " + "sudo chown -R root /var/lib/ambari-agent/tmp ; " "sudo chmod 755 /var/lib/ambari-agent ; " "sudo chmod 755 /var/lib/ambari-agent/data ; " - "sudo chmod 777 /var/lib/ambari-agent/data/tmp") + "sudo chmod 777 /var/lib/ambari-agent/tmp") @patch.object(BootstrapDefault, "getOsCheckScript") @patch.object(BootstrapDefault, "getOsCheckScriptRemoteLocation") @@ -491,7 +491,7 @@ class TestBootstrap(TestCase): command = str(init_mock.call_args[0][3]) self.assertEqual(command, "chmod a+x OsCheckScriptRemoteLocation && " - "env PYTHONPATH=$PYTHONPATH:/var/lib/ambari-agent/data/tmp OsCheckScriptRemoteLocation centos6") + "env PYTHONPATH=$PYTHONPATH:/var/lib/ambari-agent/tmp OsCheckScriptRemoteLocation centos6") @patch.object(SSH, "__init__")
