Repository: ambari Updated Branches: refs/heads/branch-2.1 6ab25bbd0 -> ec0f771db
AMBARI-12072 Ambari Upgrade: 2.0.1 -> 2.1.0 fails with encrypt pwds option (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ec0f771d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ec0f771d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ec0f771d Branch: refs/heads/branch-2.1 Commit: ec0f771db19293d9622573eb6c085a6d07fc4a7e Parents: 6ab25bb Author: Dmytro Sen <[email protected]> Authored: Mon Jun 22 20:16:29 2015 +0300 Committer: Dmytro Sen <[email protected]> Committed: Mon Jun 22 20:19:29 2015 +0300 ---------------------------------------------------------------------- .../main/python/ambari_server/serverUpgrade.py | 10 ++- .../main/python/ambari_server/setupSecurity.py | 75 ++++++++++++++++- .../src/main/python/ambari_server_main.py | 85 +++----------------- .../src/test/python/TestAmbariServer.py | 52 +++++++++--- 4 files changed, 132 insertions(+), 90 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ec0f771d/ambari-server/src/main/python/ambari_server/serverUpgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverUpgrade.py b/ambari-server/src/main/python/ambari_server/serverUpgrade.py index 12a82fa..6315f9a 100644 --- a/ambari-server/src/main/python/ambari_server/serverUpgrade.py +++ b/ambari-server/src/main/python/ambari_server/serverUpgrade.py @@ -36,7 +36,8 @@ from ambari_server.serverConfiguration import configDefaults, \ update_database_name_property, get_admin_views_dir, \ AMBARI_PROPERTIES_FILE, IS_LDAP_CONFIGURED, LDAP_PRIMARY_URL_PROPERTY, RESOURCES_DIR_PROPERTY, \ SETUP_OR_UPGRADE_MSG -from ambari_server.setupSecurity import adjust_directory_permissions +from ambari_server.setupSecurity import adjust_directory_permissions, \ + generate_env, ensure_can_start_under_current_user from ambari_server.utils import compare_versions from ambari_server.serverUtils import is_server_runing, get_ambari_server_api_base from ambari_server.userInput import get_validated_string_input, get_prompt_default, read_password, get_YN_input @@ -223,7 +224,12 @@ def run_schema_upgrade(): print 'Upgrading database schema' command = SCHEMA_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath()) - (retcode, stdout, stderr) = run_os_command(command) + + ambari_user = read_ambari_user() + current_user = ensure_can_start_under_current_user(ambari_user) + environ = generate_env(ambari_user, current_user) + + (retcode, stdout, stderr) = run_os_command(command, env=environ) print_info_msg("Return code from schema upgrade command, retcode = " + str(retcode)) if retcode > 0: print_error_msg("Error executing schema upgrade, please check the server logs.") http://git-wip-us.apache.org/repos/asf/ambari/blob/ec0f771d/ambari-server/src/main/python/ambari_server/setupSecurity.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/setupSecurity.py b/ambari-server/src/main/python/ambari_server/setupSecurity.py index 1bc2222..6bb8e4b 100644 --- a/ambari-server/src/main/python/ambari_server/setupSecurity.py +++ b/ambari-server/src/main/python/ambari_server/setupSecurity.py @@ -19,6 +19,9 @@ limitations under the License. ''' import base64 import fileinput +import getpass +import stat +import tempfile import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. import os import re @@ -45,7 +48,7 @@ from ambari_server.serverConfiguration import configDefaults, \ SECURITY_PROVIDER_KEY_CMD, SECURITY_MASTER_KEY_FILENAME, SSL_TRUSTSTORE_PASSWORD_ALIAS, \ SSL_TRUSTSTORE_PASSWORD_PROPERTY, SSL_TRUSTSTORE_PATH_PROPERTY, SSL_TRUSTSTORE_TYPE_PROPERTY, \ SSL_API, SSL_API_PORT, DEFAULT_SSL_API_PORT, CLIENT_API_PORT, JDK_NAME_PROPERTY, JCE_NAME_PROPERTY, JAVA_HOME_PROPERTY, \ - get_resources_location + get_resources_location, SECURITY_MASTER_KEY_LOCATION, SETUP_OR_UPGRADE_MSG from ambari_server.serverUtils import is_server_runing, get_ambari_server_api_base from ambari_server.setupActions import SETUP_ACTION, LDAP_SETUP_ACTION from ambari_server.userInput import get_validated_string_input, get_prompt_default, read_password, get_YN_input @@ -703,3 +706,73 @@ def setup_ldap(): print 'Saving...done' return 0 + +@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT) +def generate_env(ambari_user, current_user): + properties = get_ambari_properties() + isSecure = get_is_secure(properties) + (isPersisted, masterKeyFile) = get_is_persisted(properties) + environ = os.environ.copy() + # Need to handle master key not persisted scenario + if isSecure and not masterKeyFile: + prompt = False + masterKey = environ.get(SECURITY_KEY_ENV_VAR_NAME) + + if masterKey is not None and masterKey != "": + pass + else: + keyLocation = environ.get(SECURITY_MASTER_KEY_LOCATION) + + if keyLocation is not None: + try: + # Verify master key can be read by the java process + with open(keyLocation, 'r'): + pass + except IOError: + print_warning_msg("Cannot read Master key from path specified in " + "environemnt.") + prompt = True + else: + # Key not provided in the environment + prompt = True + + if prompt: + import pwd + + masterKey = get_original_master_key(properties) + tempDir = tempfile.gettempdir() + tempFilePath = tempDir + os.sep + "masterkey" + save_master_key(masterKey, tempFilePath, True) + if ambari_user != current_user: + uid = pwd.getpwnam(ambari_user).pw_uid + gid = pwd.getpwnam(ambari_user).pw_gid + os.chown(tempFilePath, uid, gid) + else: + os.chmod(tempFilePath, stat.S_IREAD | stat.S_IWRITE) + + if tempFilePath is not None: + environ[SECURITY_MASTER_KEY_LOCATION] = tempFilePath + + return environ + +@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) +def generate_env(ambari_user, current_user): + return os.environ.copy() + +@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) +def ensure_can_start_under_current_user(ambari_user): + #Ignore the requirement to run as root. In Windows, by default the child process inherits the security context + # and the environment from the parent process. + return "" + +@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT) +def ensure_can_start_under_current_user(ambari_user): + current_user = getpass.getuser() + if ambari_user is None: + err = "Unable to detect a system user for Ambari Server.\n" + SETUP_OR_UPGRADE_MSG + raise FatalException(1, err) + if current_user != ambari_user and not is_root(): + err = "Unable to start Ambari Server as user {0}. Please either run \"ambari-server start\" " \ + "command as root, as sudo or as user \"{1}\"".format(current_user, ambari_user) + raise FatalException(1, err) + return current_user http://git-wip-us.apache.org/repos/asf/ambari/blob/ec0f771d/ambari-server/src/main/python/ambari_server_main.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server_main.py b/ambari-server/src/main/python/ambari_server_main.py index 35201cd..bc561a1 100644 --- a/ambari-server/src/main/python/ambari_server_main.py +++ b/ambari-server/src/main/python/ambari_server_main.py @@ -17,11 +17,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ''' -import getpass import os -import stat import subprocess -import tempfile import sys from ambari_commons.exceptions import FatalException @@ -37,7 +34,8 @@ from ambari_server.serverConfiguration import configDefaults, find_jdk, get_amba SETUP_OR_UPGRADE_MSG, check_database_name_property, parse_properties_file from ambari_server.serverUtils import refresh_stack_hash from ambari_server.setupHttps import get_fqdn -from ambari_server.setupSecurity import save_master_key +from ambari_server.setupSecurity import generate_env, \ + ensure_can_start_under_current_user from ambari_server.utils import check_reverse_lookup, save_pid, locate_file, looking_for_pid, wait_for_pid, \ save_main_pid_ex, check_exitcode @@ -107,26 +105,6 @@ AMBARI_SERVER_DIE_MSG = "Ambari Server java process died with exitcode {0}. Chec ULIMIT_OPEN_FILES_KEY = 'ulimit.open.files' ULIMIT_OPEN_FILES_DEFAULT = 10000 - -@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) -def ensure_can_start_under_current_user(ambari_user): - #Ignore the requirement to run as root. In Windows, by default the child process inherits the security context - # and the environment from the parent process. - return "" - -@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT) -def ensure_can_start_under_current_user(ambari_user): - current_user = getpass.getuser() - if ambari_user is None: - err = "Unable to detect a system user for Ambari Server.\n" + SETUP_OR_UPGRADE_MSG - raise FatalException(1, err) - if current_user != ambari_user and not is_root(): - err = "Unable to start Ambari Server as user {0}. Please either run \"ambari-server start\" " \ - "command as root, as sudo or as user \"{1}\"".format(current_user, ambari_user) - raise FatalException(1, err) - return current_user - - @OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) def ensure_server_security_is_configured(): pass @@ -144,7 +122,7 @@ def get_ulimit_open_files(properties): return open_files @OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) -def generate_child_process_param_list(ambari_user, current_user, java_exe, class_path, debug_start, suspend_mode): +def generate_child_process_param_list(ambari_user, java_exe, class_path, debug_start, suspend_mode): conf_dir = class_path if class_path.find(' ') != -1: conf_dir = '"' + class_path + '"' @@ -155,58 +133,15 @@ def generate_child_process_param_list(ambari_user, current_user, java_exe, class jvm_args, conf_dir, suspend_mode) - environ = os.environ.copy() - return (command, environ) + return command @OsFamilyFuncImpl(OsFamilyImpl.DEFAULT) -def generate_child_process_param_list(ambari_user, current_user, java_exe, class_path, debug_start, suspend_mode): +def generate_child_process_param_list(ambari_user, java_exe, class_path, + debug_start, suspend_mode): from ambari_commons.os_linux import ULIMIT_CMD properties = get_ambari_properties() - isSecure = get_is_secure(properties) - (isPersisted, masterKeyFile) = get_is_persisted(properties) - environ = os.environ.copy() - # Need to handle master key not persisted scenario - if isSecure and not masterKeyFile: - prompt = False - masterKey = environ.get(SECURITY_KEY_ENV_VAR_NAME) - - if masterKey is not None and masterKey != "": - pass - else: - keyLocation = environ.get(SECURITY_MASTER_KEY_LOCATION) - - if keyLocation is not None: - try: - # Verify master key can be read by the java process - with open(keyLocation, 'r'): - pass - except IOError: - print_warning_msg("Cannot read Master key from path specified in " - "environemnt.") - prompt = True - else: - # Key not provided in the environment - prompt = True - - if prompt: - import pwd - - masterKey = get_original_master_key(properties) - tempDir = tempfile.gettempdir() - tempFilePath = tempDir + os.sep + "masterkey" - save_master_key(masterKey, tempFilePath, True) - if ambari_user != current_user: - uid = pwd.getpwnam(ambari_user).pw_uid - gid = pwd.getpwnam(ambari_user).pw_gid - os.chown(tempFilePath, uid, gid) - else: - os.chmod(tempFilePath, stat.S_IREAD | stat.S_IWRITE) - - if tempFilePath is not None: - environ[SECURITY_MASTER_KEY_LOCATION] = tempFilePath - command_base = SERVER_START_CMD_DEBUG if debug_start else SERVER_START_CMD ulimit_cmd = "%s %s" % (ULIMIT_CMD, str(get_ulimit_open_files(properties))) @@ -235,7 +170,7 @@ def generate_child_process_param_list(ambari_user, current_user, java_exe, class cmd = "{ulimit_cmd} ; {command}".format(ulimit_cmd=ulimit_cmd, command=command) param_list.append(cmd) - return (param_list, environ) + return param_list @OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) def wait_for_server_start(pidFile, scmStatus): @@ -329,8 +264,10 @@ def server_process_main(options, scmStatus=None): suspend_start = (debug_mode & 2) or SUSPEND_START_MODE suspend_mode = 'y' if suspend_start else 'n' - (param_list, environ) = generate_child_process_param_list(ambari_user, current_user, - java_exe, class_path, debug_start, suspend_mode) + param_list = generate_child_process_param_list(ambari_user, java_exe, + class_path, debug_start, + suspend_mode) + environ = generate_env(ambari_user, current_user) if not os.path.exists(configDefaults.PID_DIR): os.makedirs(configDefaults.PID_DIR, 0755) http://git-wip-us.apache.org/repos/asf/ambari/blob/ec0f771d/ambari-server/src/test/python/TestAmbariServer.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py index 430fa60..777a63a 100644 --- a/ambari-server/src/test/python/TestAmbariServer.py +++ b/ambari-server/src/test/python/TestAmbariServer.py @@ -87,7 +87,7 @@ with patch("platform.linux_distribution", return_value = os_distro_value): SRVR_ONE_WAY_SSL_PORT_PROPERTY, SRVR_TWO_WAY_SSL_PORT_PROPERTY, GANGLIA_HTTPS from ambari_server.setupSecurity import adjust_directory_permissions, get_alias_string, get_ldap_event_spec_names, sync_ldap, LdapSyncOptions, \ configure_ldap_password, setup_ldap, REGEX_HOSTNAME_PORT, REGEX_TRUE_FALSE, REGEX_ANYTHING, setup_master_key, \ - setup_ambari_krb5_jaas + setup_ambari_krb5_jaas, ensure_can_start_under_current_user, generate_env from ambari_server.userInput import get_YN_input, get_choice_string_input, get_validated_string_input, \ read_password from ambari_server_main import get_ulimit_open_files, ULIMIT_OPEN_FILES_KEY, ULIMIT_OPEN_FILES_DEFAULT @@ -3287,7 +3287,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV @patch.object(_ambari_server_, "is_server_runing") @patch("os.chown") @patch("ambari_server.setupSecurity.get_master_key_location") - @patch("ambari_server_main.save_master_key") + @patch("ambari_server.setupSecurity.save_master_key") @patch("ambari_server_main.get_is_persisted") @patch("ambari_server_main.get_is_secure") @patch('os.chmod', autospec=True) @@ -3308,6 +3308,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV @patch("ambari_server_main.print_info_msg") @patch.object(PGConfig, "_check_postgre_up") @patch("ambari_server_main.read_ambari_user") + @patch("ambari_server.setupSecurity.is_root") @patch("ambari_server.dbConfiguration_linux.is_root") @patch("ambari_server_main.is_root") @patch.object(LinuxDBMSConfig, "_find_jdbc_driver") @@ -3315,7 +3316,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV @patch("os.chdir") @patch.object(ResourceFilesKeeper, "perform_housekeeping") def test_start(self, perform_housekeeping_mock, chdir_mock, getuser_mock, find_jdbc_driver_mock, - is_root_mock, is_root_2_mock, read_ambari_user_mock, + is_root_mock, is_root_2_mock, is_root_3_mock, read_ambari_user_mock, check_postgre_up_mock, print_info_msg_mock, print_warning_msg_mock, find_jdk_mock, check_database_name_property_mock, search_file_mock, popenMock, openMock, pexistsMock, @@ -3408,7 +3409,9 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV args = reset_mocks() read_ambari_user_mock.return_value = "dummy-user" getuser_mock.return_value = "non_custom_user" - is_root_2_mock.return_value = is_root_mock.return_value = False + is_root_3_mock.return_value = \ + is_root_2_mock.return_value = \ + is_root_mock.return_value = False try: _ambari_server_.start(args) self.fail("Should fail with 'Can not start ambari-server as user...'") @@ -3419,7 +3422,9 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV # Checking "jdk not found" args = reset_mocks() - is_root_2_mock.return_value = is_root_mock.return_value = True + is_root_3_mock.return_value = \ + is_root_2_mock.return_value = \ + is_root_mock.return_value = True find_jdk_mock.return_value = None try: @@ -3433,7 +3438,9 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV find_jdk_mock.return_value = "somewhere" ## Testing workflow under root - is_root_2_mock.return_value = is_root_mock.return_value = True + is_root_3_mock.return_value = \ + is_root_2_mock.return_value = \ + is_root_mock.return_value = True # Remote DB p.process_pair(JDBC_DATABASE_PROPERTY, 'oracle') @@ -3571,7 +3578,9 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV popenMock.reset_mock() ## Testing workflow under non-root - is_root_2_mock.return_value = is_root_mock.return_value = False + is_root_3_mock.return_value = \ + is_root_2_mock.return_value = \ + is_root_mock.return_value = False read_ambari_user_mock.return_value = "not-root-user" getuser_mock.return_value = read_ambari_user_mock.return_value @@ -3626,7 +3635,9 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV p.process_pair(JDBC_DATABASE_PROPERTY, 'postgres') p.process_pair(PERSISTENCE_TYPE_PROPERTY, 'local') read_ambari_user_mock.return_value = "root" - is_root_2_mock.return_value = is_root_mock.return_value = True + is_root_3_mock.return_value = \ + is_root_2_mock.return_value = \ + is_root_mock.return_value = True _ambari_server_.start(args) @@ -3645,7 +3656,9 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV p.process_pair(JDBC_DATABASE_PROPERTY, 'postgres') p.process_pair(PERSISTENCE_TYPE_PROPERTY, 'local') read_ambari_user_mock.return_value = "root" - is_root_2_mock.return_value = is_root_mock.return_value = True + is_root_3_mock.return_value = \ + is_root_2_mock.return_value = \ + is_root_mock.return_value = True get_validated_string_input_method.return_value = "masterkey" os_chmod_method.return_value = None get_is_secure_mock.return_value = True @@ -3820,26 +3833,39 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV pass + @patch("ambari_server.serverUpgrade.ensure_can_start_under_current_user") + @patch("ambari_server.serverUpgrade.generate_env") + @patch("ambari_server.serverUpgrade.read_ambari_user") @patch("ambari_server.serverConfiguration.get_conf_dir") @patch("ambari_server.serverConfiguration.get_ambari_classpath") @patch("ambari_server.serverUpgrade.run_os_command") @patch("ambari_server.serverUpgrade.get_java_exe_path") def test_run_schema_upgrade(self, java_exe_path_mock, run_os_command_mock, - get_ambari_classpath_mock, get_conf_dir_mock): + get_ambari_classpath_mock, get_conf_dir_mock, + read_ambari_user_mock, generate_env_mock, + ensure_can_start_under_current_user_mock): java_exe_path_mock.return_value = "/usr/lib/java/bin/java" run_os_command_mock.return_value = (0, None, None) get_ambari_classpath_mock.return_value = 'test:path12' get_conf_dir_mock.return_value = '/etc/conf' + command = '/usr/lib/java/bin/java -cp /etc/conf:test:path12 ' \ + 'org.apache.ambari.server.upgrade.SchemaUpgradeHelper ' \ + '> /var/log/ambari-server/ambari-server.out 2>&1' + environ = {} + generate_env_mock.return_value = environ + ensure_can_start_under_current_user_mock.return_value = "root" + read_ambari_user_mock.return_value = "ambari" run_schema_upgrade() self.assertTrue(java_exe_path_mock.called) + self.assertTrue(ensure_can_start_under_current_user_mock.called) + self.assertTrue(generate_env_mock.called) + self.assertTrue(read_ambari_user_mock.called) self.assertTrue(get_ambari_classpath_mock.called) self.assertTrue(get_conf_dir_mock.called) self.assertTrue(run_os_command_mock.called) - run_os_command_mock.assert_called_with('/usr/lib/java/bin/java -cp /etc/conf:test:path12 ' - 'org.apache.ambari.server.upgrade.SchemaUpgradeHelper ' - '> /var/log/ambari-server/ambari-server.out 2>&1') + run_os_command_mock.assert_called_with(command, env=environ) pass
