http://git-wip-us.apache.org/repos/asf/ambari/blob/fad56746/ambari-server/src/main/python/ambari_server/serverSetup_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverSetup_linux.py b/ambari-server/src/main/python/ambari_server/serverSetup_linux.py new file mode 100644 index 0000000..b5436e0 --- /dev/null +++ b/ambari-server/src/main/python/ambari_server/serverSetup_linux.py @@ -0,0 +1,795 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +import glob +import optparse + +import os +import re +import socket + +from ambari_commons.exceptions import * +from ambari_commons.logging_utils import * +from ambari_commons.os_linux import run_os_command +from ambari_server.dbConfiguration_linux import SERVICE_CMD, PG_HBA_CONF_FILE_BACKUP +from ambari_server.serverConfiguration import * +from ambari_server.serverConfiguration_linux import JAVA_SHARE_PATH +from ambari_server.setupSecurity import * +from ambari_server.userInput import get_YN_input, get_validated_string_input +from ambari_server import utils + +# selinux commands +GET_SE_LINUX_ST_CMD = utils.locate_file('sestatus', '/usr/sbin') +SE_SETENFORCE_CMD = "setenforce 0" +SE_STATUS_DISABLED = "disabled" +SE_STATUS_ENABLED = "enabled" +SE_MODE_ENFORCING = "enforcing" +SE_MODE_PERMISSIVE = "permissive" + +# Non-root user setup commands +NR_USER_COMMENT = "Ambari user" +NR_GET_OWNER_CMD = 'stat -c "%U" {0}' +NR_USERADD_CMD = 'useradd -M --comment "{1}" ' \ + '--shell %s -d /var/lib/ambari-server/keys/ {0}' % utils.locate_file('nologin', '/sbin') +NR_SET_USER_COMMENT_CMD = 'usermod -c "{0}" {1}' + +NR_USER_CHANGE_PROMPT = "Ambari-server daemon is configured to run under user '{0}'. Change this setting [y/n] (n)? " +NR_USER_CUSTOMIZE_PROMPT = "Customize user account for ambari-server daemon [y/n] (n)? " +NR_DEFAULT_USER = "root" + +# jdk commands +JDK_URL_PROPERTIES = ["jdk1.7.url", "jdk1.6.url"] +JCE_URL_PROPERTIES = ["jce_policy1.7.url", "jce_policy1.6.url"] +JDK_VERSION_REs = ["(jdk.*)/jre", "Creating (jdk.*)/jre"] +JDK_CHOICE_PROMPT = "[1] - Oracle JDK 1.7\n[2] - Oracle JDK 1.6\n[3] - Custom JDK\n==============================================================================\nEnter choice ({0}): " +JDK_VALID_CHOICES = "^[123]$" +CUSTOM_JDK_NUMBER = "3" +JDK_MIN_FILESIZE = 5000 +CREATE_JDK_DIR_CMD = "/bin/mkdir -p {0}" +MAKE_FILE_EXECUTABLE_CMD = "chmod a+x {0}" + +JDK_DOWNLOAD_CMD = "curl --create-dirs -o {0} {1}" +JDK_DOWNLOAD_SIZE_CMD = "curl -I {0}" + +# use --no-same-owner when running as root to prevent uucp as the user (AMBARI-6478) +UNTAR_JDK_ARCHIVE = "tar --no-same-owner -xvf {0}" + + +#JDBC +DATABASE_INDEX = 0 +USERNAME_PATTERN = "^[a-zA-Z_][a-zA-Z0-9_\-]*$" +DATABASE_NAMES = ["postgres", "oracle", "mysql"] +DATABASE_STORAGE_NAMES = ["Database", "Service", "Database"] +DATABASE_PORTS = ["5432", "1521", "3306"] +DATABASE_DRIVER_NAMES = ["org.postgresql.Driver", "oracle.jdbc.driver.OracleDriver", "com.mysql.jdbc.Driver"] +DATABASE_CONNECTION_STRINGS = [ + "jdbc:postgresql://{0}:{1}/{2}", + "jdbc:oracle:thin:@{0}:{1}/{2}", + "jdbc:mysql://{0}:{1}/{2}"] +DATABASE_CONNECTION_STRINGS_ALT = [ + "jdbc:postgresql://{0}:{1}/{2}", + "jdbc:oracle:thin:@{0}:{1}:{2}", + "jdbc:mysql://{0}:{1}/{2}"] +ORACLE_SID_PATTERN = "jdbc:oracle:thin:@.+:.+/.+" +ORACLE_SNAME_PATTERN = "jdbc:oracle:thin:@.+:.+:.+" + +DATABASE_CLI_TOOLS = [["psql"], ["sqlplus", "sqlplus64"], ["mysql"]] +DATABASE_CLI_TOOLS_DESC = ["psql", "sqlplus", "mysql"] +DATABASE_CLI_TOOLS_USAGE = ['su -postgres --command=psql -f {0} -v username=\'"{1}"\' -v password="\'{2}\'"', + 'sqlplus {1}/{2} < {0} ', + 'mysql --user={1} --password={2} {3}<{0}'] + +MYSQL_INIT_SCRIPT = '/var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql' +DATABASE_INIT_SCRIPTS = ['/var/lib/ambari-server/resources/Ambari-DDL-Postgres-CREATE.sql', + '/var/lib/ambari-server/resources/Ambari-DDL-Oracle-CREATE.sql', + MYSQL_INIT_SCRIPT] +DATABASE_DROP_SCRIPTS = ['/var/lib/ambari-server/resources/Ambari-DDL-Postgres-DROP.sql', + '/var/lib/ambari-server/resources/Ambari-DDL-Oracle-DROP.sql', + '/var/lib/ambari-server/resources/Ambari-DDL-MySQL-DROP.sql'] + +JDBC_PROPERTIES_PREFIX = "server.jdbc.properties." +DATABASE_JDBC_PROPERTIES = [ + [ ], + [ + ["oracle.net.CONNECT_TIMEOUT", "2000"], # socket level timeout + ["oracle.net.READ_TIMEOUT", "2000"], # socket level timeout + ["oracle.jdbc.ReadTimeout", "8000"] # query fetch timeout + ], + [ ] + ] + +POSTGRES_EXEC_ARGS = "-h {0} -p {1} -d {2} -U {3} -f {4} -v username='{3}'" +ORACLE_EXEC_ARGS = "-S -L '{0}/{1}@(description=(address=(protocol=TCP)(host={2})(port={3}))(connect_data=({6}={4})))' @{5} {0}" +MYSQL_EXEC_ARGS_WITH_USER_VARS = "--host={0} --port={1} --user={2} --password={3} {4} " \ + "-e\"set @schema=\'{4}\'; set @username=\'{2}\'; source {5};\"" +MYSQL_EXEC_ARGS_WO_USER_VARS = "--force --host={0} --port={1} --user={2} --password={3} --database={4} < {5} 2> /dev/null" +MYSQL_UPGRADE_STACK_ARGS = "--host={0} --port={1} --user={2} --password={3} --database={4} " \ + "-e\"set @stackName=\'{6}\'; set @stackVersion=\'{7}\'; source {5};\"" + +ORACLE_UPGRADE_STACK_ARGS = "-S -L '{0}/{1}@(description=(address=(protocol=TCP)(host={2})(port={3}))(connect_data=({6}={4})))' @{5} {7} {8}" + +JDBC_PATTERNS = {"oracle": "*ojdbc*.jar", "mysql": "*mysql*.jar"} +DATABASE_FULL_NAMES = {"oracle": "Oracle", "mysql": "MySQL", "postgres": "PostgreSQL"} +JDBC_DB_OPTION_VALUES = ["postgres", "mysql", "oracle"] +JDBC_DB_DEFAULT_DRIVER = {"postgresql" : "postgresql-jdbc.jar", "mysql" : "mysql-connector-java.jar", "oracle" : "ojdbc6.jar"} +ORACLE_DB_ID_TYPES = ["Service Name", "SID"] + + +DEFAULT_DB_NAME = "ambari" + + +MESSAGE_ERROR_NOT_ROOT = 'Ambari-server setup should be run with root-level privileges' + +MESSAGE_CHECK_FIREWALL = 'Checking iptables...' + +class FirewallChecks(object): + def __init__(self): + + self.FIREWALL_SERVICE_NAME = "iptables" + self.SERVICE_CMD = SERVICE_CMD + self.SERVICE_SUBCMD = "status" + + def get_command(self): + return "%s %s %s" % (self.SERVICE_CMD, self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD) + + def check_result(self, retcode, out, err): + return retcode == 0 + + def check_iptables(self): + retcode, out, err = run_os_command(self.get_command()) + if err and len(err) > 0: + print err + if self.check_result(retcode, out, err): + print_warning_msg("%s is running. Confirm the necessary Ambari ports are accessible. " % + self.FIREWALL_SERVICE_NAME + + "Refer to the Ambari documentation for more details on ports.") + ok = get_YN_input("OK to continue [y/n] (y)? ", True) + if not ok: + raise FatalException(1, None) + + def get_running_result(self): + # To support test code. Expected ouput from run_os_command. + return (0, "", "") + + def get_stopped_result(self): + # To support test code. Expected output from run_os_command. + return (3, "", "") + + +class UbuntuFirewallChecks(FirewallChecks): + def __init__(self): + super(UbuntuFirewallChecks, self).__init__() + + self.FIREWALL_SERVICE_NAME = "ufw" + self.SERVICE_CMD = utils.locate_file('service', '/usr/sbin') + + def check_result(self, retcode, out, err): + # On ubuntu, the status command returns 0 whether running or not + return out and len(out) > 0 and out.strip() != "ufw stop/waiting" + + def get_running_result(self): + # To support test code. Expected ouput from run_os_command. + return (0, "ufw start/running", "") + + def get_stopped_result(self): + # To support test code. Expected output from run_os_command. + return (0, "ufw stop/waiting", "") + + +class Fedora18FirewallChecks(FirewallChecks): + def __init__(self): + self.FIREWALL_SERVICE_NAME = "firewalld.service" + + def get_command(self): + return "systemctl is-active firewalld.service" + + +class OpenSuseFirewallChecks(FirewallChecks): + def __init__(self): + self.FIREWALL_SERVICE_NAME = "SuSEfirewall2" + + def get_command(self): + return "/sbin/SuSEfirewall2 status" + + +def get_firewall_object(): + if OS_TYPE == OSConst.OS_UBUNTU: + return UbuntuFirewallChecks() + elif OS_TYPE == OSConst.OS_FEDORA and int(OS_VERSION) >= 18: + return Fedora18FirewallChecks() + elif OS_TYPE == OSConst.OS_OPENSUSE: + return OpenSuseFirewallChecks() + else: + return FirewallChecks() + + +def get_firewall_object_types(): + # To support test code, so tests can loop through the types + return (FirewallChecks, + UbuntuFirewallChecks, + Fedora18FirewallChecks, + OpenSuseFirewallChecks) + + +def os_check_firewall(): + return get_firewall_object().check_iptables() + + +# +# Checks SELinux +# +def check_selinux(): + try: + retcode, out, err = run_os_command(GET_SE_LINUX_ST_CMD) + se_status = re.search('(disabled|enabled)', out).group(0) + print "SELinux status is '" + se_status + "'" + if se_status == SE_STATUS_DISABLED: + return 0 + else: + try: + se_mode = re.search('(enforcing|permissive)', out).group(0) + except AttributeError: + err = "Error determining SELinux mode. Exiting." + raise FatalException(1, err) + print "SELinux mode is '" + se_mode + "'" + if se_mode == SE_MODE_ENFORCING: + print "Temporarily disabling SELinux" + run_os_command(SE_SETENFORCE_CMD) + print_warning_msg( + "SELinux is set to 'permissive' mode and temporarily disabled.") + ok = get_YN_input("OK to continue [y/n] (y)? ", True) + if not ok: + raise FatalException(1, None) + return 0 + except OSError: + print_warning_msg("Could not run {0}: OK".format(GET_SE_LINUX_ST_CMD)) + return 0 + +def disable_security_enhancements(): + print 'Checking SELinux...' + err = '' + retcode = check_selinux() + if not retcode == 0: + err = 'Failed to disable SELinux. Exiting.' + return (retcode, err) + + +# +# User account creation +# + +def os_create_custom_user(): + user = get_validated_string_input( + "Enter user account for ambari-server daemon (root):", + "root", + "^[a-z_][a-z0-9_-]{1,31}$", + "Invalid username.", + False + ) + + print_info_msg("Trying to create user {0}".format(user)) + command = NR_USERADD_CMD.format(user, NR_USER_COMMENT) + retcode, out, err = run_os_command(command) + if retcode == 9: # 9 = username already in use + print_info_msg("User {0} already exists, " + "skipping user creation".format(user)) + + elif retcode != 0: # fail + print_warning_msg("Can't create user {0}. Command {1} " + "finished with {2}: \n{3}".format(user, command, retcode, err)) + return retcode, None + + print_info_msg("User configuration is done.") + return 0, user + + +# +# JDK Setup +# + +def os_install_jdk(java_inst_file, java_home_dir): + print "Installing JDK to {0}".format(java_home_dir) + retcode, out, err = run_os_command(CREATE_JDK_DIR_CMD.format(java_home_dir)) + savedPath = os.getcwd() + os.chdir(java_home_dir) + + if java_inst_file.endswith(".bin"): + retcode, out, err = run_os_command(MAKE_FILE_EXECUTABLE_CMD.format(java_inst_file)) + retcode, out, err = run_os_command(java_inst_file + ' -noregister') + elif java_inst_file.endswith(".gz"): + retcode, out, err = run_os_command(UNTAR_JDK_ARCHIVE.format(java_inst_file)) + else: + err = "JDK installation failed.Unknown file mask." + raise FatalException(1, err) + + os.chdir(savedPath) + + if retcode != 0: + err = "Installation of JDK returned exit code %s" % retcode + raise FatalException(retcode, err) + + print "Successfully installed JDK to {0}".format(java_home_dir) + return (retcode, out) + +def os_ensure_java_home_env_var_is_set(java_home_var): + if not os.environ.has_key(JAVA_HOME) or os.environ[JAVA_HOME] != java_home_var: + os.system("SETX {0} {1} /M".format(JAVA_HOME, java_home_var)) + os.environ[JAVA_HOME] = java_home_var + pass + + +# +# JDBC Setup +# + +def os_check_jdbc_options(options): + return (options.jdbc_driver is not None and options.jdbc_db is not None) + +#Check if required jdbc drivers present +def os_find_jdbc_driver(args): + if args.dbms in JDBC_PATTERNS.keys(): + drivers = [] + drivers.extend(glob.glob(JAVA_SHARE_PATH + os.sep + JDBC_PATTERNS[args.dbms])) + if drivers: + return drivers + return -1 + return 0 + +def os_setup_jdbc_drivers(args): + result = os_find_jdbc_driver(args) + + msg = 'Before starting Ambari Server, ' \ + 'you must copy the {0} JDBC driver JAR file to {1}.'.format( + DATABASE_FULL_NAMES[args.dbms], + JAVA_SHARE_PATH) + + if result == -1: + if SILENT: + print_error_msg(msg) + raise FatalException(-1, msg) + else: + print_warning_msg(msg) + raw_input(PRESS_ENTER_MSG) + result = os_find_jdbc_driver(args) + if result == -1: + print_error_msg(msg) + raise FatalException(-1, msg) + + # Check if selected RDBMS requires drivers to copy + if type(result) is not int: + print 'Copying JDBC drivers to server resources...' + try: + resources_dir = args[RESOURCES_DIR_PROPERTY] + except KeyError: + print_error_msg("There is no value for " + RESOURCES_DIR_PROPERTY + "in " + AMBARI_PROPERTIES_FILE) + return -1 + + db_name = DATABASE_FULL_NAMES[args.dbms].lower() + jdbc_symlink = os.path.join(resources_dir, db_name + "-jdbc-driver.jar") + db_default_driver_path = os.path.join(JAVA_SHARE_PATH, JDBC_DB_DEFAULT_DRIVER[db_name]) + + if os.path.lexists(jdbc_symlink): + os.remove(jdbc_symlink) + + copy_status = copy_files(result, resources_dir) + + if not copy_status == 0: + raise FatalException(-1, "Failed to copy JDBC drivers to server resources") + + if db_default_driver_path in result: + os.symlink(os.path.join(resources_dir, JDBC_DB_DEFAULT_DRIVER[db_name]), jdbc_symlink) + +def os_load_default_db_properties(args): + args.persistence_type = 'local' + args.dbms = DATABASE_NAMES[DATABASE_INDEX] + args.database_host = "localhost" + args.database_port = DATABASE_PORTS[DATABASE_INDEX] + args.database_name = DEFAULT_DB_NAME + args.database_username = "ambari" + args.database_password = "bigdata" + args.sid_or_sname = "sname" + pass + +# Check if jdbc user is changed +def is_jdbc_user_changed(args): + properties = get_ambari_properties() + if properties == -1: + print_error_msg("Error getting ambari properties") + return None + + previos_user = properties[JDBC_USER_NAME_PROPERTY] + new_user = args.database_username + + if previos_user and new_user: + if previos_user != new_user: + return True + else: + return False + + return None + +def os_setup_database(options): + if is_local_database(options): + os_setup_local_database(options) + else: + os_setup_remote_database(options) + +def os_setup_local_database(options): + #check if jdbc user is changed + is_user_changed = is_jdbc_user_changed(options) + + print 'Default properties detected. Using built-in database.' + os_store_local_properties(options) + + print 'Checking PostgreSQL...' + pg_status, retcode, out, err = check_postgre_up() + if not retcode == 0: + err = 'Unable to start PostgreSQL server. Status {0}. {1}.' \ + ' Exiting'.format(pg_status, err) + raise FatalException(retcode, err) + + print 'Configuring local database...' + retcode, outdata, errdata = setup_db(options) + if not retcode == 0: + err = 'Running database init script was failed. {0}. Exiting.'.format(errdata) + raise FatalException(retcode, err) + + if is_user_changed: + #remove backup for pg_hba in order to reconfigure postgres + remove_file(PG_HBA_CONF_FILE_BACKUP) + + print 'Configuring PostgreSQL...' + retcode, out, err = configure_postgres() + if not retcode == 0: + err = 'Unable to configure PostgreSQL server. {0} Exiting'.format(err) + raise FatalException(retcode, err) + pass + +def os_setup_remote_database(options): + retcode = os_store_remote_properties(options) + if retcode != 0: + err = 'Unable to save config file' + raise FatalException(retcode, err) + + os_setup_jdbc_drivers(options) + + print 'Configuring remote database connection properties...' + retcode = setup_remote_db(options) + if retcode == -1: + err = "Remote database setup aborted." + raise NonFatalException(err) + + if not retcode == 0: + err = 'Error while configuring connection properties. Exiting' + raise FatalException(retcode, err) + pass + +def os_reset_database(options): + pass + + +### Postgres ### + + +def configure_pg_hba_ambaridb_users(): + args = optparse.Values() + configure_database_username_password(args) + + with open(PG_HBA_CONF_FILE, "a") as pgHbaConf: + pgHbaConf.write("\n") + pgHbaConf.write("local all " + args.database_username + + ",mapred md5") + pgHbaConf.write("\n") + pgHbaConf.write("host all " + args.database_username + + ",mapred 0.0.0.0/0 md5") + pgHbaConf.write("\n") + pgHbaConf.write("host all " + args.database_username + + ",mapred ::/0 md5") + pgHbaConf.write("\n") + retcode, out, err = run_os_command(PG_HBA_RELOAD_CMD) + if not retcode == 0: + raise FatalException(retcode, err) + + +def configure_pg_hba_postgres_user(): + postgresString = "all postgres" + for line in fileinput.input(PG_HBA_CONF_FILE, inplace=1): + print re.sub('all\s*all', postgresString, line), + os.chmod(PG_HBA_CONF_FILE, 0644) + + +def configure_postgresql_conf(): + listenAddress = "listen_addresses = '*' #" + for line in fileinput.input(POSTGRESQL_CONF_FILE, inplace=1): + print re.sub('#+listen_addresses.*?(#|$)', listenAddress, line), + os.chmod(POSTGRESQL_CONF_FILE, 0644) + + +def configure_postgres(): + if os.path.isfile(PG_HBA_CONF_FILE): + if not os.path.isfile(PG_HBA_CONF_FILE_BACKUP): + shutil.copyfile(PG_HBA_CONF_FILE, PG_HBA_CONF_FILE_BACKUP) + else: + #Postgres has been configured before, must not override backup + print "Backup for pg_hba found, reconfiguration not required" + return 0, "", "" + configure_pg_hba_postgres_user() + configure_pg_hba_ambaridb_users() + os.chmod(PG_HBA_CONF_FILE, 0644) + configure_postgresql_conf() + #restart postgresql if already running + pg_status, retcode, out, err = get_postgre_status() + if pg_status == PG_STATUS_RUNNING: + retcode, out, err = restart_postgres() + return retcode, out, err + return 0, "", "" + + +def restart_postgres(): + print "Restarting PostgreSQL" + process = subprocess.Popen(PG_RESTART_CMD.split(' '), + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE + ) + time.sleep(5) + result = process.poll() + if result is None: + print_info_msg("Killing restart PostgresSQL process") + process.kill() + pg_status, retcode, out, err = get_postgre_status() + # SUSE linux set status of stopped postgresql proc to unused + if pg_status == "unused" or pg_status == "stopped": + print_info_msg("PostgreSQL is stopped. Restarting ...") + retcode, out, err = run_os_command(PG_START_CMD) + return retcode, out, err + return 0, "", "" + + +# todo: check if the scheme is already exist + + +def setup_db(args): + #password access to ambari-server and mapred + configure_database_username_password(args) + dbname = args.database_name + scriptFile = args.init_script_file + username = args.database_username + password = args.database_password + + #setup DB + command = SETUP_DB_CMD[:] + command[-1] = command[-1].format(scriptFile, username, password, dbname) + + for i in range(SETUP_DB_CONNECT_ATTEMPTS): + sys.stdout.write('Connecting to local database...') + retcode, outdata, errdata = run_os_command(command) + if retcode == 0: + print 'done.' + return retcode, outdata, errdata + timeOutMsg = 'connection timed out' + if (i+1) < SETUP_DB_CONNECT_ATTEMPTS: + timeOutMsg += '...retrying (%d)' % (i+1) + print timeOutMsg + time.sleep(SETUP_DB_CONNECT_TIMEOUT) + + print 'unable to connect to database' + print_error_msg(errdata) + return retcode, outdata, errdata + + +def execute_db_script(args, file): + #password access to ambari-server and mapred + configure_database_username_password(args) + dbname = args.database_name + username = args.database_username + password = args.database_password + command = SETUP_DB_CMD[:] + command[-1] = command[-1].format(file, username, password, dbname) + retcode, outdata, errdata = run_os_command(command) + if not retcode == 0: + print errdata + return retcode + + +def check_db_consistency(args, file): + #password access to ambari-server and mapred + configure_database_username_password(args) + dbname = args.database_name + username = args.database_username + password = args.database_password + command = SETUP_DB_CMD[:] + command[-1] = command[-1].format(file, username, password, dbname) + retcode, outdata, errdata = run_os_command(command) + if not retcode == 0: + print errdata + return retcode + else: + # Assumes that the output is of the form ...\n<count> + print_info_msg("Parsing output: " + outdata) + lines = outdata.splitlines() + if (lines[-1] == '3' or lines[-1] == '0'): + return 0 + return -1 + + +def get_postgre_status(): + retcode, out, err = run_os_command(PG_ST_CMD) + try: + pg_status = re.search('(stopped|running)', out, re.IGNORECASE).group(0).lower() + except AttributeError: + pg_status = None + return pg_status, retcode, out, err + + +def check_postgre_up(): + pg_status, retcode, out, err = get_postgre_status() + if pg_status == PG_STATUS_RUNNING: + print_info_msg("PostgreSQL is running") + return pg_status, 0, out, err + else: + # run initdb only on non ubuntu systems as ubuntu does not have initdb cmd. + if OS_TYPE != OSConst.OS_UBUNTU: + print "Running initdb: This may take upto a minute." + retcode, out, err = run_os_command(PG_INITDB_CMD) + if retcode == 0: + print out + print "About to start PostgreSQL" + try: + process = subprocess.Popen(PG_START_CMD.split(' '), + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE + ) + if OS_TYPE == OSConst.OS_SUSE: + time.sleep(20) + result = process.poll() + print_info_msg("Result of postgres start cmd: " + str(result)) + if result is None: + process.kill() + pg_status, retcode, out, err = get_postgre_status() + else: + retcode = result + else: + out, err = process.communicate() + retcode = process.returncode + if pg_status == PG_STATUS_RUNNING: + print_info_msg("Postgres process is running. Returning...") + return pg_status, 0, out, err + except (Exception), e: + pg_status, retcode, out, err = get_postgre_status() + if pg_status == PG_STATUS_RUNNING: + return pg_status, 0, out, err + else: + print_error_msg("Postgres start failed. " + str(e)) + return pg_status, retcode, out, err + +def get_validated_db_name(database_name): + return get_validated_string_input( + DATABASE_STORAGE_NAMES[DATABASE_INDEX] + " Name (" + + database_name + "): ", + database_name, + ".*", + "Invalid " + DATABASE_STORAGE_NAMES[DATABASE_INDEX] + " name.", + False + ) + +def get_validated_service_name(service_name, index): + return get_validated_string_input( + ORACLE_DB_ID_TYPES[index] + " (" + service_name + "): ", + service_name, + ".*", + "Invalid " + ORACLE_DB_ID_TYPES[index] + ".", + False + ) + +def get_pass_file_path(conf_file): + return os.path.join(os.path.dirname(conf_file), + JDBC_PASSWORD_FILENAME) + +# Store local database connection properties +def os_store_local_properties(args): + properties = get_ambari_properties() + if properties == -1: + print_error_msg("Error getting ambari properties") + return -1 + + isSecure = get_is_secure(properties) + + properties.removeOldProp(JDBC_SCHEMA_PROPERTY) + properties.removeOldProp(JDBC_HOSTNAME_PROPERTY) + properties.removeOldProp(JDBC_RCA_DRIVER_PROPERTY) + properties.removeOldProp(JDBC_RCA_URL_PROPERTY) + properties.removeOldProp(JDBC_PORT_PROPERTY) + properties.removeOldProp(JDBC_DRIVER_PROPERTY) + properties.removeOldProp(JDBC_URL_PROPERTY) + properties.process_pair(PERSISTENCE_TYPE_PROPERTY, "local") + properties.process_pair(JDBC_DATABASE_PROPERTY, args.database_name) + properties.process_pair(JDBC_USER_NAME_PROPERTY, args.database_username) + properties.process_pair(JDBC_PASSWORD_PROPERTY, + store_password_file(args.database_password, JDBC_PASSWORD_FILENAME)) + + if isSecure: + encrypted_password = encrypt_password(JDBC_RCA_PASSWORD_ALIAS, args.database_password) + if args.database_password != encrypted_password: + properties.process_pair(JDBC_PASSWORD_PROPERTY, encrypted_password) + pass + pass + + return 0 + + +# Store set of properties for remote database connection +def os_store_remote_properties(args): + properties = get_ambari_properties() + if properties == -1: + print_error_msg("Error getting ambari properties") + return -1 + + isSecure = get_is_secure(properties) + + properties.process_pair(PERSISTENCE_TYPE_PROPERTY, "remote") + + properties.process_pair(JDBC_DATABASE_PROPERTY, args.dbms) + properties.process_pair(JDBC_HOSTNAME_PROPERTY, args.database_host) + properties.process_pair(JDBC_PORT_PROPERTY, args.database_port) + properties.process_pair(JDBC_SCHEMA_PROPERTY, args.database_name) + + properties.process_pair(JDBC_DRIVER_PROPERTY, DATABASE_DRIVER_NAMES[DATABASE_INDEX]) + # fully qualify the hostname to make sure all the other hosts can connect + # to the jdbc hostname since its passed onto the agents for RCA + jdbc_hostname = args.database_host + if (args.database_host == "localhost"): + jdbc_hostname = socket.getfqdn() + + connectionStringFormat = DATABASE_CONNECTION_STRINGS + if args.sid_or_sname == "sid": + connectionStringFormat = DATABASE_CONNECTION_STRINGS_ALT + properties.process_pair(JDBC_URL_PROPERTY, + connectionStringFormat[DATABASE_INDEX].format(jdbc_hostname, args.database_port, + args.database_name)) + properties.process_pair(JDBC_USER_NAME_PROPERTY, args.database_username) + properties.process_pair(JDBC_PASSWORD_PROPERTY, + store_password_file(args.database_password, JDBC_PASSWORD_FILENAME)) + + # save any other defined properties to pass to JDBC + if DATABASE_INDEX < len(DATABASE_JDBC_PROPERTIES): + for pair in DATABASE_JDBC_PROPERTIES[DATABASE_INDEX]: + properties.process_pair(JDBC_PROPERTIES_PREFIX + pair[0], pair[1]) + + if isSecure: + encrypted_password = encrypt_password(JDBC_RCA_PASSWORD_ALIAS, args.database_password) + if encrypted_password != args.database_password: + properties.process_pair(JDBC_PASSWORD_PROPERTY, encrypted_password) + pass + + properties.process_pair(JDBC_RCA_DRIVER_PROPERTY, DATABASE_DRIVER_NAMES[DATABASE_INDEX]) + properties.process_pair(JDBC_RCA_URL_PROPERTY, + connectionStringFormat[DATABASE_INDEX].format(jdbc_hostname, args.database_port, + args.database_name)) + properties.process_pair(JDBC_RCA_USER_NAME_PROPERTY, args.database_username) + properties.process_pair(JDBC_RCA_PASSWORD_FILE_PROPERTY, + store_password_file(args.database_password, JDBC_PASSWORD_FILENAME)) + if isSecure: + encrypted_password = encrypt_password(JDBC_RCA_PASSWORD_ALIAS, args.database_password) + if encrypted_password != args.database_password: + properties.process_pair(JDBC_RCA_PASSWORD_FILE_PROPERTY, encrypted_password) + pass + + return 0
http://git-wip-us.apache.org/repos/asf/ambari/blob/fad56746/ambari-server/src/main/python/ambari_server/serverSetup_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverSetup_windows.py b/ambari-server/src/main/python/ambari_server/serverSetup_windows.py new file mode 100644 index 0000000..a906ef5 --- /dev/null +++ b/ambari-server/src/main/python/ambari_server/serverSetup_windows.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import optparse +import os +import socket +import string + +from _winreg import (OpenKey, EnumValue, HKEY_LOCAL_MACHINE, KEY_READ, CloseKey, KEY_WRITE, QueryValueEx, SetValueEx, + REG_EXPAND_SZ) + +from ambari_commons import os_utils + +from ambari_commons.exceptions import * +from ambari_commons.logging_utils import * +from ambari_commons.os_windows import run_os_command, UserHelper +from ambari_server.dbConfiguration import DBMSConfig +from ambari_server.serverConfiguration import * +from ambari_server.serverConfiguration_windows import OUT_DIR +from ambari_server.userInput import get_validated_string_input + +# Non-root user setup commands +NR_USER_COMMENT = "Ambari user" +NR_GET_OWNER_CMD = 'stat -c "%U" {0}' +NR_USERADD_CMD = 'cmd /C net user {0} {1} /ADD' +NR_SET_USER_COMMENT_CMD = 'usermod -c "{0}" {1}' + +NR_USER_CHANGE_PROMPT = "Ambari-server service is configured to run under user '{0}'. Change this setting [y/n] (n)? " +NR_USER_CUSTOMIZE_PROMPT = "Customize user account for ambari-server service [y/n] (n)? " +NR_DEFAULT_USER = "NT AUTHORITY\SYSTEM" + +SERVICE_USERNAME_KEY = "TMP_AMBARI_USERNAME" +SERVICE_PASSWORD_KEY = "TMP_AMBARI_PASSWORD" + +# JDK setup choices +JDK_DEFAULT_CONFIGS = [ + JDKRelease("jdk7.67", "Oracle JDK 7.67", + "http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-windows-x64.exe";, "jdk-7u67-windows-x64.exe", + "http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip";, "UnlimitedJCEPolicyJDK7.zip", + None) +] + +JDK_VERSION_REs = ["(jdk.*)/jre", "Creating (jdk.*)/jre"] +JDK_PROMPT = "[{0}] {1}\n" +JDK_CUSTOM_CHOICE_PROMPT = "[{0}] - Custom JDK\n==============================================================================\nEnter choice ({1}): " +JDK_VALID_CHOICES = "^[{0}{1:d}]$" +CUSTOM_JDK_NUMBER = "4" +JDK_MIN_FILESIZE = 5000 +MAKE_FILE_EXECUTABLE_CMD = "chmod a+x {0}" + +JDK_DOWNLOAD_CMD = "curl --create-dirs -o {0} {1}" +JDK_DOWNLOAD_SIZE_CMD = "curl -I {0}" + +# use --no-same-owner when running as root to prevent uucp as the user (AMBARI-6478) +UNTAR_JDK_ARCHIVE = "tar --no-same-owner -xvf {0}" + + +#JDBC +USERNAME_PATTERN = "^[a-zA-Z_][a-zA-Z0-9_\-]*$" +DATABASE_DBMS = "sqlserver" +DATABASE_NAME = "ambari" +DATABASE_SERVER = "localhost\\\\SQLEXPRESS" +DATABASE_DRIVER_NAME = "com.microsoft.sqlserver.jdbc.SQLServerDriver" + +METRICS_DATABASE_NAME = "HadoopMetrics" + +JDBC_PATTERNS = {"sqlserver": "sqljdbc*.jar"} +DATABASE_FULL_NAMES = {"sqlserver": "SQL Server"} +JDBC_DB_OPTION_VALUES = ["sqlserver"] +JDBC_DB_DEFAULT_DRIVER = {"sqlserver" : "sqljdbc4.jar"} + + +ERROR_NOT_ROOT = 'Ambari-server setup should be run with administrator-level privileges' + +MESSAGE_CHECK_FIREWALL = 'Checking firewall status...' + +def os_check_firewall(): + out = run_powershell_script(CHECK_FIREWALL_SCRIPT) + if out[0] != 0: + print_warning_msg("Unable to check firewall status:{0}".format(out[2])) + return False + profiles_status = [i for i in out[1].split("\n") if not i == ""] + if "1" in profiles_status: + enabled_profiles = [] + if profiles_status[0] == "1": + enabled_profiles.append("DomainProfile") + if profiles_status[1] == "1": + enabled_profiles.append("StandardProfile") + if profiles_status[2] == "1": + enabled_profiles.append("PublicProfile") + print_warning_msg("Following firewall profiles enabled:{0}. Make sure that firewall properly configured.".format(",".join(enabled_profiles))) + return False + return True + +# No security enhancements in Windows +def disable_security_enhancements(): + retcode = 0 + err = '' + return (retcode, err) + + +# +# User account creation +# + +def os_create_custom_user(): + user = get_validated_string_input( + "Enter user account for ambari-server service ({0}):".format(NR_DEFAULT_USER), + NR_DEFAULT_USER, None, + "Invalid username.", + False + ) + if user == NR_DEFAULT_USER: + return 0, user + password = get_validated_string_input("Enter password for user {0}:".format(user), "", None, "Password", True, False) + + uh = UserHelper() + + status, message = uh.create_user(user,password) + if status == UserHelper.USER_EXISTS: + print_info_msg("User {0} already exists, make sure that you typed correct password for user, " + "skipping user creation".format(user)) + + elif status == UserHelper.ACTION_FAILED: # fail + print_warning_msg("Can't create user {0}. Failed with message {1}".format(user, message)) + return UserHelper.ACTION_FAILED, None + + # setting SeServiceLogonRight to user + + status, message = uh.add_user_privilege(user, 'SeServiceLogonRight') + if status == UserHelper.ACTION_FAILED: + print_warning_msg("Can't add SeServiceLogonRight to user {0}. Failed with message {1}".format(user, message)) + return UserHelper.ACTION_FAILED, None + + print_info_msg("User configuration is done.") + print_warning_msg("When using non SYSTEM user make sure that your user have read\write access to log directories and " + "all server directories. In case of integrated authentication for SQL Server make sure that your " + "user properly configured to use ambari and metric database.") + #storing username and password in os.environ temporary to pass them to service + os.environ[SERVICE_USERNAME_KEY] = user + os.environ[SERVICE_PASSWORD_KEY] = password + return 0, user + + +# +# JDK Setup +# +def populate_jdk_configs(properties, jdk_num): + if properties.has_key(JDK_RELEASES): + jdk_names = properties[JDK_RELEASES].split(',') + jdks = [] + for jdk_name in jdk_names: + jdkR = JDKRelease.from_properties(properties, jdk_name) + jdks.append(jdkR) + else: + jdks = JDK_DEFAULT_CONFIGS + + n_config = 1 + jdk_choice_prompt = '' + jdk_choices = '' + for jdk in jdks: + jdk_choice_prompt += JDK_PROMPT.format(n_config, jdk.desc) + jdk_choices_tmp = '{0}{1:d}'.format(jdk_choices, n_config) + jdk_choices = jdk_choices_tmp + n_config += 1 + + jdk_choice_prompt += JDK_CUSTOM_CHOICE_PROMPT.format(n_config, jdk_num) + jdk_valid_choices = JDK_VALID_CHOICES.format(jdk_choices, n_config) + + return (jdks, jdk_choice_prompt, jdk_valid_choices, n_config) + + +def os_install_jdk(java_inst_file, java_home_dir): + print "Installing JDK to {0}".format(java_home_dir) + + if not os.path.exists(java_home_dir): + os.makedirs(java_home_dir) + + if java_inst_file.endswith(".exe"): + (dirname, filename) = os.path.split(java_inst_file) + installLogFilePath = os.path.join(OUT_DIR, filename + "-install.log") + #jre7u67.exe /s INSTALLDIR=<dir> STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-server\\jre7u67.exe-install.log + installCmd = [ + java_inst_file, + "/s", + "INSTALLDIR=" + java_home_dir, + "STATIC=1", + "WEB_JAVA=0", + "/L", + installLogFilePath + ] + retcode, out, err = run_os_command(installCmd) + #TODO: support .msi file installations + #msiexec.exe jre.msi /s INSTALLDIR=<dir> STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-server\\jre7u67-install.log ? + else: + err = "JDK installation failed.Unknown file mask." + raise FatalException(1, err) + + if retcode == 1603: + # JDK already installed + print "JDK already installed in {0}".format(java_home_dir) + retcode = 0 + else: + if retcode != 0: + err = "Installation of JDK returned exit code %s" % retcode + raise FatalException(retcode, err) + + print "Successfully installed JDK to {0}".format(java_home_dir) + + # Don't forget to adjust the JAVA_HOME env var + + return (retcode, out) + +def os_ensure_java_home_env_var_is_set(java_home_var): + if not os.environ.has_key(JAVA_HOME) or os.environ[JAVA_HOME] != java_home_var: + java_home_var_val = java_home_var.replace('\\\\', '\\') + os.system("SETX {0} {1} /M".format(JAVA_HOME, java_home_var_val)) + os.environ[JAVA_HOME] = java_home_var + pass + +# +# JDBC Setup +# + +def os_check_jdbc_options(options): + #Only SQL Server supported, no point in checking options.jdbc_db + return (options.jdbc_driver is not None) + +def os_setup_jdbc_drivers(args): + properties = get_ambari_properties() + if properties == -1: + print_error_msg("Error getting ambari properties") + return -1 + + #Only support SQL Server + dbms = DBMSConfig.create(args, properties) + if dbms.ensure_jdbc_driver_installed(args, properties): + # Now save the properties file + update_properties(properties) + pass + +def os_setup_database(options): + properties = get_ambari_properties() + if properties == -1: + raise FatalException(-1, "Error getting ambari properties") + + #Ensure the default database host is set + options.default_database_host = "localhost\\SQLEXPRESS" + + #Only support SQL Server + dbmsAmbari = DBMSConfig.create(options, properties, "Ambari") + resultA = dbmsAmbari.configure_database(options, properties) + + #By default, use the same server for Metrics + options.default_database_host = dbmsAmbari.database_host + + dbmsMetrics = DBMSConfig.create(options, properties, "Metrics") + resultM = dbmsMetrics.configure_database(options, properties) + + # Now save the properties file + if resultA or resultM: + update_properties(properties) + + dbmsAmbari.setup_database() + dbmsMetrics.setup_database() + +def os_reset_database(options): + properties = get_ambari_properties() + if properties == -1: + raise FatalException(-1, "Error getting ambari properties") + + if not (properties.getPropertyDict().has_key(JDBC_URL_PROPERTY) and + properties.getPropertyDict().has_key(JDBC_RCA_URL_PROPERTY) and + properties.getPropertyDict().has_key(JDBC_METRICS_URL_PROPERTY)): + raise FatalException(-1, "Ambari Server not set up yet. Nothing to reset.") + + empty_options = optparse.Values() + empty_options.silent = options.silent + empty_options.database_host = "" + empty_options.database_port = "" + empty_options.database_name = "" + empty_options.database_windows_auth = False + empty_options.database_username = "" + empty_options.database_password = "" + empty_options.init_db_script_file = "" + empty_options.cleanup_db_script_file = "" + empty_options.init_metrics_db_script_file = "" + empty_options.cleanup_metrics_db_script_file = "" + + #Only support SQL Server + dbmsAmbari = DBMSConfig.create(empty_options, properties, "Ambari") + dbmsAmbari.reset_database() + + dbmsMetrics = DBMSConfig.create(empty_options, properties, "Metrics") + dbmsMetrics.reset_database() + pass http://git-wip-us.apache.org/repos/asf/ambari/blob/fad56746/ambari-server/src/main/python/ambari_server/setupSecurity.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/setupSecurity.py b/ambari-server/src/main/python/ambari_server/setupSecurity.py index fd1ee8a..57a3231 100644 --- a/ambari-server/src/main/python/ambari_server/setupSecurity.py +++ b/ambari-server/src/main/python/ambari_server/setupSecurity.py @@ -19,32 +19,16 @@ limitations under the License. ''' import datetime import fileinput -import os import random -import re -import shutil import socket import stat -import string import sys -import tempfile -import time import urllib2 -from ambari_commons.exceptions import FatalException, NonFatalException -from ambari_commons.logging_utils import print_error_msg, print_info_msg, print_warning_msg, SILENT -from ambari_commons.os_utils import copy_file, remove_file, search_file, set_file_permissions, is_valid_filepath, \ - is_root, run_os_command -from ambari_server.serverConfiguration import find_properties_file, get_ambari_properties, get_value_from_properties, \ - get_conf_dir, get_full_ambari_classpath, read_ambari_user, find_jdk, get_prompt_default, update_properties_2, \ - configDefaults, \ - BOOTSTRAP_DIR_PROPERTY, GET_FQDN_SERVICE_URL, \ - JDBC_USE_INTEGRATED_AUTH_PROPERTY, JDBC_PASSWORD_PROPERTY, JDBC_PASSWORD_FILENAME, \ - JDBC_RCA_PASSWORD_ALIAS, JDBC_RCA_PASSWORD_FILE_PROPERTY, \ - JDBC_METRICS_USE_INTEGRATED_AUTH_PROPERTY, JDBC_METRICS_PASSWORD_PROPERTY, \ - JDBC_METRICS_PASSWORD_ALIAS, JDBC_METRICS_PASSWORD_FILENAME, BLIND_PASSWORD -from ambari_server.setupActions import SETUP_ACTION, LDAP_SETUP_ACTION -from ambari_server.userInput import get_validated_filepath_input, get_validated_string_input, get_YN_input +from ambari_commons.exceptions import * +from serverConfiguration import * +from setupActions import * +from userInput import * SSL_PASSWORD_FILE = "pass.txt" @@ -56,25 +40,36 @@ EXPRT_KSTR_CMD = "openssl pkcs12 -export -in '{0}' -inkey '{1}' -certfile '{0}' CHANGE_KEY_PWD_CND = 'openssl rsa -in {0} -des3 -out {0}.secured -passout pass:{1}' GET_CRT_INFO_CMD = 'openssl x509 -dates -subject -in {0}' -#keytool command -KEYTOOL_IMPORT_CERT_CMD = "{0}" + os.sep + "bin" + os.sep + configDefaults.keytool_bin + " -import -alias '{1}' -storetype '{2}' -file '{3}' -storepass '{4}' -noprompt" -KEYTOOL_DELETE_CERT_CMD = "{0}" + os.sep + "bin" + os.sep + configDefaults.keytool_bin + " -delete -alias '{1}' -storepass '{2}' -noprompt" +#keytool commands +keytool_bin = "keytool" +if OSCheck.is_windows_family(): + keytool_bin = "keytool.exe" + +KEYTOOL_IMPORT_CERT_CMD = "{0}" + os.sep + "bin" + os.sep + keytool_bin + " -import -alias '{1}' -storetype '{2}' -file '{3}' -storepass '{4}' -noprompt" +KEYTOOL_DELETE_CERT_CMD = "{0}" + os.sep + "bin" + os.sep + keytool_bin + " -delete -alias '{1}' -storepass '{2}' -noprompt" KEYTOOL_KEYSTORE = " -keystore '{0}'" -SECURITY_PROVIDER_GET_CMD = "{0}" + os.sep + configDefaults.JAVA_EXE_SUBPATH + " -cp {1} " +\ +java_bin = "java" +if OSCheck.is_windows_family(): + java_bin = "java.exe" + +SECURITY_PROVIDER_GET_CMD = "{0}" + os.sep + "bin" + os.sep + java_bin + " -cp {1}" +\ + os.pathsep + "{2} " +\ "org.apache.ambari.server.security.encryption" +\ - ".CredentialProvider GET {2} {3} {4} " +\ - "> " + configDefaults.SERVER_OUT_FILE + " 2>&1" + ".CredentialProvider GET {3} {4} {5} " +\ + "> " + SERVER_OUT_FILE + " 2>&1" -SECURITY_PROVIDER_PUT_CMD = "{0}" + os.sep + configDefaults.JAVA_EXE_SUBPATH + " -cp {1} " +\ +SECURITY_PROVIDER_PUT_CMD = "{0}" + os.sep + "bin" + os.sep + java_bin + " -cp {1}" +\ + os.pathsep + "{2} " +\ "org.apache.ambari.server.security.encryption" +\ - ".CredentialProvider PUT {2} {3} {4} " +\ - "> " + configDefaults.SERVER_OUT_FILE + " 2>&1" + ".CredentialProvider PUT {3} {4} {5} " +\ + "> " + SERVER_OUT_FILE + " 2>&1" -SECURITY_PROVIDER_KEY_CMD = "{0}" + os.sep + configDefaults.JAVA_EXE_SUBPATH + " -cp {1}" +\ +SECURITY_PROVIDER_KEY_CMD = "{0}" + os.sep + "bin" + os.sep + java_bin + " -cp {1}" +\ + os.pathsep + "{2} " +\ "org.apache.ambari.server.security.encryption" +\ - ".MasterKeyServiceImpl {2} {3} {4} " +\ - "> " + configDefaults.SERVER_OUT_FILE + " 2>&1" + ".MasterKeyServiceImpl {3} {4} {5} " +\ + "> " + SERVER_OUT_FILE + " 2>&1" SSL_KEY_DIR = 'security.server.keys_dir' SSL_API_PORT = 'client.api.ssl.port' @@ -526,7 +521,7 @@ def read_passwd_for_alias(alias, masterKey=""): if jdk_path is None: print_error_msg("No JDK found, please run the \"setup\" " "command to install a JDK automatically or install any " - "JDK manually to " + configDefaults.JDK_INSTALL_DIR) + "JDK manually to " + JDK_INSTALL_DIR) return 1 tempFileName = "ambari.passwd" @@ -541,7 +536,8 @@ def read_passwd_for_alias(alias, masterKey=""): if masterKey is None or masterKey == "": masterKey = "None" - command = SECURITY_PROVIDER_GET_CMD.format(jdk_path, get_full_ambari_classpath(), alias, tempFilePath, masterKey) + command = SECURITY_PROVIDER_GET_CMD.format(jdk_path, + get_conf_dir(), get_ambari_classpath(), alias, tempFilePath, masterKey) (retcode, stdout, stderr) = run_os_command(command) print_info_msg("Return code from credential provider get passwd: " + str(retcode)) @@ -578,13 +574,14 @@ def save_passwd_for_alias(alias, passwd, masterKey=""): if jdk_path is None: print_error_msg("No JDK found, please run the \"setup\" " "command to install a JDK automatically or install any " - "JDK manually to " + configDefaults.JDK_INSTALL_DIR) + "JDK manually to " + JDK_INSTALL_DIR) return 1 if masterKey is None or masterKey == "": masterKey = "None" - command = SECURITY_PROVIDER_PUT_CMD.format(jdk_path, get_full_ambari_classpath(), alias, passwd, masterKey) + command = SECURITY_PROVIDER_PUT_CMD.format(jdk_path, get_conf_dir(), + get_ambari_classpath(), alias, passwd, masterKey) (retcode, stdout, stderr) = run_os_command(command) print_info_msg("Return code from credential provider save passwd: " + str(retcode)) @@ -600,7 +597,7 @@ def get_is_persisted(properties): return (isPersisted, masterKeyFile) def get_credential_store_location(properties): - store_loc = get_value_from_properties(properties, SECURITY_KEYS_DIR, "") + store_loc = properties[SECURITY_KEYS_DIR] if store_loc is None or store_loc == "": store_loc = "/var/lib/ambari-server/keys/credentials.jceks" else: @@ -608,9 +605,9 @@ def get_credential_store_location(properties): return store_loc def get_master_key_location(properties): - keyLocation = get_value_from_properties(properties, SECURITY_MASTER_KEY_LOCATION, "") + keyLocation = properties[SECURITY_MASTER_KEY_LOCATION] if keyLocation is None or keyLocation == "": - keyLocation = get_value_from_properties(properties, SECURITY_KEYS_DIR, "") + keyLocation = properties[SECURITY_KEYS_DIR] return keyLocation def get_original_master_key(properties): @@ -682,9 +679,10 @@ def save_master_key(master_key, key_location, persist=True): if jdk_path is None: print_error_msg("No JDK found, please run the \"setup\" " "command to install a JDK automatically or install any " - "JDK manually to " + configDefaults.JDK_INSTALL_DIR) + "JDK manually to " + JDK_INSTALL_DIR) return 1 - command = SECURITY_PROVIDER_KEY_CMD.format(jdk_path, get_full_ambari_classpath(), master_key, key_location, persist) + command = SECURITY_PROVIDER_KEY_CMD.format(jdk_path, + get_ambari_classpath(), get_conf_dir(), master_key, key_location, persist) (retcode, stdout, stderr) = run_os_command(command) print_info_msg("Return code from credential provider save KEY: " + str(retcode)) @@ -724,35 +722,20 @@ def adjust_directory_permissions(ambari_user): bootstrap_dir = get_value_from_properties(properties, BOOTSTRAP_DIR_PROPERTY) print_info_msg("Cleaning bootstrap directory ({0}) contents...".format(bootstrap_dir)) shutil.rmtree(bootstrap_dir, True) #Ignore the non-existent dir error - - #makedirs may fail with "Access denied" if executed right after rmtree. The only known good solution is a delayed retry. - retry_makedirs = 0 - while True: - try: - os.makedirs(bootstrap_dir) - except OSError as e: - time.sleep(0.01) #Unnoticeable at this scale - retry_makedirs += 1 - if retry_makedirs == 3: - raise - continue - break - - ownership_list = configDefaults.NR_ADJUST_OWNERSHIP_LIST - + os.makedirs(bootstrap_dir) # Add master key and credential store if exists keyLocation = get_master_key_location(properties) masterKeyFile = search_file(SECURITY_MASTER_KEY_FILENAME, keyLocation) if masterKeyFile: - ownership_list.append((masterKeyFile, configDefaults.MASTER_KEY_FILE_PERMISSIONS, "{0}", "{0}", False)) + NR_ADJUST_OWNERSHIP_LIST.append((masterKeyFile, MASTER_KEY_FILE_PERMISSIONS, "{0}", "{0}", False)) credStoreFile = get_credential_store_location(properties) if os.path.exists(credStoreFile): - ownership_list.append((credStoreFile, configDefaults.CREDENTIALS_STORE_FILE_PERMISSIONS, "{0}", "{0}", False)) - trust_store_location = get_value_from_properties(properties, SSL_TRUSTSTORE_PATH_PROPERTY, None) + NR_ADJUST_OWNERSHIP_LIST.append((credStoreFile, CREDENTIALS_STORE_FILE_PERMISSIONS, "{0}", "{0}", False)) + trust_store_location = properties[SSL_TRUSTSTORE_PATH_PROPERTY] if trust_store_location: - ownership_list.append((trust_store_location, configDefaults.TRUST_STORE_LOCATION_PERMISSIONS, "{0}", "{0}", False)) + NR_ADJUST_OWNERSHIP_LIST.append((trust_store_location, TRUST_STORE_LOCATION_PERMISSIONS, "{0}", "{0}", False)) print "Adjusting ambari-server permissions and ownership..." - for pack in ownership_list: + for pack in NR_ADJUST_OWNERSHIP_LIST: file = pack[0] mod = pack[1] user = pack[2].format(ambari_user) @@ -846,7 +829,7 @@ def setup_component_https(component, command, property, alias): if jdk_path is None: err = "No JDK found, please run the \"ambari-server setup\" " \ "command to install a JDK automatically or install any " \ - "JDK manually to " + configDefaults.JDK_INSTALL_DIR + "JDK manually to " + JDK_INSTALL_DIR raise FatalException(1, err) properties = get_ambari_properties() @@ -996,7 +979,7 @@ def setup_master_key(): print_info_msg("Deleting master key file at location: " + str( masterKeyFile)) except Exception, e: - print 'ERROR: Could not remove master key file. %s' % str(e) + print 'ERROR: Could not remove master key file. %s' % e # Blow up the credential store made with previous key, if any store_file = get_credential_store_location(properties) if os.path.exists(store_file): http://git-wip-us.apache.org/repos/asf/ambari/blob/fad56746/ambari-server/src/main/python/ambari_server/userInput.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/userInput.py b/ambari-server/src/main/python/ambari_server/userInput.py index ff4c0c7..7a35831 100644 --- a/ambari-server/src/main/python/ambari_server/userInput.py +++ b/ambari-server/src/main/python/ambari_server/userInput.py @@ -22,7 +22,7 @@ import os import getpass import re -from ambari_commons.logging_utils import SILENT +from ambari_commons.logging_utils import * # http://git-wip-us.apache.org/repos/asf/ambari/blob/fad56746/ambari-server/src/main/resources/custom_actions/scripts/check_host.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py index f414d18..7430ba1 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py @@ -24,10 +24,9 @@ import os import subprocess import socket +from ambari_commons import os_utils from ambari_commons.os_check import OSCheck, OSConst -from ambari_commons.os_utils import quote_path from ambari_commons.inet_utils import download_file -from ambari_server.serverConfiguration import configDefaults from resource_management import Script, Execute, format from ambari_agent.HostInfo import HostInfo @@ -103,8 +102,11 @@ class CheckHost(Script): java64_home = config['commandParams']['java_home'] print "Java home to check: " + java64_home - - if not os.path.isfile(os.path.join(java64_home, configDefaults.JAVA_EXE_SUBPATH)): + java_bin = "java" + if OSCheck.is_windows_family(): + java_bin = "java.exe" + + if not os.path.isfile(os.path.join(java64_home, "bin", java_bin)): print "Java home doesn't exist!" java_home_check_structured_output = {"exit_code" : 1, "message": "Java home doesn't exist!"} else: @@ -150,9 +152,13 @@ class CheckHost(Script): jdbc_path = os.path.join(agent_cache_dir, jdbc_name) check_db_connection_path = os.path.join(agent_cache_dir, check_db_connection_jar_name) - class_path_delimiter = os.pathsep + java_bin = "java" + class_path_delimiter = ":" + if OSCheck.is_windows_family(): + java_bin = "java.exe" + class_path_delimiter = ";" - java_exec = os.path.join(java64_home, configDefaults.JAVA_EXE_SUBPATH) + java_exec = os.path.join(java64_home, "bin",java_bin) if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \ or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec): @@ -166,7 +172,7 @@ class CheckHost(Script): # download and install java if it doesn't exists if not os.path.isfile(java_exec): jdk_name = config['commandParams']['jdk_name'] - jdk_url = "{0}/{1}".format(jdk_location, jdk_name) + jdk_url = "{}/{}".format(jdk_location, jdk_name) jdk_download_target = os.path.join(agent_cache_dir, jdk_name) java_dir = os.path.dirname(java64_home) try: @@ -186,8 +192,8 @@ class CheckHost(Script): install_cmd = format("mkdir -p {java_dir} ; cd {java_dir} ; tar -xf {jdk_download_target} > /dev/null 2>&1") install_path = ["/bin","/usr/bin/"] elif jdk_name.endswith(".exe"): - install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format( - quote_path(jdk_download_target), quote_path(java64_home), + install_cmd = "{} /s INSTALLDIR={} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format( + os_utils.quote_path(jdk_download_target), os_utils.quote_path(java64_home), ) install_path = [java_dir]
