http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-server/src/main/python/ambari_server/dbConfiguration_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/dbConfiguration_windows.py b/ambari-server/src/main/python/ambari_server/dbConfiguration_windows.py index ab038db..58fa7c4 100644 --- a/ambari-server/src/main/python/ambari_server/dbConfiguration_windows.py +++ b/ambari-server/src/main/python/ambari_server/dbConfiguration_windows.py @@ -21,12 +21,12 @@ limitations under the License. import os import socket import string - from ambari_commons.exceptions import FatalException from ambari_commons.logging_utils import print_info_msg, print_warning_msg from ambari_commons.os_utils import search_file, run_os_command from ambari_commons.os_windows import WinServiceController from ambari_commons.str_utils import compress_backslashes, ensure_double_backslashes +from ambari_server.dbConfiguration import AMBARI_DATABASE_NAME, DEFAULT_USERNAME, DBMSConfig, DbPropKeys, DbAuthenticationKeys from ambari_server.serverConfiguration import JDBC_DRIVER_PROPERTY, JDBC_DRIVER_PATH_PROPERTY, JDBC_URL_PROPERTY, \ JDBC_DATABASE_PROPERTY, JDBC_DATABASE_NAME_PROPERTY, \ JDBC_HOSTNAME_PROPERTY, JDBC_PORT_PROPERTY, JDBC_USE_INTEGRATED_AUTH_PROPERTY, JDBC_USER_NAME_PROPERTY, JDBC_PASSWORD_PROPERTY, \ @@ -35,59 +35,35 @@ from ambari_server.serverConfiguration import JDBC_DRIVER_PROPERTY, JDBC_DRIVER_ JDBC_RCA_HOSTNAME_PROPERTY, JDBC_RCA_PORT_PROPERTY, JDBC_RCA_USE_INTEGRATED_AUTH_PROPERTY, \ JDBC_RCA_USER_NAME_PROPERTY, JDBC_RCA_PASSWORD_FILE_PROPERTY, JDBC_RCA_PASSWORD_FILENAME, JDBC_RCA_PASSWORD_ALIAS, \ PERSISTENCE_TYPE_PROPERTY, \ - PRESS_ENTER_MSG + get_value_from_properties, configDefaults from ambari_server.setupSecurity import encrypt_password, store_password_file -from dbConfiguration import DBMSConfig, DB_STATUS_RUNNING_DEFAULT -from userInput import get_validated_string_input +from ambari_server.userInput import get_validated_string_input -#Import the SQL Server libraries # SQL Server settings -DBPATH = 'C:\\Program Files\\Microsoft SQL Server\\MSSQL12.SQLEXPRESS\\MSSQL\\DATA\\' -# DBPATH = 'C:\\Program Files\\Microsoft SQL Server\\MSSQL10_50.MSSQLSERVER\\MSSQL\\DATA\\' - -DATABASE_DBMS = "sqlserver" -DATABASE_DRIVER_NAME = "com.microsoft.sqlserver.jdbc.SQLServerDriver" -LOCAL_DATABASE_SERVER = "localhost\\SQLEXPRESS" -AMBARI_DATABASE_NAME = "ambari" - -class DbPropKeys: - def __init__(self, i_dbms_key, i_driver_key, i_server_key, i_port_key, i_db_name_key, i_db_url_key): - self.reset(i_dbms_key, i_driver_key, i_server_key, i_port_key, i_db_name_key, i_db_url_key) - pass - - def reset(self, i_dbms_key, i_driver_key, i_server_key, i_port_key, i_db_name_key, i_db_url_key): - self.dbms_key = i_dbms_key - self.driver_key = i_driver_key - self.server_key = i_server_key - self.port_key = i_port_key - self.db_name_key = i_db_name_key - self.db_url_key = i_db_url_key - pass +DATABASE_DBMS_SQLSERVER = "sqlserver" +DATABASE_DRIVER_NAME_SQLSERVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver" +DATABASE_SERVER_SQLSERVER_DEFAULT = "localhost\\SQLEXPRESS" -class AuthenticationKeys: +class SqlServerAuthenticationKeys(DbAuthenticationKeys): def __init__(self, i_integrated_auth_key, i_user_name_key, i_password_key, i_password_alias, i_password_filename): - self.reset(i_integrated_auth_key, i_user_name_key, i_password_key, i_password_alias, i_password_filename) - pass - - def reset(self, i_integrated_auth_key, i_user_name_key, i_password_key, i_password_alias, i_password_filename): self.integrated_auth_key = i_integrated_auth_key - self.user_name_key = i_user_name_key - self.password_key = i_password_key - self.password_alias = i_password_alias - self.password_filename = i_password_filename - pass + DbAuthenticationKeys.__init__(self, i_user_name_key, i_password_key, i_password_alias, i_password_filename) +# # SQL Server configuration and setup +# class SQLServerConfig(DBMSConfig): - def __init__(self, options, properties): - super(SQLServerConfig, self).__init__(options, properties) + def __init__(self, options, properties, storage_type): + super(SQLServerConfig, self).__init__(options, properties, storage_type) """ #Just load the defaults. The derived classes will be able to modify them later """ - self.dbms = DATABASE_DBMS - self.driver_name = DATABASE_DRIVER_NAME + self.dbms = DATABASE_DBMS_SQLSERVER + self.driver_class_name = DATABASE_DRIVER_NAME_SQLSERVER + + self.JDBC_DRIVER_INSTALL_MSG = 'Before starting Ambari Server, you must install the SQL Server JDBC driver.' # The values from options supersede the values from properties self.database_host = options.database_host if options.database_host is not None and options.database_host is not "" else \ @@ -98,27 +74,26 @@ class SQLServerConfig(DBMSConfig): else: self.database_host = compress_backslashes(self.database_host) except: - self.database_host = "localhost\\SQLEXPRESS" + self.database_host = DATABASE_SERVER_SQLSERVER_DEFAULT pass - self.database_port = options.database_port if options.database_port is not None and options.database_port is not "" else \ - properties.get_property(self.dbPropKeys.port_key) - self.database_name = options.database_name if options.database_name is not None and options.database_name is not "" else \ - properties.get_property(self.dbPropKeys.db_name_key) - - self.use_windows_authentication = options.database_windows_auth if options.database_windows_auth is True else \ - properties.get_property(self.dbAuthKeys.integrated_auth_key) - self.database_username = options.database_username if options.database_username is not None and options.database_username is not "" \ - else properties.get_property(self.dbAuthKeys.user_name_key) + self.database_port = DBMSConfig._init_member_with_prop_default(options, "database_port", + properties, self.dbPropKeys.port_key, "1433") + self.database_name = DBMSConfig._init_member_with_prop_default(options, "database_name", + properties, self.dbPropKeys.db_name_key, configDefaults.DEFAULT_DB_NAME) + + self.use_windows_authentication = DBMSConfig._init_member_with_prop_default(options, "database_windows_auth", + properties, self.dbAuthKeys.integrated_auth_key, False) + self.database_username = DBMSConfig._init_member_with_prop_default(options, "database_username", + properties, self.dbAuthKeys.user_name_key, DEFAULT_USERNAME) self.database_password = options.database_password if options.database_password is not None and options.database_password is not "" \ - else "" - self.password_file = properties[self.dbAuthKeys.password_key] + else "" + if not self.database_password: + self.database_password = DBMSConfig._read_password_from_properties(properties) self.database_url = self._build_sql_server_connection_string() self.persistence_property = None - self.db_title = "" - self.env_var_db_name = "" self.env_var_db_log_name = "" self.env_var_db_owner = "" @@ -132,137 +107,113 @@ class SQLServerConfig(DBMSConfig): def _is_local_database(self): return False - def _is_jdbc_driver_installed(self, properties): - """ - #Attempt to load the sqljdbc4.jar and sqljdbc_auth.dll. This will automatically scan the PATH. - :param None - :rtype : bool - """ - paths = "." + os.pathsep + os.environ["PATH"] - - # Find the jar by attempting to load it as a resource dll - driver_path = search_file("sqljdbc4.jar", paths) - if not driver_path: - return 0 - - auth_dll_path = search_file("sqljdbc_auth.dll", paths) - if not auth_dll_path: - return 0 - - try: - driver_path = properties[JDBC_DRIVER_PATH_PROPERTY] - if driver_path is None or driver_path is "": - return 0 - except Exception: - # No such attribute set - return 0 - - return 1 - - def get_jdbc_driver_path(self): - paths = "." + os.pathsep + os.environ["PATH"] - - # Find the jar by attempting to load it as a resource dll - driver_path = search_file("sqljdbc4.jar", paths) - return driver_path - - def configure_database_password(showDefault=True): + def _configure_database_password(showDefault=True): #No password needed, using SQL Server integrated authentication pass def _prompt_db_properties(self): - if self.silent: - # All the settings are supposed to be retrieved from the command-line parameters - return True - - #prompt for SQL Server host and instance name - hostname_prompt = "SQL Server host and instance for the {} database: ({}) ".format(self.db_title, self.database_host) - self.database_host = get_validated_string_input(hostname_prompt, self.database_host, None, None, False, True) - - #prompt for SQL Server authentication method - if (not self.use_windows_authentication is None and self.use_windows_authentication.lower() == "true") or \ - self.database_username is None or self.database_username == "": - auth_option_default = '1' - else: - auth_option_default = '2' - - user_prompt = \ - "[1] - Use SQL Server integrated authentication\n[2] - Use username+password authentication\n" \ - "Enter choice ({}): ".format(auth_option_default) - auth_option = get_validated_string_input(user_prompt, - auth_option_default, - "^[12]$", - "Invalid number.", - False - ) - if str(auth_option) == '1': - self.use_windows_authentication = True - self.database_password = None - else: - self.use_windows_authentication = False + if self.must_set_database_options: + #prompt for SQL Server host and instance name + hostname_prompt = "SQL Server host and instance for the {0} database: ({1}) ".format(self.db_title, self.database_host) + self.database_host = get_validated_string_input(hostname_prompt, self.database_host, None, None, False, True) + + #prompt for SQL Server authentication method + if (not self.use_windows_authentication is None and self.use_windows_authentication.lower() == "true") or \ + self.database_username is None or self.database_username == "": + auth_option_default = '1' + else: + auth_option_default = '2' + + user_prompt = \ + "[1] - Use SQL Server integrated authentication\n[2] - Use username+password authentication\n" \ + "Enter choice ({0}): ".format(auth_option_default) + auth_option = get_validated_string_input(user_prompt, + auth_option_default, + "^[12]$", + "Invalid number.", + False + ) + if str(auth_option) == '1': + self.use_windows_authentication = True + self.database_password = None + else: + self.use_windows_authentication = False - user_prompt = "SQL Server user name for the {} database: ({}) ".format(self.db_title, self.database_username) - username = get_validated_string_input(user_prompt, self.database_username, None, "User name", False, - False) - self.database_username = username + user_prompt = "SQL Server user name for the {0} database: ({1}) ".format(self.db_title, self.database_username) + username = get_validated_string_input(user_prompt, self.database_username, None, "User name", False, + False) + self.database_username = username - user_prompt = "SQL Server password for the {} database: ".format(self.db_title) - password = get_validated_string_input(user_prompt, "", None, "Password", True, False) - self.database_password = password + user_prompt = "SQL Server password for the {0} database: ".format(self.db_title) + password = get_validated_string_input(user_prompt, "", None, "Password", True, False) + self.database_password = password self.database_url = self._build_sql_server_connection_string() return True def _setup_remote_server(self, properties): - properties.removeOldProp(self.dbPropKeys.port_key) - properties.removeOldProp(self.dbAuthKeys.integrated_auth_key) - properties.removeOldProp(self.dbAuthKeys.user_name_key) - properties.removeOldProp(self.dbAuthKeys.password_key) + if self.ensure_jdbc_driver_installed(properties): + properties.removeOldProp(self.dbPropKeys.port_key) + properties.removeOldProp(self.dbAuthKeys.integrated_auth_key) + properties.removeOldProp(self.dbAuthKeys.user_name_key) + properties.removeOldProp(self.dbAuthKeys.password_key) - properties.process_pair(self.persistence_property, 'remote') + properties.process_pair(self.persistence_property, 'remote') - properties.process_pair(self.dbPropKeys.dbms_key, self.dbms) - properties.process_pair(self.dbPropKeys.driver_key, self.driver_name) - properties.process_pair(self.dbPropKeys.server_key, ensure_double_backslashes(self.database_host)) - if self.database_port is not None and self.database_port != "": - properties.process_pair(self.dbPropKeys.port_key, self.database_port) - properties.process_pair(self.dbPropKeys.db_name_key, self.database_name) + properties.process_pair(self.dbPropKeys.dbms_key, self.dbms) + properties.process_pair(self.dbPropKeys.driver_key, self.driver_class_name) + properties.process_pair(self.dbPropKeys.server_key, ensure_double_backslashes(self.database_host)) + if self.database_port is not None and self.database_port != "": + properties.process_pair(self.dbPropKeys.port_key, self.database_port) + properties.process_pair(self.dbPropKeys.db_name_key, self.database_name) - self._store_db_auth_config(properties, self.dbAuthKeys) + self._store_db_auth_config(properties, self.dbAuthKeys) - properties.process_pair(self.dbPropKeys.db_url_key, self.database_url) + properties.process_pair(self.dbPropKeys.db_url_key, self.database_url) pass def _setup_remote_database(self): - print 'Populating {} database structure...'.format(self.db_title) + print 'Populating the {0} database structure...'.format(self.db_title) self._populate_database_structure() def _reset_remote_database(self): - print 'Resetting {} database structure...'.format(self.db_title) + print 'Resetting the {0} database structure...'.format(self.db_title) self._populate_database_structure() - def _prompt_jdbc_driver_install(self, properties): - result = False - msg = 'Before starting Ambari Server, you must install the SQL Server JDBC driver.' + def _is_jdbc_driver_installed(self, properties): + """ + #Attempt to find the sqljdbc4.jar and sqljdbc_auth.dll by scanning the PATH. + :param None + :rtype : bool + """ + paths = "." + os.pathsep + os.environ["PATH"] - if not self.silent: - print_warning_msg(msg) - raw_input(PRESS_ENTER_MSG) - result = self._is_jdbc_driver_installed(properties) - return (result, msg) + # Find the jar by attempting to load it as a resource dll + driver_path = search_file("sqljdbc4.jar", paths) + if not driver_path: + return 0 + + auth_dll_path = search_file("sqljdbc_auth.dll", paths) + if not auth_dll_path: + return 0 - def _install_jdbc_driver(self, options, properties): try: driver_path = properties[JDBC_DRIVER_PATH_PROPERTY] + if driver_path is None or driver_path is "": + return 0 except Exception: # No such attribute set - driver_path = None + return 0 + return 1 + + def _install_jdbc_driver(self, properties, files_list): + driver_path = get_value_from_properties(properties, JDBC_DRIVER_PATH_PROPERTY, None) if driver_path is None or driver_path == "": - driver_path = self.get_jdbc_driver_path() + driver_path = self._get_jdbc_driver_path() properties.process_pair(JDBC_DRIVER_PATH_PROPERTY, driver_path) return True @@ -283,7 +234,7 @@ class SQLServerConfig(DBMSConfig): sql_svc_name = "MSSQL$" + db_host_components[1] if db_machine == "localhost" or db_machine.lower() == os.getenv("COMPUTERNAME").lower() or \ - db_machine.lower() == socket.getfqdn().lower(): + db_machine.lower() == socket.getfqdn().lower(): #TODO: Configure the SQL Server service name in ambari.properties ret = WinServiceController.EnsureServiceIsStarted(sql_svc_name) if 0 != ret: @@ -298,11 +249,18 @@ class SQLServerConfig(DBMSConfig): pass + def _get_jdbc_driver_path(self): + paths = "." + os.pathsep + os.environ["PATH"] + + # Find the jar in the PATH + driver_path = search_file("sqljdbc4.jar", paths) + return driver_path + def _build_sql_server_connection_string(self): - databaseUrl = "jdbc:sqlserver://{}".format(ensure_double_backslashes(self.database_host)) + databaseUrl = "jdbc:sqlserver://{0}".format(ensure_double_backslashes(self.database_host)) if self.database_port is not None and self.database_port != "": - databaseUrl += ":{}".format(self.database_port) - databaseUrl += ";databaseName={}".format(self.database_name) + databaseUrl += ":{0}".format(self.database_port) + databaseUrl += ";databaseName={0}".format(self.database_name) if(self.use_windows_authentication): databaseUrl += ";integratedSecurity=true" #No need to append the username and password, the Ambari server adds them by itself when connecting to the database @@ -340,18 +298,20 @@ class SQLServerConfig(DBMSConfig): @staticmethod def _execute_db_script(databaseHost, databaseScript): - dbCmd = 'sqlcmd -S {} -i {}'.format(databaseHost, databaseScript) + dbCmd = 'sqlcmd -S {0} -i {1}'.format(databaseHost, databaseScript) retCode, outData, errData = run_os_command(['cmd', '/C', dbCmd]) if not retCode == 0: - err = 'Running database create script failed. Error output: {} Output: {} Exiting.'.format(errData, outData) + err = 'Running database create script failed. Error output: {0} Output: {1} Exiting.'.format(errData, outData) raise FatalException(retCode, err) print_info_msg("sqlcmd output:") print_info_msg(outData) pass +# # SQL Server Ambari database configuration and setup +# class SQLServerAmbariDBConfig(SQLServerConfig): - def __init__(self, options, properties): + def __init__(self, options, properties, storage_type): self.dbPropKeys = DbPropKeys( JDBC_DATABASE_PROPERTY, JDBC_DRIVER_PROPERTY, @@ -359,7 +319,7 @@ class SQLServerAmbariDBConfig(SQLServerConfig): JDBC_PORT_PROPERTY, JDBC_DATABASE_NAME_PROPERTY, JDBC_URL_PROPERTY) - self.dbAuthKeys = AuthenticationKeys( + self.dbAuthKeys = SqlServerAuthenticationKeys( JDBC_USE_INTEGRATED_AUTH_PROPERTY, JDBC_USER_NAME_PROPERTY, JDBC_PASSWORD_PROPERTY, @@ -367,15 +327,13 @@ class SQLServerAmbariDBConfig(SQLServerConfig): JDBC_PASSWORD_FILENAME ) - super(SQLServerAmbariDBConfig, self).__init__(options, properties) + super(SQLServerAmbariDBConfig, self).__init__(options, properties, storage_type) if self.database_name is None or self.database_name is "": self.database_name = AMBARI_DATABASE_NAME self.persistence_property = PERSISTENCE_TYPE_PROPERTY - self.db_title = "ambari" - self.env_var_db_name ='AMBARIDBNAME' self.env_var_db_log_name = 'AMBARIDBLOGNAME' self.env_var_db_owner = 'AMBARIDBOWNER' @@ -394,13 +352,12 @@ class SQLServerAmbariDBConfig(SQLServerConfig): def _setup_remote_server(self, properties): super(SQLServerAmbariDBConfig, self)._setup_remote_server(properties) - properties.process_pair(JDBC_RCA_DRIVER_PROPERTY, self.driver_name) + properties.process_pair(JDBC_RCA_DRIVER_PROPERTY, self.driver_class_name) properties.process_pair(JDBC_RCA_HOSTNAME_PROPERTY, ensure_double_backslashes(self.database_host)) if self.database_port is not None and self.database_port != "": properties.process_pair(JDBC_RCA_PORT_PROPERTY, self.database_port) - properties.process_pair(JDBC_RCA_SCHEMA_PROPERTY, self.database_name) - authKeys = AuthenticationKeys( + authKeys = SqlServerAuthenticationKeys( JDBC_RCA_USE_INTEGRATED_AUTH_PROPERTY, JDBC_RCA_USER_NAME_PROPERTY, JDBC_RCA_PASSWORD_FILE_PROPERTY, @@ -412,13 +369,6 @@ class SQLServerAmbariDBConfig(SQLServerConfig): properties.process_pair(JDBC_RCA_URL_PROPERTY, self.database_url) pass -# SQL Server database -class SQLServerDatabase: - def __init__(self): - #Init the database connection here - pass - def get_running_status(self): - #if the connection is active, return running - #else return stopped - return DB_STATUS_RUNNING_DEFAULT +def createSQLServerConfig(options, properties, storage_type, dbId): + return SQLServerAmbariDBConfig(options, properties, storage_type)
http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-server/src/main/python/ambari_server/serverConfiguration.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py index 5547e44..6b3f350 100644 --- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py +++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py @@ -23,21 +23,27 @@ import glob import os import re import shutil +import stat import string +import sys import tempfile from ambari_commons.exceptions import FatalException from ambari_commons.os_check import OSCheck, OSConst from ambari_commons.os_family_impl import OsFamilyImpl from ambari_commons.os_utils import run_os_command, search_file -from ambari_commons.logging_utils import print_warning_msg, print_info_msg, print_error_msg -from properties import Properties +from ambari_commons.logging_utils import get_debug_mode, print_info_msg, print_warning_msg, print_error_msg, \ + set_debug_mode +from ambari_server.properties import Properties +from ambari_server.userInput import get_validated_string_input +from ambari_server.utils import compare_versions, locate_file OS_VERSION = OSCheck().get_os_major_version() OS_TYPE = OSCheck.get_os_type() OS_FAMILY = OSCheck.get_os_family() +PID_NAME = "ambari-server.pid" # Non-root user setup commands NR_USER_PROPERTY = "ambari-server.user" @@ -56,10 +62,6 @@ BOOTSTRAP_DIR_PROPERTY = "bootstrap.dir" AMBARI_CONF_VAR = "AMBARI_CONF_DIR" AMBARI_PROPERTIES_FILE = "ambari.properties" -if OSCheck.is_windows_family(): - AMBARI_PROPERTIES_BACKUP_FILE = "ambari.properties.backup" -else: - AMBARI_PROPERTIES_BACKUP_FILE = "ambari.properties.rpmsave" GET_FQDN_SERVICE_URL = "server.fqdn.service.url" @@ -85,10 +87,10 @@ JDK_NAMES = ["jdk-7u67-linux-x64.tar.gz", "jdk-6u31-linux-x64.bin"] #JCE Policy files JCE_POLICY_FILENAMES = ["UnlimitedJCEPolicyJDK7.zip", "jce_policy-6.zip"] -JCE_DOWNLOAD_CMD = "curl -o {0} {1}" -JCE_MIN_FILESIZE = 5000 # JDBC +JDBC_PATTERNS = {"oracle": "*ojdbc*.jar", "mysql": "*mysql*.jar"} + #TODO property used incorrectly in local case, it was meant to be dbms name, not postgres database name, # has workaround for now, as we don't need dbms name if persistence_type=local JDBC_DATABASE_PROPERTY = "server.jdbc.database" # E.g., embedded|oracle|mysql|postgres|sqlserver @@ -105,6 +107,8 @@ JDBC_RCA_PASSWORD_FILENAME = "rca_password.dat" CLIENT_API_PORT_PROPERTY = "client.api.port" CLIENT_API_PORT = "8080" +SERVER_VERSION_FILE_PATH = "server.version.file" + PERSISTENCE_TYPE_PROPERTY = "server.persistence.type" JDBC_DRIVER_PROPERTY = "server.jdbc.driver" JDBC_DRIVER_PATH_PROPERTY = "server.jdbc.driver.path" @@ -122,12 +126,16 @@ JDBC_RCA_PASSWORD_FILE_PROPERTY = "server.jdbc.rca.user.passwd" JDBC_RCA_PASSWORD_ALIAS = "ambari.db.password" -#Windows-specific settings +### # Windows-specific # ### JDBC_USE_INTEGRATED_AUTH_PROPERTY = "server.jdbc.use.integrated.auth" JDBC_RCA_USE_INTEGRATED_AUTH_PROPERTY = "server.jdbc.rca.use.integrated.auth" +### # End Windows-specific # ### + +SERVICE_USERNAME_KEY = "TMP_AMBARI_USERNAME" +SERVICE_PASSWORD_KEY = "TMP_AMBARI_PASSWORD" # resources repo configuration RESOURCES_DIR_PROPERTY = "resources.dir" @@ -135,11 +143,26 @@ RESOURCES_DIR_DEFAULT = "resources" # stack repo upgrade STACK_LOCATION_KEY = 'metadata.path' -STACK_LOCATION_DEFAULT = "resources" + os.sep + "stacks" + +# LDAP security +IS_LDAP_CONFIGURED = "ambari.ldap.isConfigured" +LDAP_MGR_PASSWORD_ALIAS = "ambari.ldap.manager.password" +LDAP_MGR_PASSWORD_PROPERTY = "authentication.ldap.managerPassword" +LDAP_MGR_PASSWORD_FILENAME = "ldap-password.dat" +LDAP_MGR_USERNAME_PROPERTY = "authentication.ldap.managerDn" +LDAP_PRIMARY_URL_PROPERTY = "authentication.ldap.primaryUrl" + +# SSL truststore +SSL_TRUSTSTORE_PASSWORD_ALIAS = "ambari.ssl.trustStore.password" +SSL_TRUSTSTORE_PATH_PROPERTY = "ssl.trustStore.path" +SSL_TRUSTSTORE_PASSWORD_PROPERTY = "ssl.trustStore.password" +SSL_TRUSTSTORE_TYPE_PROPERTY = "ssl.trustStore.type" # JDK JDK_RELEASES="java.releases" +VIEWS_DIR_PROPERTY = "views.dir" + #Common setup or upgrade message SETUP_OR_UPGRADE_MSG = "- If this is a new setup, then run the \"ambari-server setup\" command to create the user\n" \ "- If this is an upgrade of an existing setup, run the \"ambari-server upgrade\" command.\n" \ @@ -147,7 +170,6 @@ SETUP_OR_UPGRADE_MSG = "- If this is a new setup, then run the \"ambari-server s DEFAULT_DB_NAME = "ambari" - class ServerConfigDefaults(object): def __init__(self): self.JAVA_SHARE_PATH = "/usr/share/java" @@ -165,11 +187,14 @@ class ServerConfigDefaults(object): self.PID_DIR = os.sep + os.path.join("var", "run", "ambari-server") self.DEFAULT_LIBS_DIR = "" + self.AMBARI_PROPERTIES_BACKUP_FILE = "" + # ownership/permissions mapping # path - permissions - user - group - recursive # Rules are executed in the same order as they are listed # {0} in user/group will be replaced by customized ambari-server username self.NR_ADJUST_OWNERSHIP_LIST = [] + self.NR_USERADD_CMD = "" self.MASTER_KEY_FILE_PERMISSIONS = "600" self.CREDENTIALS_STORE_FILE_PERMISSIONS = "600" @@ -177,11 +202,19 @@ class ServerConfigDefaults(object): self.DEFAULT_DB_NAME = "ambari" + self.STACK_LOCATION_DEFAULT = "" + self.DEFAULT_VIEWS_DIR = "" #keytool commands self.keytool_bin = "" + #Standard messages + self.MESSAGE_ERROR_SETUP_NOT_ROOT = "" + self.MESSAGE_ERROR_RESET_NOT_ROOT = "" + self.MESSAGE_ERROR_UPGRADE_NOT_ROOT = "" + self.MESSAGE_CHECK_FIREWALL = "" + @OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) class ServerConfigDefaultsWindows(ServerConfigDefaults): def __init__(self): @@ -194,6 +227,8 @@ class ServerConfigDefaultsWindows(ServerConfigDefaults): self.DEFAULT_CONF_DIR = "conf" self.DEFAULT_LIBS_DIR = "lib" + self.AMBARI_PROPERTIES_BACKUP_FILE = "ambari.properties.backup" + # ownership/permissions mapping # path - permissions - user - group - recursive # Rules are executed in the same order as they are listed @@ -218,12 +253,21 @@ class ServerConfigDefaultsWindows(ServerConfigDefaults): # Also, /etc/ambari-server/conf/password.dat # is generated later at store_password_file ] + self.NR_USERADD_CMD = "cmd /C net user {0} {1} /ADD" + + self.STACK_LOCATION_DEFAULT = "resources\\stacks" self.DEFAULT_VIEWS_DIR = "resources\\views" #keytool commands self.keytool_bin = "keytool.exe" + #Standard messages + self.MESSAGE_ERROR_SETUP_NOT_ROOT = "Ambari-server setup must be run with administrator-level privileges" + self.MESSAGE_ERROR_RESET_NOT_ROOT = "Ambari-server reset must be run with administrator-level privileges" + self.MESSAGE_ERROR_UPGRADE_NOT_ROOT = "Ambari-server upgrade must be run with administrator-level privileges" + self.MESSAGE_CHECK_FIREWALL = "Checking firewall status..." + @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) class ServerConfigDefaultsLinux(ServerConfigDefaults): def __init__(self): @@ -237,6 +281,8 @@ class ServerConfigDefaultsLinux(ServerConfigDefaults): self.DEFAULT_CONF_DIR = "/etc/ambari-server/conf" self.DEFAULT_LIBS_DIR = "/usr/lib/ambari-server" + self.AMBARI_PROPERTIES_BACKUP_FILE = "ambari.properties.rpmsave" + # ownership/permissions mapping # path - permissions - user - group - recursive # Rules are executed in the same order as they are listed @@ -271,18 +317,48 @@ class ServerConfigDefaultsLinux(ServerConfigDefaults): # Also, /etc/ambari-server/conf/password.dat # is generated later at store_password_file ] + self.NR_USERADD_CMD = 'useradd -M --comment "{1}" ' \ + '--shell %s -d /var/lib/ambari-server/keys/ {0}' % locate_file('nologin', '/sbin') + + self.STACK_LOCATION_DEFAULT = "/var/lib/ambari-server/resources/stacks" self.DEFAULT_VIEWS_DIR = "/var/lib/ambari-server/resources/views" #keytool commands self.keytool_bin = "keytool" + #Standard messages + self.MESSAGE_ERROR_SETUP_NOT_ROOT = "Ambari-server setup should be run with root-level privileges" + self.MESSAGE_ERROR_RESET_NOT_ROOT = "Ambari-server reset should be run with root-level privileges" + self.MESSAGE_ERROR_UPGRADE_NOT_ROOT = "Ambari-server upgrade must be run with root-level privileges" + self.MESSAGE_CHECK_FIREWALL = "Checking iptables..." + configDefaults = ServerConfigDefaults() +# Security +SECURITY_KEYS_DIR = "security.server.keys_dir" +SECURITY_MASTER_KEY_LOCATION = "security.master.key.location" +SECURITY_KEY_IS_PERSISTED = "security.master.key.ispersisted" +SECURITY_KEY_ENV_VAR_NAME = "AMBARI_SECURITY_MASTER_KEY" +SECURITY_MASTER_KEY_FILENAME = "master" +SECURITY_IS_ENCRYPTION_ENABLED = "security.passwords.encryption.enabled" +SECURITY_KERBEROS_JASS_FILENAME = "krb5JAASLogin.conf" + +SECURITY_PROVIDER_GET_CMD = "{0} -cp {1}" + \ + "org.apache.ambari.server.security.encryption" + \ + ".CredentialProvider GET {2} {3} {4} " + \ + "> " + configDefaults.SERVER_OUT_FILE + " 2>&1" + +SECURITY_PROVIDER_PUT_CMD = "{0} -cp {1}" + \ + "org.apache.ambari.server.security.encryption" + \ + ".CredentialProvider PUT {2} {3} {4} " + \ + "> " + configDefaults.SERVER_OUT_FILE + " 2>&1" + +SECURITY_PROVIDER_KEY_CMD = "{0} -cp {1}" + \ + "org.apache.ambari.server.security.encryption" + \ + ".MasterKeyServiceImpl {2} {3} {4} " + \ + "> " + configDefaults.SERVER_OUT_FILE + " 2>&1" -SCHEMA_UPGRADE_HELPER_CMD = "{0} -cp {1} " + \ - "org.apache.ambari.server.upgrade.SchemaUpgradeHelper" + \ - " > " + configDefaults.SERVER_OUT_FILE + " 2>&1" def get_conf_dir(): @@ -322,18 +398,12 @@ def read_ambari_user(): ''' Reads ambari user from properties file ''' - conf_file = find_properties_file() - try: - properties = Properties() - properties.load(open(conf_file)) + properties = get_ambari_properties() + if properties != -1: user = properties[NR_USER_PROPERTY] if user: return user - else: - return None - except Exception, e: - print_error_msg('Could not read "%s": %s' % (conf_file, e)) - return None + return None def get_value_from_properties(properties, key, default=""): try: @@ -344,6 +414,32 @@ def get_value_from_properties(properties, key, default=""): return default return value +def get_is_secure(properties): + isSecure = properties.get_property(SECURITY_IS_ENCRYPTION_ENABLED) + isSecure = True if isSecure and isSecure.lower() == 'true' else False + return isSecure + +def get_is_persisted(properties): + keyLocation = get_master_key_location(properties) + masterKeyFile = search_file(SECURITY_MASTER_KEY_FILENAME, keyLocation) + isPersisted = True if masterKeyFile else False + + return (isPersisted, masterKeyFile) + +def get_credential_store_location(properties): + store_loc = properties[SECURITY_KEYS_DIR] + if store_loc is None or store_loc == "": + store_loc = "/var/lib/ambari-server/keys/credentials.jceks" + else: + store_loc += os.sep + "credentials.jceks" + return store_loc + +def get_master_key_location(properties): + keyLocation = properties[SECURITY_MASTER_KEY_LOCATION] + if keyLocation is None or keyLocation == "": + keyLocation = properties[SECURITY_KEYS_DIR] + return keyLocation + # Copy file to /tmp and save with file.# (largest # is latest file) def backup_file_in_temp(filePath): if filePath is not None: @@ -353,10 +449,99 @@ def backup_file_in_temp(filePath): shutil.copyfile(filePath, tmpDir + os.sep + AMBARI_PROPERTIES_FILE + "." + str(back_up_file_count + 1)) except (Exception), e: - print_error_msg('Could not backup file in temp "%s": %s' % (str( - back_up_file_count, e))) + print_error_msg('Could not backup file in temp "%s": %s' % ( + back_up_file_count, str(e))) return 0 +def get_ambari_version(properties): + """ + :param properties: Ambari properties + :return: Return a string of the ambari version. When comparing versions, please use "compare_versions" function. + """ + version = None + try: + server_version_file_path = properties[SERVER_VERSION_FILE_PATH] + if server_version_file_path and os.path.exists(server_version_file_path): + with open(server_version_file_path, 'r') as file: + version = file.read().strip() + except: + print_error_msg("Error getting ambari version") + return version + +def get_db_type(properties): + db_type = None + if properties[JDBC_URL_PROPERTY]: + jdbc_url = properties[JDBC_URL_PROPERTY].lower() + if "postgres" in jdbc_url: + db_type = "postgres" + elif "oracle" in jdbc_url: + db_type = "oracle" + elif "mysql" in jdbc_url: + db_type = "mysql" + elif "derby" in jdbc_url: + db_type = "derby" + + return db_type + +def check_database_name_property(upgrade=False): + """ + :param upgrade: If Ambari is being upgraded. + :return: + """ + properties = get_ambari_properties() + if properties == -1: + print_error_msg("Error getting ambari properties") + return -1 + + version = get_ambari_version(properties) + if upgrade and compare_versions(version, "1.7.0") == 0: + # This code exists for historic reasons in which property names changed from Ambari 1.6.1 to 1.7.0 + persistence_type = properties[PERSISTENCE_TYPE_PROPERTY] + if persistence_type == "remote": + db_name = properties["server.jdbc.schema"] # this was a property in Ambari 1.6.1, but not after 1.7.0 + if db_name: + write_property(JDBC_DATABASE_NAME_PROPERTY, db_name) + + # If DB type is missing, attempt to reconstruct it from the JDBC URL + db_type = properties[JDBC_DATABASE_PROPERTY] + if db_type is None or db_type.strip().lower() not in ["postgres", "oracle", "mysql", "derby"]: + db_type = get_db_type(properties) + if db_type: + write_property(JDBC_DATABASE_PROPERTY, db_type) + + properties = get_ambari_properties() + elif persistence_type == "local": + # Ambari 1.6.1, had "server.jdbc.database" as the DB name, and the + # DB type was assumed to be "postgres" if was embedded ("local") + db_name = properties[JDBC_DATABASE_PROPERTY] + if db_name: + write_property(JDBC_DATABASE_NAME_PROPERTY, db_name) + write_property(JDBC_DATABASE_PROPERTY, "postgres") + properties = get_ambari_properties() + + dbname = properties[JDBC_DATABASE_NAME_PROPERTY] + if dbname is None or dbname == "": + err = "DB Name property not set in config file.\n" + SETUP_OR_UPGRADE_MSG + raise FatalException(-1, err) + +def update_database_name_property(upgrade=False): + try: + check_database_name_property(upgrade) + except FatalException: + properties = get_ambari_properties() + if properties == -1: + err = "Error getting ambari properties" + raise FatalException(-1, err) + print_warning_msg(JDBC_DATABASE_NAME_PROPERTY + " property isn't set in " + + AMBARI_PROPERTIES_FILE + ". Setting it to default value - " + configDefaults.DEFAULT_DB_NAME) + properties.process_pair(JDBC_DATABASE_NAME_PROPERTY, configDefaults.DEFAULT_DB_NAME) + conf_file = find_properties_file() + try: + properties.store(open(conf_file, "w")) + except Exception, e: + err = 'Could not write ambari config file "%s": %s' % (conf_file, e) + raise FatalException(-1, err) + def is_alias_string(passwdStr): regex = re.compile("\$\{alias=[\w\.]+\}") @@ -367,9 +552,127 @@ def is_alias_string(passwdStr): else: return False +def read_passwd_for_alias(alias, masterKey=""): + if alias: + jdk_path = find_jdk() + if jdk_path is None: + print_error_msg("No JDK found, please run the \"setup\" " + "command to install a JDK automatically or install any " + "JDK manually to " + configDefaults.JDK_INSTALL_DIR) + return 1 + + tempFileName = "ambari.passwd" + passwd = "" + tempDir = tempfile.gettempdir() + #create temporary file for writing + tempFilePath = tempDir + os.sep + tempFileName + file = open(tempFilePath, 'w+') + os.chmod(tempFilePath, stat.S_IREAD | stat.S_IWRITE) + file.close() + + if masterKey is None or masterKey == "": + masterKey = "None" + + command = SECURITY_PROVIDER_GET_CMD.format(get_java_exe_path(), + get_full_ambari_classpath(), alias, tempFilePath, masterKey) + (retcode, stdout, stderr) = run_os_command(command) + print_info_msg("Return code from credential provider get passwd: " + + str(retcode)) + if retcode != 0: + print 'ERROR: Unable to read password from store. alias = ' + alias + else: + passwd = open(tempFilePath, 'r').read() + # Remove temporary file + os.remove(tempFilePath) + return passwd + else: + print_error_msg("Alias is unreadable.") + +def decrypt_password_for_alias(properties, alias): + isSecure = get_is_secure(properties) + if isSecure: + masterKey = None + (isPersisted, masterKeyFile) = get_is_persisted(properties) + if not masterKeyFile: + # Encryption enabled but no master key file found + masterKey = get_original_master_key(properties) + + return read_passwd_for_alias(alias, masterKey) + else: + return alias + +def get_original_master_key(properties): + input = True + while(input): + try: + masterKey = get_validated_string_input('Enter current Master Key: ', + "", ".*", "", True, False) + except KeyboardInterrupt: + print 'Exiting...' + sys.exit(1) + + # Find an alias that exists + alias = None + property = properties.get_property(JDBC_PASSWORD_PROPERTY) + if property and is_alias_string(property): + alias = JDBC_RCA_PASSWORD_ALIAS + + if not alias: + property = properties.get_property(LDAP_MGR_PASSWORD_PROPERTY) + if property and is_alias_string(property): + alias = LDAP_MGR_PASSWORD_ALIAS + + if not alias: + property = properties.get_property(SSL_TRUSTSTORE_PASSWORD_PROPERTY) + if property and is_alias_string(property): + alias = SSL_TRUSTSTORE_PASSWORD_ALIAS + + # Decrypt alias with master to validate it, if no master return + if alias and masterKey: + password = read_passwd_for_alias(alias, masterKey) + if not password: + print "ERROR: Master key does not match." + continue + + input = False + + return masterKey + + +# Load database connection properties from conf file +def parse_properties_file(args): + properties = get_ambari_properties() + if properties == -1: + print_error_msg("Error getting ambari properties") + return -1 + + args.server_version_file_path = properties[SERVER_VERSION_FILE_PATH] + args.persistence_type = properties[PERSISTENCE_TYPE_PROPERTY] + args.jdbc_url = properties[JDBC_URL_PROPERTY] + + args.dbms = properties[JDBC_DATABASE_PROPERTY] + if not args.persistence_type: + args.persistence_type = "local" + + if args.persistence_type == 'remote': + args.database_host = properties[JDBC_HOSTNAME_PROPERTY] + args.database_port = properties[JDBC_PORT_PROPERTY] + + args.database_name = properties[JDBC_DATABASE_NAME_PROPERTY] + args.database_username = properties[JDBC_USER_NAME_PROPERTY] + args.postgres_schema = properties[JDBC_POSTGRES_SCHEMA_PROPERTY] \ + if JDBC_POSTGRES_SCHEMA_PROPERTY in properties.propertyNames() else None + args.database_password_file = properties[JDBC_PASSWORD_PROPERTY] + if args.database_password_file: + if not is_alias_string(args.database_password_file): + args.database_password = open(properties[JDBC_PASSWORD_PROPERTY]).read() + else: + args.database_password = args.database_password_file + return 0 + def update_ambari_properties(): - prev_conf_file = search_file(AMBARI_PROPERTIES_BACKUP_FILE, get_conf_dir()) + prev_conf_file = search_file(configDefaults.AMBARI_PROPERTIES_BACKUP_FILE, get_conf_dir()) conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir()) # Previous config file does not exist @@ -482,27 +785,31 @@ def write_property(key, value): return -1 return 0 -def remove_property(key): - conf_file = find_properties_file() - properties = Properties() - try: - properties.load(open(conf_file)) - except Exception, e: - print_error_msg('Could not read ambari config file "%s": %s' % (conf_file, e)) - return -1 - properties.removeOldProp(key) - try: - properties.store(open(conf_file, "w")) - except Exception, e: - print_error_msg('Could not write ambari config file "%s": %s' % (conf_file, e)) - return -1 - return 0 - # # Checks if options determine local DB configuration # def is_local_database(args): - return hasattr(args, 'persistence_type') and args.persistence_type == 'local' + try: + return args.persistence_type == 'local' + except AttributeError: + return False + + +def update_debug_mode(): + debug_mode = get_debug_mode() + # The command-line settings supersede the ones in ambari.properties + if not debug_mode & 1: + properties = get_ambari_properties() + if properties == -1: + print_error_msg("Error getting ambari properties") + return -1 + + if get_value_from_properties(properties, DEBUG_MODE_KEY, False): + debug_mode = debug_mode | 1 + if get_value_from_properties(properties, SUSPEND_START_MODE_KEY, False): + debug_mode = debug_mode | 2 + + set_debug_mode(debug_mode) # ### JDK ### @@ -520,7 +827,7 @@ class JDKRelease: dest_jcpol_file = "" inst_dir = "" - def __init__(self, i_name, i_desc, i_url, i_dest_file, i_jcpol_url, i_dest_jcpol_file, i_inst_dir): + def __init__(self, i_name, i_desc, i_url, i_dest_file, i_jcpol_url, i_dest_jcpol_file, i_inst_dir, i_reg_exp): if i_name is None or i_name is "": raise FatalException(-1, "Invalid JDK name: " + (i_desc or "")) self.name = i_name @@ -542,14 +849,17 @@ class JDKRelease: else: self.dest_jcpol_file = i_dest_jcpol_file if i_inst_dir is None or i_inst_dir is "": - self.inst_dir = "C:\\" + i_desc + self.inst_dir = os.path.join(configDefaults.JDK_INSTALL_DIR, i_desc) else: self.inst_dir = i_inst_dir + if i_reg_exp is None or i_reg_exp is "": + raise FatalException(-1, "Invalid output parsing regular expression for JDK " + i_name) + self.reg_exp = i_reg_exp @classmethod def from_properties(cls, properties, section_name): - (desc, url, dest_file, jcpol_url, jcpol_file, inst_dir) = JDKRelease.__load_properties(properties, section_name) - cls = JDKRelease(section_name, desc, url, dest_file, jcpol_url, jcpol_file, inst_dir) + (desc, url, dest_file, jcpol_url, jcpol_file, inst_dir, reg_exp) = JDKRelease.__load_properties(properties, section_name) + cls = JDKRelease(section_name, desc, url, dest_file, jcpol_url, jcpol_file, inst_dir, reg_exp) return cls @staticmethod @@ -563,6 +873,9 @@ class JDKRelease: if not properties.has_key(section_name + ".url"): raise FatalException(-1, "Invalid JDK URL in the properties section: " + section_name) url = properties[section_name + ".url"] #Required + if not properties.has_key(section_name + ".re"): + raise FatalException(-1, "Invalid JDK output parsing regular expression in the properties section: " + section_name) + reg_exp = properties[section_name + ".re"] #Required if(properties.has_key(section_name + ".dest-file")): #Not critical dest_file = properties[section_name + ".dest-file"] else: @@ -579,7 +892,7 @@ class JDKRelease: inst_dir = properties[section_name + ".home"] else: inst_dir = "C:\\" + section_name - return (desc, url, dest_file, jcpol_url, jcpol_file, inst_dir) + return (desc, url, dest_file, jcpol_url, jcpol_file, inst_dir, reg_exp) pass def get_ambari_jars(): @@ -618,6 +931,14 @@ def get_ambari_classpath(): ambari_cp = ambari_cp + os.pathsep + share_cp return ambari_cp +def get_full_ambari_classpath(conf_dir = None): + if conf_dir is None: + conf_dir = get_conf_dir() + cp = conf_dir + os.pathsep + get_ambari_classpath() + if cp.find(' ') != -1: + cp = '"' + cp + '"' + return cp + def get_JAVA_HOME(): properties = get_ambari_properties() if properties == -1: @@ -665,3 +986,35 @@ def find_jdk(): else: print "JDK {0} is invalid".format(jdkPath) return + +def get_java_exe_path(): + jdkPath = find_jdk() + if jdkPath: + java_exe = os.path.join(jdkPath, configDefaults.JAVA_EXE_SUBPATH) + return java_exe + return + +# +# JDBC +# + +#Check if required jdbc drivers present +def find_jdbc_driver(args): + if args.dbms in JDBC_PATTERNS.keys(): + drivers = [] + drivers.extend(glob.glob(os.path.join(configDefaults.JAVA_SHARE_PATH, JDBC_PATTERNS[args.dbms]))) + if drivers: + return drivers + return -1 + return 0 + + +# +# Stack upgrade +# + +def get_stack_location(properties): + stack_location = properties[STACK_LOCATION_KEY] + if stack_location is None: + stack_location = configDefaults.STACK_LOCATION_DEFAULT + return stack_location