Updated Branches: refs/heads/trunk a6b4a81be -> 90b4fa9bf
AMBARI-3236. Add support for ambari-server upgrade against oracle db using the ambari-server script. (myroslav via mahadev) Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/90b4fa9b Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/90b4fa9b Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/90b4fa9b Branch: refs/heads/trunk Commit: 90b4fa9bf6b9d91781388e2cbc953681d41df5de Parents: a6b4a81 Author: Mahadev Konar <[email protected]> Authored: Fri Sep 13 16:08:03 2013 -0700 Committer: Mahadev Konar <[email protected]> Committed: Fri Sep 13 16:08:07 2013 -0700 ---------------------------------------------------------------------- ambari-server/src/main/python/ambari-server.py | 130 ++++++++++++++++--- .../src/test/python/TestAmbariServer.py | 39 +++++- 2 files changed, 150 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/90b4fa9b/ambari-server/src/main/python/ambari-server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py index 3069c89..7301826 100755 --- a/ambari-server/src/main/python/ambari-server.py +++ b/ambari-server/src/main/python/ambari-server.py @@ -287,6 +287,9 @@ DATABASE_CONNECTION_STRINGS_ALT = [ "jdbc:postgresql://{0}:{1}/{2}", "jdbc:oracle:thin:@{0}:{1}:{2}", "jdbc:mysql://{0}:{1}/{2}"] +ORACLE_SID_PATTERN = "jdbc:oracle:thin:@.+:.+/.+" +ORACLE_SNAME_PATTERN = "jdbc:oracle:thin:@.+:.+:.+" + DATABASE_CLI_TOOLS = [["psql"], ["sqlplus", "sqlplus64"], ["mysql"]] DATABASE_CLI_TOOLS_DESC = ["psql", "sqlplus", "mysql"] DATABASE_CLI_TOOLS_USAGE = ['su -postgres --command=psql -f {0} -v username=\'"{1}"\' -v password="\'{2}\'"', @@ -299,6 +302,12 @@ DATABASE_INIT_SCRIPTS = ['/var/lib/ambari-server/resources/Ambari-DDL-Postgres-R DATABASE_DROP_SCRIPTS = ['/var/lib/ambari-server/resources/Ambari-DDL-Postgres-REMOTE-DROP.sql', '/var/lib/ambari-server/resources/Ambari-DDL-Oracle-DROP.sql', '/var/lib/ambari-server/resources/Ambari-DDL-MySQL-DROP.sql'] +DATABASE_UPGRADE_SCRIPTS = ['/var/lib/ambari-server/resources/upgrade/ddl/Ambari-DDL-Postgres-REMOTE-UPGRADE.sql', + '/var/lib/ambari-server/resources/upgrade/ddl/Ambari-DDL-Oracle-UPGRADE.sql', + '/var/lib/ambari-server/resources/upgrade/ddl/Ambari-DDL-MySQL-UPGRADE.sql'] +DATABASE_STACK_UPGRADE_SCRIPTS = ['/var/lib/ambari-server/resources/upgrade/dml/Ambari-DML-Postgres-REMOTE-UPGRADE_STACK.sql', + '/var/lib/ambari-server/resources/upgrade/dml/Ambari-DML-Oracle-UPGRADE_STACK.sql', + '/var/lib/ambari-server/resources/upgrade/dml/Ambari-DML-MySQL-UPGRADE_STACK.sql'] JDBC_PROPERTIES_PREFIX = "server.jdbc.properties." DATABASE_JDBC_PROPERTIES = [ @@ -317,11 +326,14 @@ REGEX_HOSTNAME_PORT = "^(.*:[0-9]{1,5}$)" REGEX_TRUE_FALSE = "^(true|false)?$" REGEX_ANYTHING = ".*" + POSTGRES_EXEC_ARGS = "-h {0} -p {1} -d {2} -U {3} -f {4} -v username='\"{3}\"'" -ORACLE_EXEC_ARGS = "-S '{0}/{1}@(description=(address=(protocol=TCP)(host={2})(port={3}))(connect_data=(sid={4})))' @{5} {0}" +ORACLE_EXEC_ARGS = "-S '{0}/{1}@(description=(address=(protocol=TCP)(host={2})(port={3}))(connect_data=({6}={4})))' @{5} {0}" MYSQL_EXEC_ARGS = "--host={0} --port={1} --user={2} --password={3} {4} " \ "-e\"set @schema=\'{4}\'; set @username=\'{2}\'; source {5};\"" +ORACLE_UPGRADE_STACK_ARGS = "-S '{0}/{1}@(description=(address=(protocol=TCP)(host={2})(port={3}))(connect_data=({6}={4})))' @{5} {0} {7}" + JDBC_PATTERNS = {"oracle":"*ojdbc*.jar", "mysql":"*mysql*.jar"} DATABASE_FULL_NAMES = {"oracle":"Oracle", "mysql":"MySQL", "postgres":"PostgreSQL"} ORACLE_DB_ID_TYPES = ["Service Name", "SID"] @@ -1251,6 +1263,36 @@ def get_db_cli_tool(args): return None +def remote_stack_upgrade(args, scriptPath, stackId): + tool = get_db_cli_tool(args) + if not tool: + args.warnings.append('{0} not found. Please, run DDL script manually'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX])) + if VERBOSE: + print_warning_msg('{0} not found'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX])) + return -1, "Client wasn't found", "Client wasn't found" + + #TODO add support of other databases with scripts + if args.database == "oracle": + sid_or_sname = "sid" + if args.sid_or_sname == "sname" or (args.jdbc_url and re.match(ORACLE_SNAME_PATTERN, args.jdbc_url)): + print_info_msg("using SERVICE_NAME instead of SID for Oracle") + sid_or_sname = "service_name" + + retcode, out, err = run_in_shell('{0} {1}'.format(tool, ORACLE_UPGRADE_STACK_ARGS.format( + args.database_username, + args.database_password, + args.database_host, + args.database_port, + args.database_name, + scriptPath, + sid_or_sname, + stackId + ))) + return retcode, out, err + + return -2, "Wrong database", "Wrong database" + pass + #execute SQL script on remote database def execute_remote_script(args, scriptPath): tool = get_db_cli_tool(args) @@ -1272,16 +1314,22 @@ def execute_remote_script(args, scriptPath): ))) return retcode, out, err elif args.database == "oracle": + sid_or_sname = "sid" + if args.sid_or_sname == "sname" or (args.jdbc_url and re.match(ORACLE_SNAME_PATTERN, args.jdbc_url)): + print_info_msg("using SERVICE_NAME instead of SID for Oracle") + sid_or_sname = "service_name" + retcode, out, err = run_in_shell('{0} {1}'.format(tool, ORACLE_EXEC_ARGS.format( args.database_username, args.database_password, args.database_host, args.database_port, args.database_name, - scriptPath + scriptPath, + sid_or_sname ))) return retcode, out, err - elif args.database=="mysql": + elif args.database == "mysql": retcode, out, err = run_in_shell('{0} {1}'.format(tool, MYSQL_EXEC_ARGS.format( args.database_host, args.database_port, @@ -1425,6 +1473,7 @@ def parse_properties_file(args): return -1 args.persistence_type = properties[PERSISTENCE_TYPE_PROPERTY] + args.jdbc_url = properties[JDBC_URL_PROPERTY] if not args.persistence_type: args.persistence_type = "local" @@ -1439,9 +1488,11 @@ def parse_properties_file(args): DATABASE_INDEX = DATABASE_NAMES.index(args.database) except ValueError: pass + else: + #TODO incorrect property used!! leads to bunch of troubles. Workaround for now + args.database_name = properties[JDBC_DATABASE_PROPERTY] args.database_username = properties[JDBC_USER_NAME_PROPERTY] - args.database_name = properties[JDBC_DATABASE_PROPERTY] args.database_password_file = properties[JDBC_PASSWORD_PROPERTY] if args.database_password_file: if not is_alias_string(args.database_password_file): @@ -2273,19 +2324,44 @@ def upgrade_stack(args, stack_id): 'root-level privileges' raise FatalException(4, err) check_database_name_property() - #password access to ambari-server and mapred - configure_database_username_password(args) - dbname = args.database_name - file = args.upgrade_stack_script_file - stack_name, stack_version = stack_id.split(STACK_NAME_VER_SEP) - command = UPGRADE_STACK_CMD[:] - command[-1] = command[-1].format(file, stack_name, stack_version, dbname) - retcode, outdata, errdata = run_os_command(command) - if not retcode == 0: - raise FatalException(retcode, errdata) - if errdata: - print_warning_msg(errdata) - return retcode + + parse_properties_file(args) + if args.persistence_type == "remote": + client_desc = DATABASE_NAMES[DATABASE_INDEX] + ' ' + DATABASE_CLI_TOOLS_DESC[DATABASE_INDEX] + client_usage_cmd = DATABASE_CLI_TOOLS_USAGE[DATABASE_INDEX].format(DATABASE_STACK_UPGRADE_SCRIPTS[DATABASE_INDEX], args.database_username, + BLIND_PASSWORD, args.database_name) + #TODO temporarty code + if not args.database == "oracle": + raise FatalException(-20, "Upgrade for remote database only supports Oracle.") + + if get_db_cli_tool(args): + retcode, out, err = remote_stack_upgrade(args, DATABASE_STACK_UPGRADE_SCRIPTS[DATABASE_INDEX], stack_id) + if not retcode == 0: + raise NonFatalException(err) + + else: + err = 'Cannot find ' + client_desc + ' client in the path to upgrade the Ambari ' + \ + 'Server stack. To upgrade stack of Ambari Server ' + \ + 'you must run the following DML against the database:' + \ + os.linesep + client_usage_cmd + raise NonFatalException(err) + + + pass + else: + #password access to ambari-server and mapred + configure_database_username_password(args) + dbname = args.database_name + file = args.upgrade_stack_script_file + stack_name, stack_version = stack_id.split(STACK_NAME_VER_SEP) + command = UPGRADE_STACK_CMD[:] + command[-1] = command[-1].format(file, stack_name, stack_version, dbname) + retcode, outdata, errdata = run_os_command(command) + if not retcode == 0: + raise FatalException(retcode, errdata) + if errdata: + print_warning_msg(errdata) + return retcode # @@ -2320,6 +2396,26 @@ def upgrade(args): parse_properties_file(args) if args.persistence_type == "remote": + client_desc = DATABASE_NAMES[DATABASE_INDEX] + ' ' + DATABASE_CLI_TOOLS_DESC[DATABASE_INDEX] + client_usage_cmd = DATABASE_CLI_TOOLS_USAGE[DATABASE_INDEX].format(DATABASE_UPGRADE_SCRIPTS[DATABASE_INDEX], args.database_username, + BLIND_PASSWORD, args.database_name) + + #TODO temporarty code + if not args.database == "oracle": + raise FatalException(-20, "Upgrade for remote database only supports Oracle.") + + if get_db_cli_tool(args): + retcode, out, err = execute_remote_script(args, DATABASE_UPGRADE_SCRIPTS[DATABASE_INDEX]) + if not retcode == 0: + raise NonFatalException(err) + + else: + err = 'Cannot find ' + client_desc + ' client in the path to upgrade the Ambari ' + \ + 'Server schema. To upgrade Ambari Server schema ' + \ + 'you must run the following DDL against the database:' + \ + os.linesep + client_usage_cmd + raise NonFatalException(err) + pass else: print 'Checking PostgreSQL...' http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/90b4fa9b/ambari-server/src/test/python/TestAmbariServer.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py index 3023e90..b62252c 100644 --- a/ambari-server/src/test/python/TestAmbariServer.py +++ b/ambari-server/src/test/python/TestAmbariServer.py @@ -2351,9 +2351,14 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV @patch.object(ambari_server, "run_os_command") @patch.object(ambari_server, "is_root") @patch.object(ambari_server, "check_database_name_property") - def test_upgrade_stack(self, check_database_name_property_mock, is_root_mock, run_os_command_mock, + @patch.object(ambari_server, "parse_properties_file") + @patch.object(ambari_server, "get_db_cli_tool") + @patch.object(ambari_server, "remote_stack_upgrade") + def test_upgrade_stack(self, remote_stack_upgrade_mock, get_db_cli_tool_mock, parse_properties_file_mock, + check_database_name_property_mock, is_root_mock, run_os_command_mock, configure_postgres_username_password_mock): args = MagicMock() + args.persistence_type = "local" # Testing call under non-root is_root_mock.return_value = False @@ -2374,6 +2379,22 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV self.assertTrue(configure_postgres_username_password_mock.called) self.assertTrue(run_os_command_mock.called) + # Test remote + configure_postgres_username_password_mock.reset_mock() + run_os_command_mock.reset_mock() + args.persistence_type = "remote" + args.database = "oracle" + + get_db_cli_tool_mock.return_value = "psql" + remote_stack_upgrade_mock.return_value = (0, "test", "test") + + ambari_server.upgrade_stack(args, 'HDP-2.0') + + self.assertTrue(get_db_cli_tool_mock.called) + self.assertTrue(remote_stack_upgrade_mock.called) + self.assertFalse(run_os_command_mock.called) + + @patch.object(ambari_server, "adjust_directory_permissions") @patch.object(ambari_server, "print_warning_msg") @@ -2385,7 +2406,10 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV @patch.object(ambari_server, "parse_properties_file") @patch.object(ambari_server, "is_root") @patch.object(ambari_server, "get_ambari_properties") - def test_upgrade(self, get_ambari_properties_mock, is_root_mock, parse_properties_file_mock, + @patch.object(ambari_server, "get_db_cli_tool") + @patch.object(ambari_server, "execute_remote_script") + def test_upgrade(self, execute_remote_script_mock, get_db_cli_tool_mock, get_ambari_properties_mock, is_root_mock, + parse_properties_file_mock, update_ambari_properties_mock, check_postgre_up_mock, execute_db_script_mock, check_db_consistency_mock, read_ambari_user_mock, @@ -2435,6 +2459,17 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV ambari_server.upgrade(args) self.assertTrue(get_ambari_properties_mock.called) + #Test remote upgrade + get_db_cli_tool_mock.return_value = "psql" + execute_remote_script_mock.return_value = (0, "test", "test") + args.persistence_type = "remote" + args.database = "oracle" + + ambari_server.upgrade(args) + + self.assertTrue(get_db_cli_tool_mock.called) + self.assertTrue(execute_remote_script_mock.called) + def test_print_info_msg(self): out = StringIO.StringIO() sys.stdout = out
