Repository: ambari Updated Branches: refs/heads/trunk 66c6b06c3 -> 78bde257f
AMBARI-16912. Oozie and Hive Server start fail during EU with missing DB class exception.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/78bde257 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/78bde257 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/78bde257 Branch: refs/heads/trunk Commit: 78bde257f6f4a2fc48577a4b27c4b2aa70a339cd Parents: 66c6b06 Author: Vitaly Brodetskyi <[email protected]> Authored: Fri May 27 20:07:33 2016 +0300 Committer: Vitaly Brodetskyi <[email protected]> Committed: Fri May 27 20:07:33 2016 +0300 ---------------------------------------------------------------------- .../python/ambari_server/dbConfiguration.py | 10 ++++++++ .../main/python/ambari_server/serverSetup.py | 9 ++----- .../main/python/ambari_server/serverUpgrade.py | 26 +++++++++++++++++--- .../0.12.0.2.0/package/scripts/hive_service.py | 6 +++-- .../package/scripts/hive_service_interactive.py | 4 ++- .../4.0.0.2.0/package/scripts/oozie_service.py | 5 +++- .../RANGER_KMS/0.5.0.2.3/package/scripts/kms.py | 4 ++- 7 files changed, 49 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/78bde257/ambari-server/src/main/python/ambari_server/dbConfiguration.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/dbConfiguration.py b/ambari-server/src/main/python/ambari_server/dbConfiguration.py index c92e580..fc68a4c 100644 --- a/ambari-server/src/main/python/ambari_server/dbConfiguration.py +++ b/ambari-server/src/main/python/ambari_server/dbConfiguration.py @@ -40,6 +40,7 @@ SETUP_DB_CONNECT_ATTEMPTS = 3 USERNAME_PATTERN = "^[a-zA-Z_][a-zA-Z0-9_\-]*$" PASSWORD_PATTERN = "^[a-zA-Z0-9_-]*$" +CUSTOM_JDBC_DB_NAMES = ["postgres", "mysql", "mssql", "oracle", "hsqldb", "sqlanywhere"] DATABASE_NAMES = ["postgres", "oracle", "mysql", "mssql", "sqlanywhere"] DATABASE_FULL_NAMES = {"oracle": "Oracle", "mysql": "MySQL", "mssql": "Microsoft SQL Server", "postgres": "PostgreSQL", "sqlanywhere": "SQL Anywhere"} @@ -47,6 +48,15 @@ LINUX_DBMS_KEYS_LIST = [ 'embedded', 'oracle', 'mysql', 'postgres', 'mssql', 'sq AMBARI_DATABASE_NAME = "ambari" AMBARI_DATABASE_TITLE = "ambari" +TAR_GZ_ARCHIVE_TYPE = ".tar.gz" + +default_connectors_map = { "mssql":"sqljdbc4.jar", + "mysql":"mysql-connector-java.jar", + "postgres":"postgresql-jdbc.jar", + "oracle":"ojdbc.jar", + "sqlanywhere":"sajdbc4.jar", + "hsqldb":"hsqldb.jar"} + STORAGE_TYPE_LOCAL = 'local' STORAGE_TYPE_REMOTE = 'remote' http://git-wip-us.apache.org/repos/asf/ambari/blob/78bde257/ambari-server/src/main/python/ambari_server/serverSetup.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverSetup.py b/ambari-server/src/main/python/ambari_server/serverSetup.py index f01465d..d862411 100644 --- a/ambari-server/src/main/python/ambari_server/serverSetup.py +++ b/ambari-server/src/main/python/ambari_server/serverSetup.py @@ -34,7 +34,7 @@ from ambari_commons.os_check import OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons.os_utils import copy_files, run_os_command, is_root from ambari_commons.str_utils import compress_backslashes -from ambari_server.dbConfiguration import DBMSConfigFactory, check_jdbc_drivers +from ambari_server.dbConfiguration import DBMSConfigFactory, TAR_GZ_ARCHIVE_TYPE, default_connectors_map, check_jdbc_drivers from ambari_server.serverConfiguration import configDefaults, JDKRelease, \ get_ambari_properties, get_is_secure, get_is_persisted, get_java_exe_path, get_JAVA_HOME, \ get_resources_location, get_value_from_properties, read_ambari_user, update_properties, validate_jdk, write_property, \ @@ -60,7 +60,6 @@ SE_MODE_PERMISSIVE = "permissive" PERSISTENCE_TYPE_PROPERTY = "server.persistence.type" -TAR_GZ_ARCHIVE_TYPE = ".tar.gz" # Non-root user setup commands NR_USER_COMMENT = "Ambari user" @@ -77,11 +76,7 @@ UNTAR_JDK_ARCHIVE = "tar --no-same-owner -xvf {0}" JDK_PROMPT = "[{0}] {1}\n" JDK_VALID_CHOICES = "^[{0}{1:d}]$" -default_connectors_map = { "mssql":"sqljdbc4.jar", - "mysql":"mysql-connector-java.jar", - "postgres":"postgresql-jdbc.jar", - "oracle":"ojdbc.jar", - "sqlanywhere":"sajdbc4.jar"} + def get_supported_jdbc_drivers(): factory = DBMSConfigFactory() http://git-wip-us.apache.org/repos/asf/ambari/blob/78bde257/ambari-server/src/main/python/ambari_server/serverUpgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverUpgrade.py b/ambari-server/src/main/python/ambari_server/serverUpgrade.py index cdcce1c..483b79e 100644 --- a/ambari-server/src/main/python/ambari_server/serverUpgrade.py +++ b/ambari-server/src/main/python/ambari_server/serverUpgrade.py @@ -31,10 +31,10 @@ import optparse from ambari_commons.exceptions import FatalException from ambari_commons.logging_utils import print_info_msg, print_warning_msg, print_error_msg, get_verbose from ambari_commons.os_utils import is_root, run_os_command -from ambari_server.dbConfiguration import DBMSConfigFactory, check_jdbc_drivers, \ - get_jdbc_driver_path, ensure_jdbc_driver_is_installed, LINUX_DBMS_KEYS_LIST +from ambari_server.dbConfiguration import DBMSConfigFactory, CUSTOM_JDBC_DB_NAMES, TAR_GZ_ARCHIVE_TYPE, check_jdbc_drivers, \ + get_jdbc_driver_path, ensure_jdbc_driver_is_installed, LINUX_DBMS_KEYS_LIST, default_connectors_map from ambari_server.properties import Properties -from ambari_server.serverConfiguration import configDefaults, \ +from ambari_server.serverConfiguration import configDefaults, get_resources_location, update_properties, \ check_database_name_property, get_ambari_properties, get_ambari_version, \ get_java_exe_path, get_stack_location, parse_properties_file, read_ambari_user, update_ambari_properties, \ update_database_name_property, get_admin_views_dir, get_views_dir, \ @@ -413,6 +413,26 @@ def upgrade(args): if properties.get_property(LDAP_PRIMARY_URL_PROPERTY) and not properties.get_property(IS_LDAP_CONFIGURED): args.warnings.append("Existing LDAP configuration is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration.") + # adding custom jdbc name and previous custom jdbc properties + # we need that to support new dynamic jdbc names for upgraded ambari + add_jdbc_properties(properties) + + +def add_jdbc_properties(properties): + for db_name in CUSTOM_JDBC_DB_NAMES: + if db_name == "sqlanywhere": + symlink_name = db_name + "-jdbc-driver" + TAR_GZ_ARCHIVE_TYPE + else: + symlink_name = db_name + "-jdbc-driver.jar" + + resources_dir = get_resources_location(properties) + custom_db_jdbc_property_name = "custom." + db_name + ".jdbc.name" + + if os.path.lexists(os.path.join(resources_dir, symlink_name)): + properties.process_pair(custom_db_jdbc_property_name, symlink_name) + properties.process_pair("previous." + custom_db_jdbc_property_name, default_connectors_map[db_name]) + update_properties(properties) + # # Set current cluster version (run Finalize during manual RU) http://git-wip-us.apache.org/repos/asf/ambari/blob/78bde257/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py index a13b087..bd8c33b 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py @@ -103,8 +103,10 @@ def hive_service(name, action='start', upgrade_type=None): path_to_jdbc = format("{hive_lib}/") + params.default_connectors_map[params.hive_jdbc_driver] if not os.path.isfile(path_to_jdbc): path_to_jdbc = format("{hive_lib}/") + "*" - print "Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.hive_jdbc_driver] + \ - " in hive lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'" + error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.hive_jdbc_driver] + \ + " in hive lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'" + print error_message + Logger.error(error_message) db_connection_check_command = format( "{java64_home}/bin/java -cp {check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{hive_jdbc_connection_url}' {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_jdbc_driver}") http://git-wip-us.apache.org/repos/asf/ambari/blob/78bde257/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service_interactive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service_interactive.py index 716b612..908c726 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service_interactive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service_interactive.py @@ -73,8 +73,10 @@ def hive_service_interactive(name, action='start', upgrade_type=None): path_to_jdbc = format("{hive_interactive_lib}/") + params.default_connectors_map[params.hive_jdbc_driver] if not os.path.isfile(path_to_jdbc): path_to_jdbc = format("{hive_interactive_lib}/") + "*" - print "Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.hive_jdbc_driver] + \ + error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.hive_jdbc_driver] + \ " in hive lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'" + print error_message + Logger.error(error_message) db_connection_check_command = format( "{java64_home}/bin/java -cp {check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{hive_jdbc_connection_url}' {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_jdbc_driver}") http://git-wip-us.apache.org/repos/asf/ambari/blob/78bde257/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py index 2f091a5..ffe1783 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py @@ -20,6 +20,7 @@ limitations under the License. import os from resource_management import * from resource_management.core.shell import as_user +from resource_management.core.logger import Logger from resource_management.libraries.functions.show_logs import show_logs from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl @@ -73,8 +74,10 @@ def oozie_service(action = 'start', upgrade_type=None): path_to_jdbc = format("{oozie_libext_dir}/") + params.default_connectors_map[params.jdbc_driver_name] if not os.path.isfile(path_to_jdbc): path_to_jdbc = format("{oozie_libext_dir}/") + "*" - print "Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.hive_jdbc_driver] + \ + error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.jdbc_driver_name] + \ " in oozie lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'" + print error_message + Logger.error(error_message) db_connection_check_command = format("{java_home}/bin/java -cp {check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{oozie_jdbc_connection_url}' {oozie_metastore_user_name} {oozie_metastore_user_passwd!p} {jdbc_driver_name}") else: http://git-wip-us.apache.org/repos/asf/ambari/blob/78bde257/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py index 0a39219..e446c43 100755 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py @@ -156,8 +156,10 @@ def kms(upgrade_type=None): path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + params.default_connectors_map[params.db_flavor] if not os.path.isfile(path_to_jdbc): path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + "*" - print "Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.hive_jdbc_driver] + \ + error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.db_flavor] + \ " in oozie lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'" + print error_message + Logger.error(error_message) cp = cp + os.pathsep + path_to_jdbc
