Repository: ambari Updated Branches: refs/heads/trunk 2914d681d -> dcab10878
AMBARI-8837. Disable installing of mysql connector jars and mysql server for external mysql for Hive.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dcab1087 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dcab1087 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dcab1087 Branch: refs/heads/trunk Commit: dcab10878476a9f956ec4085f8d10224f8d8c9de Parents: 2914d68 Author: Vitaly Brodetskyi <[email protected]> Authored: Fri Dec 19 21:48:00 2014 +0200 Committer: Vitaly Brodetskyi <[email protected]> Committed: Fri Dec 19 21:48:00 2014 +0200 ---------------------------------------------------------------------- .../0.12.0.2.0/package/scripts/hcat_client.py | 3 +- .../HIVE/0.12.0.2.0/package/scripts/hive.py | 37 +++++----------- .../0.12.0.2.0/package/scripts/mysql_server.py | 4 ++ .../HIVE/0.12.0.2.0/package/scripts/params.py | 15 ++++--- .../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 23 ++++++++-- .../4.0.0.2.0/package/scripts/oozie_service.py | 2 +- .../OOZIE/4.0.0.2.0/package/scripts/params.py | 18 ++++++-- .../1.3.2/services/HIVE/package/scripts/hive.py | 32 ++++---------- .../HIVE/package/scripts/mysql_server.py | 7 ++- .../services/HIVE/package/scripts/params.py | 15 ++++++- .../HIVE/package/scripts/webhcat_server.py | 3 +- .../services/OOZIE/package/scripts/oozie.py | 26 ++++++++--- .../OOZIE/package/scripts/oozie_service.py | 2 +- .../services/OOZIE/package/scripts/params.py | 15 ++++++- .../stacks/1.3.2/HIVE/test_hive_metastore.py | 40 ++++++++++------- .../stacks/1.3.2/HIVE/test_hive_server.py | 40 ++++++++++------- .../stacks/2.0.6/HIVE/test_hive_metastore.py | 46 ++++++++++---------- .../stacks/2.0.6/HIVE/test_hive_server.py | 46 ++++++++++---------- .../stacks/2.1/HIVE/test_hive_metastore.py | 34 ++++++++------- 19 files changed, 240 insertions(+), 168 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_client.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_client.py index 8b5921a..5fc27f4 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_client.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_client.py @@ -24,7 +24,8 @@ from hcat import hcat class HCatClient(Script): def install(self, env): - self.install_packages(env) + import params + self.install_packages(env, exclude_packages=params.hive_exclude_packages) self.configure(env) def configure(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py index 2dbdd6f..8bbb569 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py @@ -196,37 +196,20 @@ def crt_file(name): def jdbc_connector(): import params - if params.hive_jdbc_driver == "com.mysql.jdbc.Driver": - File(params.target, - action="delete", - ) - Execute(('cp', format('/usr/share/java/{jdbc_jar_name}'), params.target), - not_if=format("test -f {target}"), - creates=params.target, - path=["/bin", "/usr/bin/"], - sudo=True - ) - elif params.hive_jdbc_driver == "org.postgresql.Driver": - cmd = format("mkdir -p {artifact_dir} ; rm -f {target} ; cp /usr/share/java/{jdbc_jar_name} {target}") - - Execute(cmd, - not_if=format("test -f {target}"), - creates=params.target, - environment= {'PATH' : params.execute_path }, - path=["/bin", "usr/bin/"]) - - elif params.hive_jdbc_driver == "oracle.jdbc.driver.OracleDriver": + if params.hive_jdbc_driver in params.hive_jdbc_drivers_list: environment = { "no_proxy": format("{ambari_server_hostname}") } - cmd = format( - "mkdir -p {artifact_dir} ; " - "curl -kf -x \"\" --retry 10 {driver_curl_source} -o {driver_curl_target} && " - "rm -f {target} ; " - "cp {driver_curl_target} {target}") + Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o', params.driver_curl_target), + not_if=format("test -f {target}"), + path=["/bin", "/usr/bin/"], + environment=environment, + sudo = True) + - Execute(cmd, + Execute(('cp', params.driver_curl_target, params.target), not_if=format("test -f {target}"), + creates=params.target, path=["/bin", "/usr/bin/"], - environment=environment) + sudo = True) http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/mysql_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/mysql_server.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/mysql_server.py index 946a360..c74dfe3 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/mysql_server.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/mysql_server.py @@ -19,6 +19,7 @@ limitations under the License. """ import sys +import os import mysql_users from resource_management import * @@ -29,6 +30,9 @@ from mysql import mysql_configure class MysqlServer(Script): def install(self, env): import params + Package('mysql-connector-java', action = "remove") + if os.path.exists(params.mysql_jdbc_driver_jar): + File(params.mysql_jdbc_driver_jar, action = "delete") self.install_packages(env, exclude_packages=params.hive_exclude_packages) self.configure(env) http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py index 928bbb6..4e58de0 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py @@ -99,11 +99,12 @@ elif hive_jdbc_driver == "org.postgresql.Driver": jdbc_jar_name = "postgresql-jdbc.jar" jdbc_symlink_name = "postgres-jdbc-driver.jar" elif hive_jdbc_driver == "oracle.jdbc.driver.OracleDriver": - jdbc_jar_name = "ojdbc6.jar" + jdbc_jar_name = "ojdbc.jar" jdbc_symlink_name = "oracle-jdbc-driver.jar" check_db_connection_jar_name = "DBConnectionVerification.jar" check_db_connection_jar = format("/usr/lib/ambari-agent/{check_db_connection_jar_name}") +hive_jdbc_drivers_list = ["com.mysql.jdbc.Driver","org.postgresql.Driver","oracle.jdbc.driver.OracleDriver"] #common hive_metastore_host = config['clusterHostInfo']['hive_metastore_host'][0] @@ -142,6 +143,7 @@ else: #hive-site hive_database_name = config['configurations']['hive-env']['hive_database_name'] +hive_database = config['configurations']['hive-env']['hive_database'] #Starting hiveserver2 start_hiveserver2_script = 'startHiveserver2.sh.j2' @@ -242,10 +244,13 @@ mysql_jdbc_driver_jar = "/usr/share/java/mysql-connector-java.jar" # There are other packages that contain /usr/share/java/mysql-connector-java.jar (like libmysql-java), # trying to install mysql-connector-java upon them can cause packages to conflict. -if os.path.exists(mysql_jdbc_driver_jar): - hive_exclude_packages = ['mysql-connector-java'] -else: - hive_exclude_packages = [] +if hive_database.startswith('Existing'): + hive_exclude_packages = ['mysql-connector-java','mysql','mysql-server'] +else: + if 'role' in config and config['role'] != "MYSQL_SERVER": + hive_exclude_packages = ['mysql','mysql-server'] + else: + hive_exclude_packages = [] ######################################################## ########### WebHCat related params ##################### http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py index a760ab2..dbde3ab 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py @@ -154,10 +154,25 @@ def oozie_server_specific(): ) if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver": - Execute(('cp', params.jdbc_driver_jar, params.oozie_libext_dir), - not_if = no_op_test, - sudo = True, - ) + + environment = { + "no_proxy": format("{ambari_server_hostname}") + } + + Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o', params.driver_curl_target), + not_if=format("test -f {target}"), + path=["/bin", "/usr/bin/"], + environment=environment, + sudo = True) + + + Execute(('cp', params.driver_curl_target, params.target), + not_if=format("test -f {target}"), + creates=params.target, + path=["/bin", "/usr/bin/"], + sudo = True) + + #falcon el extension if params.has_falcon_host: Execute(format('sudo cp {falcon_home}/oozie/ext/falcon-oozie-el-extension-*.jar {oozie_libext_dir}'), http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py index a1d7bad..9b89f92 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py @@ -32,7 +32,7 @@ def oozie_service(action = 'start'): # 'start' or 'stop' if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \ params.jdbc_driver_name == "org.postgresql.Driver" or \ params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver": - db_connection_check_command = format("{java_home}/bin/java -cp {check_db_connection_jar}:{jdbc_driver_jar} org.apache.ambari.server.DBConnectionVerification '{oozie_jdbc_connection_url}' {oozie_metastore_user_name} {oozie_metastore_user_passwd!p} {jdbc_driver_name}") + db_connection_check_command = format("{java_home}/bin/java -cp {check_db_connection_jar}:{target} org.apache.ambari.server.DBConnectionVerification '{oozie_jdbc_connection_url}' {oozie_metastore_user_name} {oozie_metastore_user_passwd!p} {jdbc_driver_name}") else: db_connection_check_command = None http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params.py index b5f2ea7..413ceb3 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params.py @@ -75,6 +75,7 @@ oozie_hdfs_user_dir = format("/user/{oozie_user}") oozie_pid_dir = status_params.oozie_pid_dir pid_file = status_params.pid_file hadoop_jar_location = "/usr/lib/hadoop/" +java_share_dir = "/usr/share/java" # for HDP1 it's "/usr/share/HDP-oozie/ext.zip" ext_js_file = "ext-2.2.zip" ext_js_path = format("/usr/share/HDP-oozie/{ext_js_file}") @@ -110,13 +111,24 @@ else: jdbc_driver_name = default("/configurations/oozie-site/oozie.service.JPAService.jdbc.driver", "") if jdbc_driver_name == "com.mysql.jdbc.Driver": - jdbc_driver_jar = "/usr/share/java/mysql-connector-java.jar" + jdbc_driver_jar = "mysql-connector-java.jar" + jdbc_symlink_name = "mysql-jdbc-driver.jar" elif jdbc_driver_name == "org.postgresql.Driver": - jdbc_driver_jar = format("{oozie_home}/libserver/postgresql-9.0-801.jdbc4.jar") + jdbc_driver_jar = format("{oozie_home}/libserver/postgresql-9.0-801.jdbc4.jar") #oozie using it's own postgres jdbc + jdbc_symlink_name = "postgres-jdbc-driver.jar" elif jdbc_driver_name == "oracle.jdbc.driver.OracleDriver": - jdbc_driver_jar = "/usr/share/java/ojdbc6.jar" + jdbc_driver_jar = "ojdbc.jar" + jdbc_symlink_name = "oracle-jdbc-driver.jar" else: jdbc_driver_jar = "" + jdbc_symlink_name = "" + +driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}") +driver_curl_target = format("{java_share_dir}/{jdbc_driver_jar}") +if jdbc_driver_name == "org.postgresql.Driver": + target = jdbc_driver_jar +else: + target = format("{oozie_libext_dir}/{jdbc_driver_jar}") hostname = config["hostname"] ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive.py index 19a68c6..5de6185 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive.py @@ -161,34 +161,20 @@ def crt_file(name): def jdbc_connector(): import params - if params.hive_jdbc_driver == "com.mysql.jdbc.Driver": - cmd = ('cp', format('/usr/share/java/{jdbc_jar_name}'), params.target) + if params.hive_jdbc_driver in params.hive_jdbc_drivers_list: + environment = { + "no_proxy": format("{ambari_server_hostname}") + } - Execute(cmd, + Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o', params.driver_curl_target), not_if=format("test -f {target}"), - creates=params.target, path=["/bin", "/usr/bin/"], - sudo=True - ) + environment=environment, + sudo = True) - elif params.hive_jdbc_driver == "org.postgresql.Driver": - cmd = format("mkdir -p {artifact_dir} ; cp /usr/share/java/{jdbc_jar_name} {target}") - Execute(cmd, + Execute(('cp', params.driver_curl_target, params.target), not_if=format("test -f {target}"), creates=params.target, - path=["/bin", "usr/bin/"]) - - elif params.hive_jdbc_driver == "oracle.jdbc.driver.OracleDriver": - environment = { - "no_proxy": format("{ambari_server_hostname}") - } - - cmd = format( - "mkdir -p {artifact_dir} ; curl -kf -x \"\" --retry 10 {driver_curl_source} -o {driver_curl_target} && " - "cp {driver_curl_target} {target}") - - Execute(cmd, - not_if=format("test -f {target}"), path=["/bin", "/usr/bin/"], - environment=environment) + sudo = True) http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/mysql_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/mysql_server.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/mysql_server.py index 5d9c067..936d38f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/mysql_server.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/mysql_server.py @@ -19,6 +19,7 @@ limitations under the License. """ import sys +import os from resource_management import * from mysql_service import mysql_service @@ -26,7 +27,11 @@ from mysql_service import mysql_service class MysqlServer(Script): def install(self, env): - self.install_packages(env) + import params + Package('mysql-connector-java', action = "remove") + if os.path.exists(params.mysql_jdbc_driver_jar): + File(params.mysql_jdbc_driver_jar, action = "delete") + self.install_packages(env, exclude_packages=params.hive_exclude_packages) self.configure(env) def configure(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/params.py index 69babb0..2538251 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/params.py @@ -43,11 +43,12 @@ elif hive_jdbc_driver == "org.postgresql.Driver": jdbc_jar_name = "postgresql-jdbc.jar" jdbc_symlink_name = "postgres-jdbc-driver.jar" elif hive_jdbc_driver == "oracle.jdbc.driver.OracleDriver": - jdbc_jar_name = "ojdbc6.jar" + jdbc_jar_name = "ojdbc.jar" jdbc_symlink_name = "oracle-jdbc-driver.jar" check_db_connection_jar_name = "DBConnectionVerification.jar" check_db_connection_jar = format("/usr/lib/ambari-agent/{check_db_connection_jar_name}") +hive_jdbc_drivers_list = ["com.mysql.jdbc.Driver","org.postgresql.Driver","oracle.jdbc.driver.OracleDriver"] #common hive_metastore_port = get_port_from_url(config['configurations']['hive-site']['hive.metastore.uris']) #"9083" @@ -81,6 +82,18 @@ if 'role' in config and config['role'] in ["HIVE_SERVER", "HIVE_METASTORE"]: #hive-site hive_database_name = config['configurations']['hive-env']['hive_database_name'] +hive_database = config['configurations']['hive-env']['hive_database'] + + +mysql_jdbc_driver_jar = "/usr/share/java/mysql-connector-java.jar" + +if hive_database.startswith('Existing'): + hive_exclude_packages = ['mysql-connector-java','mysql','mysql-server'] +else: + if 'role' in config and config['role'] != "MYSQL_SERVER": + hive_exclude_packages = ['mysql','mysql-server'] + else: + hive_exclude_packages = [] #Starting hiveserver2 start_hiveserver2_script = 'startHiveserver2.sh' http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/webhcat_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/webhcat_server.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/webhcat_server.py index 088cb41..0794ea1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/webhcat_server.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/webhcat_server.py @@ -26,7 +26,8 @@ from webhcat_service import webhcat_service class WebHCatServer(Script): def install(self, env): - self.install_packages(env) + import params + self.install_packages(env, exclude_packages=params.hive_exclude_packages) def configure(self, env): import params env.set_params(params) http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/oozie.py index 72d6bb1..c567515 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/oozie.py @@ -139,13 +139,27 @@ def oozie_server_specific( not_if = no_op_test, sudo = True ) - + if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver": - Execute(('cp', params.jdbc_driver_jar, params.oozie_libext_dir), - not_if = no_op_test, - sudo = True - ) - + + environment = { + "no_proxy": format("{ambari_server_hostname}") + } + + Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o', params.driver_curl_target), + not_if=format("test -f {target}"), + path=["/bin", "/usr/bin/"], + environment=environment, + sudo = True) + + + Execute(('cp', params.driver_curl_target, params.target), + not_if=format("test -f {target}"), + creates=params.target, + path=["/bin", "/usr/bin/"], + sudo = True) + + oozie_setup_cmd = format("cd {oozie_tmp_dir} && /usr/lib/oozie/bin/oozie-setup.sh -hadoop 0.20.200 {hadoop_jar_location} -extjs {ext_js_path} {jar_option} {jar_path}") Execute( oozie_setup_cmd, user = params.oozie_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/oozie_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/oozie_service.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/oozie_service.py index 6f94e01..aaabcbd 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/oozie_service.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/oozie_service.py @@ -32,7 +32,7 @@ def oozie_service(action = 'start'): # 'start' or 'stop' if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \ params.jdbc_driver_name == "org.postgresql.Driver" or \ params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver": - db_connection_check_command = format("{java_home}/bin/java -cp {check_db_connection_jar}:{jdbc_driver_jar} org.apache.ambari.server.DBConnectionVerification '{oozie_jdbc_connection_url}' {oozie_metastore_user_name} {oozie_metastore_user_passwd!p} {jdbc_driver_name}") + db_connection_check_command = format("{java_home}/bin/java -cp {check_db_connection_jar}:{target} org.apache.ambari.server.DBConnectionVerification '{oozie_jdbc_connection_url}' {oozie_metastore_user_name} {oozie_metastore_user_passwd!p} {jdbc_driver_name}") else: db_connection_check_command = None http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/params.py index deb7549..1acd55f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/params.py @@ -72,13 +72,24 @@ oozie_env_sh_template = config['configurations']['oozie-env']['content'] if jdbc_driver_name == "com.mysql.jdbc.Driver": jdbc_driver_jar = "/usr/share/java/mysql-connector-java.jar" + jdbc_symlink_name = "mysql-jdbc-driver.jar" elif jdbc_driver_name == "org.postgresql.Driver": - jdbc_driver_jar = "/usr/lib/oozie/libtools/postgresql-9.0-801.jdbc4.jar" + jdbc_driver_jar = format("{oozie_home}/libtools/postgresql-9.0-801.jdbc4.jar") #oozie using it's own postgres jdbc + jdbc_symlink_name = "postgres-jdbc-driver.jar" elif jdbc_driver_name == "oracle.jdbc.driver.OracleDriver": jdbc_driver_jar = "/usr/share/java/ojdbc6.jar" + jdbc_symlink_name = "oracle-jdbc-driver.jar" else: jdbc_driver_jar = "" - + jdbc_symlink_name = "" + +driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}") +driver_curl_target = format("{java_share_dir}/{jdbc_driver_jar}") +if jdbc_driver_name == "org.postgresql.Driver": + target = jdbc_driver_jar +else: + target = format("{oozie_libext_dir}/{jdbc_driver_jar}") + if lzo_enabled or jdbc_driver_name: jar_option = "-jars" else: http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_metastore.py index d5ecad3..f88b68b 100644 --- a/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_metastore.py @@ -165,14 +165,18 @@ class TestHiveMetastore(RMFTestCase): self.assertNoMoreResources() def assert_configure_default(self): - self.assertResourceCalled('Execute', ('cp', - '/usr/share/java/mysql-connector-java.jar', - '/usr/lib/hive/lib//mysql-connector-java.jar'), - creates = '/usr/lib/hive/lib//mysql-connector-java.jar', - path = ['/bin', '/usr/bin/'], - sudo = True, - not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', - ) + self.assertResourceCalled('Execute', ('curl', '-kf', '-x', "", '--retry', '10', u'http://c6401.ambari.apache.org:8080/resources//mysql-jdbc-driver.jar', '-o', '/usr/share/java/mysql-connector-java.jar'), + environment = {'no_proxy': 'c6401.ambari.apache.org'}, + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) + self.assertResourceCalled('Execute', ('cp', '/usr/share/java/mysql-connector-java.jar', '/usr/lib/hive/lib//mysql-connector-java.jar'), + creates = '/usr/lib/hive/lib//mysql-connector-java.jar', + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) self.assertResourceCalled('Directory', '/etc/hive/conf', owner = 'hive', group = 'hadoop', @@ -232,14 +236,18 @@ class TestHiveMetastore(RMFTestCase): ) def assert_configure_secured(self): - self.assertResourceCalled('Execute', ('cp', - '/usr/share/java/mysql-connector-java.jar', - '/usr/lib/hive/lib//mysql-connector-java.jar'), - creates = '/usr/lib/hive/lib//mysql-connector-java.jar', - path = ['/bin', '/usr/bin/'], - sudo = True, - not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', - ) + self.assertResourceCalled('Execute', ('curl', '-kf', '-x', "", '--retry', '10', u'http://c6401.ambari.apache.org:8080/resources//mysql-jdbc-driver.jar', '-o', '/usr/share/java/mysql-connector-java.jar'), + environment = {'no_proxy': 'c6401.ambari.apache.org'}, + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) + self.assertResourceCalled('Execute', ('cp', '/usr/share/java/mysql-connector-java.jar', '/usr/lib/hive/lib//mysql-connector-java.jar'), + creates = '/usr/lib/hive/lib//mysql-connector-java.jar', + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) self.assertResourceCalled('Directory', '/etc/hive/conf', owner = 'hive', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_server.py index 591f182..8f788dc 100644 --- a/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_server.py +++ b/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_server.py @@ -214,14 +214,18 @@ class TestHiveServer(RMFTestCase): kinit_path_local = "/usr/bin/kinit", action = ['create'], ) - self.assertResourceCalled('Execute', ('cp', - '/usr/share/java/mysql-connector-java.jar', - '/usr/lib/hive/lib//mysql-connector-java.jar'), - creates = '/usr/lib/hive/lib//mysql-connector-java.jar', - path = ['/bin', '/usr/bin/'], - sudo = True, - not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', - ) + self.assertResourceCalled('Execute', ('curl', '-kf', '-x', "", '--retry', '10', u'http://c6401.ambari.apache.org:8080/resources//mysql-jdbc-driver.jar', '-o', '/usr/share/java/mysql-connector-java.jar'), + environment = {'no_proxy': 'c6401.ambari.apache.org'}, + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) + self.assertResourceCalled('Execute', ('cp', '/usr/share/java/mysql-connector-java.jar', '/usr/lib/hive/lib//mysql-connector-java.jar'), + creates = '/usr/lib/hive/lib//mysql-connector-java.jar', + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) self.assertResourceCalled('Directory', '/etc/hive/conf', owner = 'hive', group = 'hadoop', @@ -309,14 +313,18 @@ class TestHiveServer(RMFTestCase): kinit_path_local = '/usr/bin/kinit', action = ['create'], ) - self.assertResourceCalled('Execute', ('cp', - '/usr/share/java/mysql-connector-java.jar', - '/usr/lib/hive/lib//mysql-connector-java.jar'), - creates = '/usr/lib/hive/lib//mysql-connector-java.jar', - path = ['/bin', '/usr/bin/'], - sudo = True, - not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', - ) + self.assertResourceCalled('Execute', ('curl', '-kf', '-x', "", '--retry', '10', u'http://c6401.ambari.apache.org:8080/resources//mysql-jdbc-driver.jar', '-o', '/usr/share/java/mysql-connector-java.jar'), + environment = {'no_proxy': 'c6401.ambari.apache.org'}, + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) + self.assertResourceCalled('Execute', ('cp', '/usr/share/java/mysql-connector-java.jar', '/usr/lib/hive/lib//mysql-connector-java.jar'), + creates = '/usr/lib/hive/lib//mysql-connector-java.jar', + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) self.assertResourceCalled('Directory', '/etc/hive/conf', owner = 'hive', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py index 853ef3e..8df34b8 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py @@ -209,17 +209,18 @@ class TestHiveMetastore(RMFTestCase): owner = 'hive', group = 'hadoop', ) - self.assertResourceCalled('File', '/usr/lib/hive/lib//mysql-connector-java.jar', - action = ['delete'], - ) - self.assertResourceCalled('Execute', ('cp', - '/usr/share/java/mysql-connector-java.jar', - '/usr/lib/hive/lib//mysql-connector-java.jar'), - creates = '/usr/lib/hive/lib//mysql-connector-java.jar', - path = ['/bin', '/usr/bin/'], - sudo = True, - not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', - ) + self.assertResourceCalled('Execute', ('curl', '-kf', '-x', "", '--retry', '10', u'http://c6401.ambari.apache.org:8080/resources//mysql-jdbc-driver.jar', '-o', '/usr/share/java/mysql-connector-java.jar'), + environment = {'no_proxy': 'c6401.ambari.apache.org'}, + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) + self.assertResourceCalled('Execute', ('cp', '/usr/share/java/mysql-connector-java.jar', '/usr/lib/hive/lib//mysql-connector-java.jar'), + creates = '/usr/lib/hive/lib//mysql-connector-java.jar', + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'', environment = {'no_proxy': 'c6401.ambari.apache.org'}, not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]', @@ -330,17 +331,18 @@ class TestHiveMetastore(RMFTestCase): owner = 'hive', group = 'hadoop', ) - self.assertResourceCalled('File', '/usr/lib/hive/lib//mysql-connector-java.jar', - action = ['delete'], - ) - self.assertResourceCalled('Execute', ('cp', - '/usr/share/java/mysql-connector-java.jar', - '/usr/lib/hive/lib//mysql-connector-java.jar'), - creates = '/usr/lib/hive/lib//mysql-connector-java.jar', - path = ['/bin', '/usr/bin/'], - sudo = True, - not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', - ) + self.assertResourceCalled('Execute', ('curl', '-kf', '-x', "", '--retry', '10', u'http://c6401.ambari.apache.org:8080/resources//mysql-jdbc-driver.jar', '-o', '/usr/share/java/mysql-connector-java.jar'), + environment = {'no_proxy': 'c6401.ambari.apache.org'}, + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) + self.assertResourceCalled('Execute', ('cp', '/usr/share/java/mysql-connector-java.jar', '/usr/lib/hive/lib//mysql-connector-java.jar'), + creates = '/usr/lib/hive/lib//mysql-connector-java.jar', + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'', environment = {'no_proxy': 'c6401.ambari.apache.org'}, not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]', http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py index 9017dd3..ee7d8af 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py @@ -329,17 +329,18 @@ class TestHiveServer(RMFTestCase): owner = 'hive', group = 'hadoop', ) - self.assertResourceCalled('File', '/usr/lib/hive/lib//mysql-connector-java.jar', - action = ['delete'], - ) - self.assertResourceCalled('Execute', ('cp', - '/usr/share/java/mysql-connector-java.jar', - '/usr/lib/hive/lib//mysql-connector-java.jar'), - creates = '/usr/lib/hive/lib//mysql-connector-java.jar', - path = ['/bin', '/usr/bin/'], - sudo = True, - not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', - ) + self.assertResourceCalled('Execute', ('curl', '-kf', '-x', "", '--retry', '10', u'http://c6401.ambari.apache.org:8080/resources//mysql-jdbc-driver.jar', '-o', '/usr/share/java/mysql-connector-java.jar'), + environment = {'no_proxy': 'c6401.ambari.apache.org'}, + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) + self.assertResourceCalled('Execute', ('cp', '/usr/share/java/mysql-connector-java.jar', '/usr/lib/hive/lib//mysql-connector-java.jar'), + creates = '/usr/lib/hive/lib//mysql-connector-java.jar', + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'', environment = {'no_proxy': 'c6401.ambari.apache.org'}, not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]', @@ -481,17 +482,18 @@ class TestHiveServer(RMFTestCase): owner = 'hive', group = 'hadoop', ) - self.assertResourceCalled('File', '/usr/lib/hive/lib//mysql-connector-java.jar', - action = ['delete'], - ) - self.assertResourceCalled('Execute', ('cp', - '/usr/share/java/mysql-connector-java.jar', - '/usr/lib/hive/lib//mysql-connector-java.jar'), - creates = '/usr/lib/hive/lib//mysql-connector-java.jar', - path = ['/bin', '/usr/bin/'], - sudo = True, - not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', - ) + self.assertResourceCalled('Execute', ('curl', '-kf', '-x', "", '--retry', '10', u'http://c6401.ambari.apache.org:8080/resources//mysql-jdbc-driver.jar', '-o', '/usr/share/java/mysql-connector-java.jar'), + environment = {'no_proxy': 'c6401.ambari.apache.org'}, + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) + self.assertResourceCalled('Execute', ('cp', '/usr/share/java/mysql-connector-java.jar', '/usr/lib/hive/lib//mysql-connector-java.jar'), + creates = '/usr/lib/hive/lib//mysql-connector-java.jar', + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'', environment = {'no_proxy': 'c6401.ambari.apache.org'}, not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]', http://git-wip-us.apache.org/repos/asf/ambari/blob/dcab1087/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py index 1709dcb..04aeb57 100644 --- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py @@ -187,17 +187,18 @@ class TestHiveMetastore(RMFTestCase): owner = 'hive', group = 'hadoop', ) - self.assertResourceCalled('File', '/usr/lib/hive/lib//mysql-connector-java.jar', - action = ['delete'], - ) - self.assertResourceCalled('Execute', ('cp', - '/usr/share/java/mysql-connector-java.jar', - '/usr/lib/hive/lib//mysql-connector-java.jar'), - creates = '/usr/lib/hive/lib//mysql-connector-java.jar', - path = ['/bin', '/usr/bin/'], - sudo = True, - not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', - ) + self.assertResourceCalled('Execute', ('curl', '-kf', '-x', "", '--retry', '10', u'http://c6401.ambari.apache.org:8080/resources//mysql-jdbc-driver.jar', '-o', '/usr/share/java/mysql-connector-java.jar'), + environment = {'no_proxy': 'c6401.ambari.apache.org'}, + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) + self.assertResourceCalled('Execute', ('cp', '/usr/share/java/mysql-connector-java.jar', '/usr/lib/hive/lib//mysql-connector-java.jar'), + creates = '/usr/lib/hive/lib//mysql-connector-java.jar', + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'', environment = {'no_proxy': 'c6401.ambari.apache.org'}, not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]', @@ -287,12 +288,13 @@ class TestHiveMetastore(RMFTestCase): owner = 'hive', group = 'hadoop', ) - self.assertResourceCalled('File', '/usr/lib/hive/lib//mysql-connector-java.jar', - action = ['delete'], + self.assertResourceCalled('Execute', ('curl', '-kf', '-x', "", '--retry', '10', u'http://c6401.ambari.apache.org:8080/resources//mysql-jdbc-driver.jar', '-o', '/usr/share/java/mysql-connector-java.jar'), + environment = {'no_proxy': 'c6401.ambari.apache.org'}, + path = ['/bin', '/usr/bin/'], + sudo = True, + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', ) - self.assertResourceCalled('Execute', ('cp', - '/usr/share/java/mysql-connector-java.jar', - '/usr/lib/hive/lib//mysql-connector-java.jar'), + self.assertResourceCalled('Execute', ('cp', '/usr/share/java/mysql-connector-java.jar', '/usr/lib/hive/lib//mysql-connector-java.jar'), creates = '/usr/lib/hive/lib//mysql-connector-java.jar', path = ['/bin', '/usr/bin/'], sudo = True,
