AMBARI-7223. Hive cannot start on HDP-2.2.1 Stack (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c2a117fe Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c2a117fe Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c2a117fe Branch: refs/heads/branch-alerts-dev Commit: c2a117febcfcc388de8dcea132fcb1fe7cc5491a Parents: 923549b Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Tue Sep 9 20:41:44 2014 +0300 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Tue Sep 9 20:41:44 2014 +0300 ---------------------------------------------------------------------- .../2.0.6/services/HIVE/package/scripts/hcat.py | 2 +- .../2.0.6/services/HIVE/package/scripts/hive.py | 118 ++--- .../services/HIVE/package/scripts/params.py | 9 +- .../stacks/2.0.6/HIVE/test_hive_client.py | 286 ++++++----- .../stacks/2.0.6/HIVE/test_hive_metastore.py | 413 +++++++-------- .../stacks/2.0.6/HIVE/test_hive_server.py | 504 ++++++++++--------- .../stacks/2.1/HIVE/test_hive_metastore.py | 304 +++++------ 7 files changed, 848 insertions(+), 788 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c2a117fe/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hcat.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hcat.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hcat.py index d14ea66..53a62ce 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hcat.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hcat.py @@ -36,7 +36,7 @@ def hcat(): ) XmlConfig("hive-site.xml", - conf_dir=params.hive_conf_dir, + conf_dir=params.hive_client_conf_dir, configurations=params.config['configurations']['hive-site'], configuration_attributes=params.config['configuration_attributes']['hive-site'], owner=params.hive_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/c2a117fe/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive.py index 690aa76..0b7fcb4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive.py @@ -39,52 +39,15 @@ def hive(name=None): mode=params.hive_hdfs_user_mode ) params.HdfsDirectory(None, action="create") + + # We should change configurations for client as well as for server. + # The reason is that stale-configs are service-level, not component. + for conf_dir in params.hive_conf_dirs_list: + fill_conf_dir(conf_dir) + if name == 'metastore' or name == 'hiveserver2': jdbc_connector() - - Directory(params.hive_conf_dir, - owner=params.hive_user, - group=params.user_group, - recursive=True - ) - Directory(params.hive_server_conf_dir, - owner=params.hive_user, - group=params.user_group, - recursive=True - ) - - XmlConfig("mapred-site.xml", - conf_dir=params.hive_conf_dir, - configurations=params.config['configurations']['mapred-site'], - configuration_attributes=params.config['configuration_attributes']['mapred-site'], - owner=params.hive_user, - group=params.user_group, - mode=0644) - - XmlConfig("hive-site.xml", - conf_dir=params.hive_conf_dir, - configurations=params.config['configurations']['hive-site'], - configuration_attributes=params.config['configuration_attributes']['hive-site'], - owner=params.hive_user, - group=params.user_group, - mode=0644) - - XmlConfig("mapred-site.xml", - conf_dir=params.hive_server_conf_dir, - configurations=params.config['configurations']['mapred-site'], - configuration_attributes=params.config['configuration_attributes']['mapred-site'], - owner=params.hive_user, - group=params.user_group, - mode=0600) - - XmlConfig("hive-site.xml", - conf_dir=params.hive_server_conf_dir, - configurations=params.config['configurations']['hive-site'], - configuration_attributes=params.config['configuration_attributes']['hive-site'], - owner=params.hive_user, - group=params.user_group, - mode=0600) - + environment = { "no_proxy": format("{ambari_server_hostname}") } @@ -98,31 +61,19 @@ def hive(name=None): not_if=format("[ -f {check_db_connection_jar_name}]"), environment = environment) - File(format("{hive_conf_dir}/hive-env.sh"), - owner=params.hive_user, - group=params.user_group, - content=InlineTemplate(params.hive_env_sh_template) - ) - - File(format("{hive_server_conf_dir}/hive-env.sh"), - owner=params.hive_user, - group=params.user_group, - content=InlineTemplate(params.hive_env_sh_template) - ) - if name == 'metastore': File(params.start_metastore_path, mode=0755, content=StaticFile('startMetastore.sh') ) if params.init_metastore_schema: - create_schema_cmd = format("export HIVE_CONF_DIR={hive_config_dir} ; " + create_schema_cmd = format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_bin}/schematool -initSchema " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p}") - check_schema_created_cmd = format("export HIVE_CONF_DIR={hive_config_dir} ; " + check_schema_created_cmd = format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_bin}/schematool -info " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " @@ -141,40 +92,71 @@ def hive(name=None): crt_directory(params.hive_pid_dir) crt_directory(params.hive_log_dir) crt_directory(params.hive_var_lib) + +def fill_conf_dir(component_conf_dir): + import params + + Directory(component_conf_dir, + owner=params.hive_user, + group=params.user_group, + recursive=True + ) - crt_file(format("{hive_conf_dir}/hive-default.xml.template")) - crt_file(format("{hive_conf_dir}/hive-env.sh.template")) + XmlConfig("mapred-site.xml", + conf_dir=component_conf_dir, + configurations=params.config['configurations']['mapred-site'], + configuration_attributes=params.config['configuration_attributes']['mapred-site'], + owner=params.hive_user, + group=params.user_group, + mode=0644) + + XmlConfig("hive-site.xml", + conf_dir=component_conf_dir, + configurations=params.config['configurations']['hive-site'], + configuration_attributes=params.config['configuration_attributes']['hive-site'], + owner=params.hive_user, + group=params.user_group, + mode=0644) + + File(format("{component_conf_dir}/hive-env.sh"), + owner=params.hive_user, + group=params.user_group, + content=InlineTemplate(params.hive_env_sh_template) + ) + + crt_file(format("{component_conf_dir}/hive-default.xml.template")) + crt_file(format("{component_conf_dir}/hive-env.sh.template")) log4j_exec_filename = 'hive-exec-log4j.properties' if (params.log4j_exec_props != None): - File(format("{params.hive_conf_dir}/{log4j_exec_filename}"), + File(format("{component_conf_dir}/{log4j_exec_filename}"), mode=0644, group=params.user_group, owner=params.hive_user, content=params.log4j_exec_props ) - elif (os.path.exists("{params.hive_conf_dir}/{log4j_exec_filename}.template")): - File(format("{params.hive_conf_dir}/{log4j_exec_filename}"), + elif (os.path.exists("{component_conf_dir}/{log4j_exec_filename}.template")): + File(format("{component_conf_dir}/{log4j_exec_filename}"), mode=0644, group=params.user_group, owner=params.hive_user, - content=StaticFile(format("{params.hive_conf_dir}/{log4j_exec_filename}.template")) + content=StaticFile(format("{component_conf_dir}/{log4j_exec_filename}.template")) ) log4j_filename = 'hive-log4j.properties' if (params.log4j_props != None): - File(format("{params.hive_conf_dir}/{log4j_filename}"), + File(format("{component_conf_dir}/{log4j_filename}"), mode=0644, group=params.user_group, owner=params.hive_user, content=params.log4j_props ) - elif (os.path.exists("{params.hive_conf_dir}/{log4j_filename}.template")): - File(format("{params.hive_conf_dir}/{log4j_filename}"), + elif (os.path.exists("{component_conf_dir}/{log4j_filename}.template")): + File(format("{component_conf_dir}/{log4j_filename}"), mode=0644, group=params.user_group, owner=params.hive_user, - content=StaticFile(format("{params.hive_conf_dir}/{log4j_filename}.template")) + content=StaticFile(format("{component_conf_dir}/{log4j_filename}.template")) ) http://git-wip-us.apache.org/repos/asf/ambari/blob/c2a117fe/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py index 5db3970..6f05b6b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py @@ -27,7 +27,6 @@ config = Script.get_config() tmp_dir = Script.get_tmp_dir() hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.option.ConnectionUserName'] -hive_server_conf_dir = "/etc/hive/conf.server" hive_jdbc_connection_url = config['configurations']['hive-site']['javax.jdo.option.ConnectionURL'] hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword'] @@ -73,15 +72,19 @@ kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/ hive_metastore_keytab_path = config['configurations']['hive-site']['hive.metastore.kerberos.keytab.file'] #hive_env -hive_conf_dir = "/etc/hive/conf" hive_dbroot = config['configurations']['hive-env']['hive_dbroot'] hive_log_dir = config['configurations']['hive-env']['hive_log_dir'] hive_pid_dir = status_params.hive_pid_dir hive_pid = status_params.hive_pid #Default conf dir for client -hive_config_dir = hive_conf_dir +hive_client_conf_dir = "/etc/hive/conf" +hive_server_conf_dir = "/etc/hive/conf.server" +hive_conf_dirs_list = [hive_server_conf_dir, hive_client_conf_dir] + if 'role' in config and config['role'] in ["HIVE_SERVER", "HIVE_METASTORE"]: hive_config_dir = hive_server_conf_dir +else: + hive_config_dir = hive_client_conf_dir #hive-site hive_database_name = config['configurations']['hive-env']['hive_database_name'] http://git-wip-us.apache.org/repos/asf/ambari/blob/c2a117fe/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py index 3ad5d86..407e0b8 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py @@ -28,84 +28,101 @@ class TestHiveClient(RMFTestCase): command = "configure", config_file="default_client.json" ) - - self.assertResourceCalled('Directory', '/etc/hive/conf', - owner = 'hive', - group = 'hadoop', - recursive = True, - ) self.assertResourceCalled('Directory', '/etc/hive/conf.server', - owner = 'hive', - group = 'hadoop', - recursive = True, + owner = 'hive', + group = 'hadoop', + recursive = True, ) self.assertResourceCalled('XmlConfig', 'mapred-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0644, - conf_dir = '/etc/hive/conf', - configurations = self.getConfig()['configurations']['mapred-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf.server', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['mapred-site'], ) self.assertResourceCalled('XmlConfig', 'hive-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0644, - conf_dir = '/etc/hive/conf', - configurations = self.getConfig()['configurations']['hive-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf.server', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['hive-site'], + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh', + content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-default.xml.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-exec-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('Directory', '/etc/hive/conf', + owner = 'hive', + group = 'hadoop', + recursive = True, ) self.assertResourceCalled('XmlConfig', 'mapred-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0600, - conf_dir = '/etc/hive/conf.server', - configurations = self.getConfig()['configurations']['mapred-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['mapred-site'], ) self.assertResourceCalled('XmlConfig', 'hive-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0600, - conf_dir = '/etc/hive/conf.server', - configurations = self.getConfig()['configurations']['hive-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'] - ) - self.assertResourceCalled('Execute', "/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf -x \"\" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar'", - not_if = '[ -f DBConnectionVerification.jar]', - environment = {'no_proxy': 'c6401.ambari.apache.org'} + group = 'hadoop', + conf_dir = '/etc/hive/conf', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['hive-site'], ) self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), - owner = 'hive', - group = 'hadoop', - ) - self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), - owner = 'hive', - group = 'hadoop', + content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), + owner = 'hive', + group = 'hadoop', ) self.assertResourceCalled('File', '/etc/hive/conf/hive-default.xml.template', - owner = 'hive', - group = 'hadoop', + owner = 'hive', + group = 'hadoop', ) self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh.template', - owner = 'hive', - group = 'hadoop', - ) - self.assertResourceCalled('File', - '/etc/hive/conf/hive-exec-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' - ) - self.assertResourceCalled('File', - '/etc/hive/conf/hive-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-exec-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'', + environment = {'no_proxy': 'c6401.ambari.apache.org'}, + not_if = '[ -f DBConnectionVerification.jar]', ) self.assertNoMoreResources() @@ -117,83 +134,100 @@ class TestHiveClient(RMFTestCase): command = "configure", config_file="secured_client.json" ) - - self.assertResourceCalled('Directory', '/etc/hive/conf', - owner = 'hive', - group = 'hadoop', - recursive = True, - ) self.assertResourceCalled('Directory', '/etc/hive/conf.server', - owner = 'hive', - group = 'hadoop', - recursive = True, + owner = 'hive', + group = 'hadoop', + recursive = True, ) self.assertResourceCalled('XmlConfig', 'mapred-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0644, - conf_dir = '/etc/hive/conf', - configurations = self.getConfig()['configurations']['mapred-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf.server', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['mapred-site'], ) self.assertResourceCalled('XmlConfig', 'hive-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0644, - conf_dir = '/etc/hive/conf', - configurations = self.getConfig()['configurations']['hive-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf.server', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['hive-site'], + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh', + content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-default.xml.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-exec-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('Directory', '/etc/hive/conf', + owner = 'hive', + group = 'hadoop', + recursive = True, ) self.assertResourceCalled('XmlConfig', 'mapred-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0600, - conf_dir = '/etc/hive/conf.server', - configurations = self.getConfig()['configurations']['mapred-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['mapred-site'], ) self.assertResourceCalled('XmlConfig', 'hive-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0600, - conf_dir = '/etc/hive/conf.server', - configurations = self.getConfig()['configurations']['hive-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'] - ) - self.assertResourceCalled('Execute', "/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf -x \"\" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar'", - not_if = '[ -f DBConnectionVerification.jar]', - environment = {'no_proxy': 'c6401.ambari.apache.org'} + group = 'hadoop', + conf_dir = '/etc/hive/conf', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['hive-site'], ) self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), - owner = 'hive', - group = 'hadoop', - ) - self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), - owner = 'hive', - group = 'hadoop', + content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), + owner = 'hive', + group = 'hadoop', ) self.assertResourceCalled('File', '/etc/hive/conf/hive-default.xml.template', - owner = 'hive', - group = 'hadoop', + owner = 'hive', + group = 'hadoop', ) self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh.template', - owner = 'hive', - group = 'hadoop', - ) - self.assertResourceCalled('File', - '/etc/hive/conf/hive-exec-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' - ) - self.assertResourceCalled('File', - '/etc/hive/conf/hive-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-exec-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'', + environment = {'no_proxy': 'c6401.ambari.apache.org'}, + not_if = '[ -f DBConnectionVerification.jar]', ) self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/c2a117fe/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py index dc76e57..0bef64d 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py @@ -38,29 +38,15 @@ class TestHiveMetastore(RMFTestCase): ) self.assert_configure_default() - self.assertResourceCalled('File', - '/etc/hive/conf/hive-exec-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' - ) - self.assertResourceCalled('File', - '/etc/hive/conf/hive-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' - ) self.assertResourceCalled('Execute', 'env HADOOP_HOME=/usr JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.log /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', - not_if = 'ls /var/run/hive/hive.pid >/dev/null 2>&1 && ps `cat /var/run/hive/hive.pid` >/dev/null 2>&1', - user = 'hive' + not_if = 'ls /var/run/hive/hive.pid >/dev/null 2>&1 && ps `cat /var/run/hive/hive.pid` >/dev/null 2>&1', + user = 'hive', ) - self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\' hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver', - path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'], tries=5, try_sleep=10 + path = ['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'], + tries = 5, + try_sleep = 10, ) - self.assertNoMoreResources() def test_stop_default(self): @@ -82,20 +68,6 @@ class TestHiveMetastore(RMFTestCase): config_file="secured.json" ) self.assert_configure_default() - self.assertResourceCalled('File', - '/etc/hive/conf/hive-exec-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' - ) - self.assertResourceCalled('File', - '/etc/hive/conf/hive-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' - ) self.assertNoMoreResources() def test_start_secured(self): @@ -106,29 +78,15 @@ class TestHiveMetastore(RMFTestCase): ) self.assert_configure_secured() - self.assertResourceCalled('File', - '/etc/hive/conf/hive-exec-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' - ) - self.assertResourceCalled('File', - '/etc/hive/conf/hive-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' - ) self.assertResourceCalled('Execute', 'env HADOOP_HOME=/usr JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.log /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', - not_if = 'ls /var/run/hive/hive.pid >/dev/null 2>&1 && ps `cat /var/run/hive/hive.pid` >/dev/null 2>&1', - user = 'hive' + not_if = 'ls /var/run/hive/hive.pid >/dev/null 2>&1 && ps `cat /var/run/hive/hive.pid` >/dev/null 2>&1', + user = 'hive', ) - self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\' hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver', - path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'], tries=5, try_sleep=10 + path = ['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'], + tries = 5, + try_sleep = 10, ) - self.assertNoMoreResources() def test_stop_secured(self): @@ -144,188 +102,251 @@ class TestHiveMetastore(RMFTestCase): self.assertNoMoreResources() def assert_configure_default(self): - self.assertResourceCalled('Execute', 'hive mkdir -p /tmp/AMBARI-artifacts/ ; cp /usr/share/java/mysql-connector-java.jar /usr/lib/hive/lib//mysql-connector-java.jar', - creates = '/usr/lib/hive/lib//mysql-connector-java.jar', - path = ['/bin', '/usr/bin/'], - not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', - ) - self.assertResourceCalled('Directory', '/etc/hive/conf', - owner = 'hive', - group = 'hadoop', - recursive = True, - ) self.assertResourceCalled('Directory', '/etc/hive/conf.server', - owner = 'hive', - group = 'hadoop', - recursive = True, + owner = 'hive', + group = 'hadoop', + recursive = True, ) self.assertResourceCalled('XmlConfig', 'mapred-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0644, - conf_dir = '/etc/hive/conf', - configurations = self.getConfig()['configurations']['mapred-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf.server', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['mapred-site'], ) self.assertResourceCalled('XmlConfig', 'hive-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0644, - conf_dir = '/etc/hive/conf', - configurations = self.getConfig()['configurations']['hive-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf.server', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['hive-site'], + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh', + content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-default.xml.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-exec-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('Directory', '/etc/hive/conf', + owner = 'hive', + group = 'hadoop', + recursive = True, ) self.assertResourceCalled('XmlConfig', 'mapred-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0600, - conf_dir = '/etc/hive/conf.server', - configurations = self.getConfig()['configurations']['mapred-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['mapred-site'], ) self.assertResourceCalled('XmlConfig', 'hive-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0600, - conf_dir = '/etc/hive/conf.server', - configurations = self.getConfig()['configurations']['hive-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'] - ) - self.assertResourceCalled('Execute', "/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf -x \"\" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar'", - not_if = '[ -f DBConnectionVerification.jar]', - environment = {'no_proxy': 'c6401.ambari.apache.org'} + group = 'hadoop', + conf_dir = '/etc/hive/conf', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['hive-site'], ) self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), - owner = 'hive', - group = 'hadoop', - ) - self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), - owner = 'hive', - group = 'hadoop', + content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-default.xml.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-exec-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('Execute', 'hive mkdir -p /tmp/AMBARI-artifacts/ ; cp /usr/share/java/mysql-connector-java.jar /usr/lib/hive/lib//mysql-connector-java.jar', + creates = '/usr/lib/hive/lib//mysql-connector-java.jar', + path = ['/bin', '/usr/bin/'], + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) + self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'', + environment = {'no_proxy': 'c6401.ambari.apache.org'}, + not_if = '[ -f DBConnectionVerification.jar]', ) self.assertResourceCalled('File', '/tmp/start_metastore_script', - content = StaticFile('startMetastore.sh'), - mode = 0755, + content = StaticFile('startMetastore.sh'), + mode = 0755, ) self.assertResourceCalled('Directory', '/var/run/hive', - owner = 'hive', - group = 'hadoop', - mode = 0755, - recursive = True, + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, ) self.assertResourceCalled('Directory', '/var/log/hive', - owner = 'hive', - group = 'hadoop', - mode = 0755, - recursive = True, + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, ) self.assertResourceCalled('Directory', '/var/lib/hive', - owner = 'hive', - group = 'hadoop', - mode = 0755, - recursive = True, - ) - self.assertResourceCalled('File', '/etc/hive/conf/hive-default.xml.template', - owner = 'hive', - group = 'hadoop', - ) - self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh.template', - owner = 'hive', - group = 'hadoop', + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, ) def assert_configure_secured(self): - self.assertResourceCalled('Execute', 'hive mkdir -p /tmp/AMBARI-artifacts/ ; cp /usr/share/java/mysql-connector-java.jar /usr/lib/hive/lib//mysql-connector-java.jar', - creates = '/usr/lib/hive/lib//mysql-connector-java.jar', - path = ['/bin', '/usr/bin/'], - not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', - ) - self.assertResourceCalled('Directory', '/etc/hive/conf', - owner = 'hive', - group = 'hadoop', - recursive = True, - ) self.assertResourceCalled('Directory', '/etc/hive/conf.server', - owner = 'hive', - group = 'hadoop', - recursive = True, + owner = 'hive', + group = 'hadoop', + recursive = True, ) self.assertResourceCalled('XmlConfig', 'mapred-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0644, - conf_dir = '/etc/hive/conf', - configurations = self.getConfig()['configurations']['mapred-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf.server', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['mapred-site'], ) self.assertResourceCalled('XmlConfig', 'hive-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0644, - conf_dir = '/etc/hive/conf', - configurations = self.getConfig()['configurations']['hive-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf.server', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['hive-site'], + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh', + content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-default.xml.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-exec-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('Directory', '/etc/hive/conf', + owner = 'hive', + group = 'hadoop', + recursive = True, ) self.assertResourceCalled('XmlConfig', 'mapred-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0600, - conf_dir = '/etc/hive/conf.server', - configurations = self.getConfig()['configurations']['mapred-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['mapred-site'], ) self.assertResourceCalled('XmlConfig', 'hive-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0600, - conf_dir = '/etc/hive/conf.server', - configurations = self.getConfig()['configurations']['hive-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'] - ) - self.assertResourceCalled('Execute', "/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf -x \"\" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar'", - not_if = '[ -f DBConnectionVerification.jar]', - environment = {'no_proxy': 'c6401.ambari.apache.org'} + group = 'hadoop', + conf_dir = '/etc/hive/conf', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['hive-site'], ) self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), - owner = 'hive', - group = 'hadoop', - ) - self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), - owner = 'hive', - group = 'hadoop', + content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-default.xml.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-exec-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('Execute', 'hive mkdir -p /tmp/AMBARI-artifacts/ ; cp /usr/share/java/mysql-connector-java.jar /usr/lib/hive/lib//mysql-connector-java.jar', + creates = '/usr/lib/hive/lib//mysql-connector-java.jar', + path = ['/bin', '/usr/bin/'], + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) + self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'', + environment = {'no_proxy': 'c6401.ambari.apache.org'}, + not_if = '[ -f DBConnectionVerification.jar]', ) self.assertResourceCalled('File', '/tmp/start_metastore_script', - content = StaticFile('startMetastore.sh'), - mode = 0755, + content = StaticFile('startMetastore.sh'), + mode = 0755, ) self.assertResourceCalled('Directory', '/var/run/hive', - owner = 'hive', - group = 'hadoop', - mode = 0755, - recursive = True, + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, ) self.assertResourceCalled('Directory', '/var/log/hive', - owner = 'hive', - group = 'hadoop', - mode = 0755, - recursive = True, + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, ) self.assertResourceCalled('Directory', '/var/lib/hive', - owner = 'hive', - group = 'hadoop', - mode = 0755, - recursive = True, - ) - self.assertResourceCalled('File', '/etc/hive/conf/hive-default.xml.template', - owner = 'hive', - group = 'hadoop', - ) - self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh.template', - owner = 'hive', - group = 'hadoop', - ) - + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, + ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c2a117fe/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py index 214d902..1c7e47e 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py @@ -31,20 +31,6 @@ class TestHiveServer(RMFTestCase): config_file="default.json" ) self.assert_configure_default() - self.assertResourceCalled('File', - '/etc/hive/conf/hive-exec-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' - ) - self.assertResourceCalled('File', - '/etc/hive/conf/hive-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' - ) self.assertNoMoreResources() @patch("hive_service.check_fs_root") @@ -59,21 +45,6 @@ class TestHiveServer(RMFTestCase): ) self.assert_configure_default() - self.assertResourceCalled('File', - '/etc/hive/conf/hive-exec-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' - ) - self.assertResourceCalled('File', - '/etc/hive/conf/hive-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' - ) - self.assertResourceCalled('HdfsDirectory', '/apps/tez/', action = ['create_delayed'], mode = 0755, @@ -157,20 +128,6 @@ class TestHiveServer(RMFTestCase): config_file="secured.json" ) self.assert_configure_secured() - self.assertResourceCalled('File', - '/etc/hive/conf/hive-exec-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' - ) - self.assertResourceCalled('File', - '/etc/hive/conf/hive-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' - ) self.assertNoMoreResources() @patch("hive_service.check_fs_root") @@ -185,20 +142,6 @@ class TestHiveServer(RMFTestCase): ) self.assert_configure_secured() - self.assertResourceCalled('File', - '/etc/hive/conf/hive-exec-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' - ) - self.assertResourceCalled('File', - '/etc/hive/conf/hive-log4j.properties', - mode=0644, - group='hadoop', - owner='hive', - content='log4jproperties\nline2' - ) self.assertResourceCalled('Execute', 'env JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /tmp/start_hiveserver2_script /var/log/hive/hive-server2.out /var/log/hive/hive-server2.log /var/run/hive/hive-server.pid /etc/hive/conf.server /var/log/hive', not_if = 'ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps `cat /var/run/hive/hive-server.pid` >/dev/null 2>&1', user = 'hive' @@ -230,245 +173,308 @@ class TestHiveServer(RMFTestCase): def assert_configure_default(self): self.assertResourceCalled('HdfsDirectory', '/apps/hive/warehouse', - security_enabled = False, - keytab = UnknownConfigurationMock(), - conf_dir = '/etc/hadoop/conf', - hdfs_user = 'hdfs', - kinit_path_local = "/usr/bin/kinit", - mode = 0777, - owner = 'hive', - action = ['create_delayed'], - ) + security_enabled = False, + keytab = UnknownConfigurationMock(), + conf_dir = '/etc/hadoop/conf', + hdfs_user = 'hdfs', + kinit_path_local = '/usr/bin/kinit', + mode = 0777, + owner = 'hive', + action = ['create_delayed'], + ) self.assertResourceCalled('HdfsDirectory', '/user/hive', - security_enabled = False, - keytab = UnknownConfigurationMock(), - conf_dir = '/etc/hadoop/conf', - hdfs_user = 'hdfs', - kinit_path_local = "/usr/bin/kinit", - mode = 0700, - owner = 'hive', - action = ['create_delayed'], - ) + security_enabled = False, + keytab = UnknownConfigurationMock(), + conf_dir = '/etc/hadoop/conf', + hdfs_user = 'hdfs', + kinit_path_local = '/usr/bin/kinit', + mode = 0700, + owner = 'hive', + action = ['create_delayed'], + ) self.assertResourceCalled('HdfsDirectory', None, - security_enabled = False, - keytab = UnknownConfigurationMock(), - conf_dir = '/etc/hadoop/conf', - hdfs_user = 'hdfs', - kinit_path_local = "/usr/bin/kinit", - action = ['create'], - ) - self.assertResourceCalled('Execute', 'hive mkdir -p /tmp/AMBARI-artifacts/ ; cp /usr/share/java/mysql-connector-java.jar /usr/lib/hive/lib//mysql-connector-java.jar', - creates = '/usr/lib/hive/lib//mysql-connector-java.jar', - path = ['/bin', '/usr/bin/'], - not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + security_enabled = False, + keytab = UnknownConfigurationMock(), + conf_dir = '/etc/hadoop/conf', + hdfs_user = 'hdfs', + kinit_path_local = '/usr/bin/kinit', + action = ['create'], ) - self.assertResourceCalled('Directory', '/etc/hive/conf', - owner = 'hive', - group = 'hadoop', - recursive = True, - ) self.assertResourceCalled('Directory', '/etc/hive/conf.server', - owner = 'hive', - group = 'hadoop', - recursive = True, + owner = 'hive', + group = 'hadoop', + recursive = True, ) self.assertResourceCalled('XmlConfig', 'mapred-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0644, - conf_dir = '/etc/hive/conf', - configurations = self.getConfig()['configurations']['mapred-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf.server', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['mapred-site'], ) self.assertResourceCalled('XmlConfig', 'hive-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0644, - conf_dir = '/etc/hive/conf', - configurations = self.getConfig()['configurations']['hive-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf.server', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['hive-site'], + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh', + content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-default.xml.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-exec-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('Directory', '/etc/hive/conf', + owner = 'hive', + group = 'hadoop', + recursive = True, ) self.assertResourceCalled('XmlConfig', 'mapred-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0600, - conf_dir = '/etc/hive/conf.server', - configurations = self.getConfig()['configurations']['mapred-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['mapred-site'], ) self.assertResourceCalled('XmlConfig', 'hive-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0600, - conf_dir = '/etc/hive/conf.server', - configurations = self.getConfig()['configurations']['hive-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'] - ) - self.assertResourceCalled('Execute', "/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf -x \"\" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar'", - not_if = '[ -f DBConnectionVerification.jar]', - environment = {'no_proxy': 'c6401.ambari.apache.org'} + group = 'hadoop', + conf_dir = '/etc/hive/conf', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['hive-site'], ) self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), - owner = 'hive', - group = 'hadoop', - ) - self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), - owner = 'hive', - group = 'hadoop', + content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-default.xml.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-exec-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('Execute', 'hive mkdir -p /tmp/AMBARI-artifacts/ ; cp /usr/share/java/mysql-connector-java.jar /usr/lib/hive/lib//mysql-connector-java.jar', + creates = '/usr/lib/hive/lib//mysql-connector-java.jar', + path = ['/bin', '/usr/bin/'], + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) + self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'', + environment = {'no_proxy': 'c6401.ambari.apache.org'}, + not_if = '[ -f DBConnectionVerification.jar]', ) self.assertResourceCalled('File', '/tmp/start_hiveserver2_script', - content = Template('startHiveserver2.sh.j2'), - mode = 0755, + content = Template('startHiveserver2.sh.j2'), + mode = 0755, ) self.assertResourceCalled('Directory', '/var/run/hive', - owner = 'hive', - group = 'hadoop', - mode = 0755, - recursive = True, + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, ) self.assertResourceCalled('Directory', '/var/log/hive', - owner = 'hive', - group = 'hadoop', - mode = 0755, - recursive = True, + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, ) self.assertResourceCalled('Directory', '/var/lib/hive', - owner = 'hive', - group = 'hadoop', - mode = 0755, - recursive = True, - ) - self.assertResourceCalled('File', '/etc/hive/conf/hive-default.xml.template', - owner = 'hive', - group = 'hadoop', + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, ) - self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh.template', - owner = 'hive', - group = 'hadoop', - ) - + def assert_configure_secured(self): self.assertResourceCalled('HdfsDirectory', '/apps/hive/warehouse', - security_enabled = True, - keytab = '/etc/security/keytabs/hdfs.headless.keytab', - conf_dir = '/etc/hadoop/conf', - hdfs_user = 'hdfs', - kinit_path_local = '/usr/bin/kinit', - mode = 0777, - owner = 'hive', - action = ['create_delayed'], - ) + security_enabled = True, + keytab = '/etc/security/keytabs/hdfs.headless.keytab', + conf_dir = '/etc/hadoop/conf', + hdfs_user = 'hdfs', + kinit_path_local = '/usr/bin/kinit', + mode = 0777, + owner = 'hive', + action = ['create_delayed'], + ) self.assertResourceCalled('HdfsDirectory', '/user/hive', - security_enabled = True, - keytab = '/etc/security/keytabs/hdfs.headless.keytab', - conf_dir = '/etc/hadoop/conf', - hdfs_user = 'hdfs', - kinit_path_local = '/usr/bin/kinit', - mode = 0700, - owner = 'hive', - action = ['create_delayed'], - ) + security_enabled = True, + keytab = '/etc/security/keytabs/hdfs.headless.keytab', + conf_dir = '/etc/hadoop/conf', + hdfs_user = 'hdfs', + kinit_path_local = '/usr/bin/kinit', + mode = 0700, + owner = 'hive', + action = ['create_delayed'], + ) self.assertResourceCalled('HdfsDirectory', None, - security_enabled = True, - keytab = '/etc/security/keytabs/hdfs.headless.keytab', - conf_dir = '/etc/hadoop/conf', - hdfs_user = 'hdfs', - kinit_path_local = '/usr/bin/kinit', - action = ['create'], - ) - - self.assertResourceCalled('Execute', 'hive mkdir -p /tmp/AMBARI-artifacts/ ; cp /usr/share/java/mysql-connector-java.jar /usr/lib/hive/lib//mysql-connector-java.jar', - creates = '/usr/lib/hive/lib//mysql-connector-java.jar', - path = ['/bin', '/usr/bin/'], - not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + security_enabled = True, + keytab = '/etc/security/keytabs/hdfs.headless.keytab', + conf_dir = '/etc/hadoop/conf', + hdfs_user = 'hdfs', + kinit_path_local = '/usr/bin/kinit', + action = ['create'], ) - self.assertResourceCalled('Directory', '/etc/hive/conf', - owner = 'hive', - group = 'hadoop', - recursive = True, - ) self.assertResourceCalled('Directory', '/etc/hive/conf.server', - owner = 'hive', - group = 'hadoop', - recursive = True, + owner = 'hive', + group = 'hadoop', + recursive = True, ) self.assertResourceCalled('XmlConfig', 'mapred-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0644, - conf_dir = '/etc/hive/conf', - configurations = self.getConfig()['configurations']['mapred-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf.server', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['mapred-site'], ) self.assertResourceCalled('XmlConfig', 'hive-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0644, - conf_dir = '/etc/hive/conf', - configurations = self.getConfig()['configurations']['hive-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf.server', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['hive-site'], + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh', + content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-default.xml.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-exec-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('Directory', '/etc/hive/conf', + owner = 'hive', + group = 'hadoop', + recursive = True, ) self.assertResourceCalled('XmlConfig', 'mapred-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0600, - conf_dir = '/etc/hive/conf.server', - configurations = self.getConfig()['configurations']['mapred-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'] + group = 'hadoop', + conf_dir = '/etc/hive/conf', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['mapred-site'], ) self.assertResourceCalled('XmlConfig', 'hive-site.xml', - owner = 'hive', - group = 'hadoop', - mode = 0600, - conf_dir = '/etc/hive/conf.server', - configurations = self.getConfig()['configurations']['hive-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'] - ) - self.assertResourceCalled('Execute', "/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf -x \"\" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar'", - not_if = '[ -f DBConnectionVerification.jar]', - environment = {'no_proxy': 'c6401.ambari.apache.org'} + group = 'hadoop', + conf_dir = '/etc/hive/conf', + mode = 0644, + configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'], + owner = 'hive', + configurations = self.getConfig()['configurations']['hive-site'], ) self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), - owner = 'hive', - group = 'hadoop', - ) - self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), - owner = 'hive', - group = 'hadoop', + content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']), + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-default.xml.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-exec-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hive', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('Execute', 'hive mkdir -p /tmp/AMBARI-artifacts/ ; cp /usr/share/java/mysql-connector-java.jar /usr/lib/hive/lib//mysql-connector-java.jar', + creates = '/usr/lib/hive/lib//mysql-connector-java.jar', + path = ['/bin', '/usr/bin/'], + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) + self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'', + environment = {'no_proxy': 'c6401.ambari.apache.org'}, + not_if = '[ -f DBConnectionVerification.jar]', ) self.assertResourceCalled('File', '/tmp/start_hiveserver2_script', - content = Template('startHiveserver2.sh.j2'), - mode = 0755, + content = Template('startHiveserver2.sh.j2'), + mode = 0755, ) self.assertResourceCalled('Directory', '/var/run/hive', - owner = 'hive', - group = 'hadoop', - mode = 0755, - recursive = True, + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, ) self.assertResourceCalled('Directory', '/var/log/hive', - owner = 'hive', - group = 'hadoop', - mode = 0755, - recursive = True, + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, ) self.assertResourceCalled('Directory', '/var/lib/hive', - owner = 'hive', - group = 'hadoop', - mode = 0755, - recursive = True, - ) - self.assertResourceCalled('File', '/etc/hive/conf/hive-default.xml.template', - owner = 'hive', - group = 'hadoop', - ) - self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh.template', - owner = 'hive', - group = 'hadoop', + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, ) @patch("hive_service.check_fs_root")