AMBARI-15079. Preupload.py should pre-create hdfs directories (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ad5ecf82 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ad5ecf82 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ad5ecf82 Branch: refs/heads/branch-2.2 Commit: ad5ecf826ab80a26f47d1637575c32d62bcc156c Parents: 878ae15 Author: Andrew Onishuk <[email protected]> Authored: Thu Feb 18 13:56:22 2016 +0200 Committer: Andrew Onishuk <[email protected]> Committed: Thu Feb 18 13:57:14 2016 +0200 ---------------------------------------------------------------------- .../libraries/providers/hdfs_resource.py | 64 +++++++++++++------- .../libraries/resources/hdfs_resource.py | 12 ++++ .../1.6.1.2.2.0/package/scripts/params.py | 1 + .../0.1.0/package/scripts/params.py | 1 + .../0.5.0.2.1/package/scripts/params_linux.py | 1 + .../0.96.0.2.0/package/scripts/params_linux.py | 1 + .../2.1.0.2.0/package/scripts/params_linux.py | 1 + .../0.12.0.2.0/package/scripts/params_linux.py | 1 + .../KAFKA/0.8.1.2.2/package/scripts/params.py | 1 + .../0.5.0.2.2/package/scripts/params_linux.py | 1 + .../MAHOUT/1.0.0.2.3/package/scripts/params.py | 1 + .../4.0.0.2.0/package/scripts/params_linux.py | 1 + .../0.12.0.2.0/package/scripts/params_linux.py | 1 + .../0.60.0.2.2/package/scripts/params_linux.py | 1 + .../SPARK/1.2.0.2.2/package/scripts/params.py | 1 + .../0.9.1.2.1/package/scripts/params_linux.py | 1 + .../0.4.0.2.1/package/scripts/params_linux.py | 1 + .../2.1.0.2.0/package/scripts/params_linux.py | 1 + .../main/resources/scripts/Ambaripreupload.py | 54 ++++++++++++++--- .../AMBARI_METRICS/test_metrics_collector.py | 6 +- .../stacks/2.0.6/HBASE/test_hbase_master.py | 18 +++--- .../python/stacks/2.0.6/HDFS/test_namenode.py | 54 ++++++++--------- .../stacks/2.0.6/HDFS/test_service_check.py | 8 +-- .../stacks/2.0.6/HIVE/test_hive_server.py | 28 ++++----- .../2.0.6/HIVE/test_hive_service_check.py | 12 ++-- .../stacks/2.0.6/OOZIE/test_oozie_server.py | 32 +++++----- .../stacks/2.0.6/OOZIE/test_service_check.py | 10 +-- .../stacks/2.0.6/PIG/test_pig_service_check.py | 12 ++-- .../stacks/2.0.6/YARN/test_historyserver.py | 32 +++++----- .../2.0.6/YARN/test_mapreduce2_service_check.py | 12 ++-- .../stacks/2.1/FALCON/test_falcon_server.py | 12 ++-- .../python/stacks/2.1/TEZ/test_service_check.py | 16 ++--- .../stacks/2.1/YARN/test_apptimelineserver.py | 2 +- .../stacks/2.2/PIG/test_pig_service_check.py | 12 ++-- .../stacks/2.2/SPARK/test_job_history_server.py | 12 ++-- .../2.3/MAHOUT/test_mahout_service_check.py | 8 +-- .../2.3/SPARK/test_spark_thrift_server.py | 4 +- .../test/python/stacks/2.3/YARN/test_ats_1_5.py | 10 +-- 38 files changed, 267 insertions(+), 179 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py index b6edf1a..de0a0ad 100644 --- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py +++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py @@ -150,32 +150,13 @@ class WebHDFSUtil: # only hdfs seems to support webHDFS return (is_webhdfs_enabled and default_fs.startswith("hdfs")) - def parse_path(self, path): - """ - hdfs://nn_url:1234/a/b/c -> /a/b/c - hdfs://nn_ha_name/a/b/c -> /a/b/c - hdfs:///a/b/c -> /a/b/c - /a/b/c -> /a/b/c - """ - math_with_protocol_and_nn_url = re.match("[a-zA-Z]+://[^/]+(/.+)", path) - math_with_protocol = re.match("[a-zA-Z]+://(/.+)", path) - - if math_with_protocol_and_nn_url: - path = math_with_protocol_and_nn_url.group(1) - elif math_with_protocol: - path = math_with_protocol.group(1) - else: - path = path - - return re.sub("[/]+", "/", path) - valid_status_codes = ["200", "201"] def run_command(self, target, operation, method='POST', assertable_result=True, file_to_put=None, ignore_status_codes=[], **kwargs): """ assertable_result - some POST requests return '{"boolean":false}' or '{"boolean":true}' depending on if query was successful or not, we can assert this for them """ - target = self.parse_path(target) + target = HdfsResourceProvider.parse_path(target) url = format("{address}/webhdfs/v1{target}?op={operation}&user.name={run_user}", address=self.address, run_user=self.run_user) for k,v in kwargs.iteritems(): @@ -393,7 +374,7 @@ class HdfsResourceWebHDFS: def _fill_in_parent_directories(self, target, results): - path_parts = self.util.parse_path(target).split("/")[1:]# [1:] remove '' from parts + path_parts = HdfsResourceProvider.parse_path(target).split("/")[1:]# [1:] remove '' from parts path = "/" for path_part in path_parts: @@ -415,11 +396,50 @@ class HdfsResourceProvider(Provider): def __init__(self, resource): super(HdfsResourceProvider,self).__init__(resource) self.assert_parameter_is_set('hdfs_site') - + self.ignored_resources_list = self.get_ignored_resources_list() self.webhdfs_enabled = self.resource.hdfs_site['dfs.webhdfs.enabled'] + + @staticmethod + def parse_path(path): + """ + hdfs://nn_url:1234/a/b/c -> /a/b/c + hdfs://nn_ha_name/a/b/c -> /a/b/c + hdfs:///a/b/c -> /a/b/c + /a/b/c -> /a/b/c + """ + math_with_protocol_and_nn_url = re.match("[a-zA-Z]+://[^/]+(/.+)", path) + math_with_protocol = re.match("[a-zA-Z]+://(/.+)", path) + + if math_with_protocol_and_nn_url: + path = math_with_protocol_and_nn_url.group(1) + elif math_with_protocol: + path = math_with_protocol.group(1) + else: + path = path + + return re.sub("[/]+", "/", path) + + def get_ignored_resources_list(self): + if not self.resource.hdfs_resource_ignore_file or not os.path.exists(self.resource.hdfs_resource_ignore_file): + return [] + + with open(self.resource.hdfs_resource_ignore_file, "rb") as fp: + content = fp.read() + + hdfs_resources_to_ignore = [] + for hdfs_resource_to_ignore in content.split("\n"): + hdfs_resources_to_ignore.append(HdfsResourceProvider.parse_path(hdfs_resource_to_ignore)) + + return hdfs_resources_to_ignore + + def action_delayed(self, action_name): self.assert_parameter_is_set('type') + + if HdfsResourceProvider.parse_path(self.resource.target) in self.ignored_resources_list: + Logger.info("Skipping '{0}' because it is in ignore file {1}.".format(self.resource, self.resource.hdfs_resource_ignore_file)) + return self.get_hdfs_resource_executor().action_delayed(action_name, self) http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py index 69aeab0..06d0f2b 100644 --- a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py +++ b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py @@ -76,6 +76,18 @@ class HdfsResource(Resource): hadoop_bin_dir = ResourceArgument() hadoop_conf_dir = ResourceArgument() + """ + Path to file which contains '\n'-separated list of hdfs resources, which should not + be managed. (simply any action to be skipped on it) + + This mean that if HdfsResource('/test1'..) is executed and /test1 is one of the lines + in the given file, the execution will be ignored. + + Example value: + /var/lib/ambari-agent/data/.hdfs_resource_ignore + """ + hdfs_resource_ignore_file = ResourceArgument() + # WebHDFS needs these hdfs_site = ResourceArgument() default_fs = ResourceArgument() http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py index 5a88591..09af8a9 100644 --- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py @@ -183,6 +183,7 @@ import functools HdfsResource = functools.partial( HdfsResource, user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py index f3a97fc..f1ff998 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py @@ -265,6 +265,7 @@ import functools HdfsResource = functools.partial( HdfsResource, user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py index d79026a..7564905 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py @@ -117,6 +117,7 @@ import functools HdfsResource = functools.partial( HdfsResource, user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py index 365a530..4052f83 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py @@ -226,6 +226,7 @@ import functools HdfsResource = functools.partial( HdfsResource, user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index a34ee04..bcfa4f3 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -322,6 +322,7 @@ import functools HdfsResource = functools.partial( HdfsResource, user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 5ee4276..a66b6d0 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -455,6 +455,7 @@ import functools HdfsResource = functools.partial( HdfsResource, user = hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py index de7b379..dc46749 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py @@ -274,6 +274,7 @@ import functools HdfsResource = functools.partial( HdfsResource, user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py index 46a6f9a..9d4847f 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py @@ -323,6 +323,7 @@ import functools HdfsResource = functools.partial( HdfsResource, user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/params.py index 7ec31de..b261b23 100644 --- a/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/params.py @@ -80,6 +80,7 @@ import functools HdfsResource = functools.partial( HdfsResource, user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py index 573d5fa..b7ee0be 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py @@ -262,6 +262,7 @@ import functools HdfsResource = functools.partial( HdfsResource, user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py index 84b34b2..17998c6 100644 --- a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py @@ -81,6 +81,7 @@ import functools HdfsResource = functools.partial( HdfsResource, user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/params_linux.py index 09b7876..132ff77 100644 --- a/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/params_linux.py @@ -64,6 +64,7 @@ import functools HdfsResource = functools.partial( HdfsResource, user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py index 4fd1d6b..52c78c5 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py @@ -181,6 +181,7 @@ import functools HdfsResource = functools.partial( HdfsResource, user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py index b76fb69..94988ce 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py @@ -295,6 +295,7 @@ import functools HdfsResource = functools.partial( HdfsResource, user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py index 31e1a7c..f0ba617 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py @@ -85,6 +85,7 @@ import functools HdfsResource = functools.partial( HdfsResource, user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index a24b849..83f47a8 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -266,6 +266,7 @@ import functools HdfsResource = functools.partial( HdfsResource, user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/main/resources/scripts/Ambaripreupload.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/scripts/Ambaripreupload.py b/ambari-server/src/main/resources/scripts/Ambaripreupload.py index 61db286..a803de4 100644 --- a/ambari-server/src/main/resources/scripts/Ambaripreupload.py +++ b/ambari-server/src/main/resources/scripts/Ambaripreupload.py @@ -166,7 +166,8 @@ with Environment() as env: hadoop_conf_dir = hadoop_conf_dir, principal_name = None, hdfs_site = hdfs_site, - default_fs = fs_default + default_fs = fs_default, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", ) def _copy_files(source_and_dest_pairs, file_owner, group_owner, kinit_if_needed): @@ -231,7 +232,40 @@ with Environment() as env: source_and_dest_pairs = [(component_tar_source_file, destination_file), ] return _copy_files(source_and_dest_pairs, file_owner, group_owner, kinit_if_needed) + + def createHdfsResources(): + params.HdfsResource('/atshistory', user='hdfs', change_permissions_for_parents=True, owner='yarn', group='hadoop', type='directory', action= ['create_on_execute'], mode=0755) + params.HdfsResource('/user/hcat', owner='hcat', type='directory', action=['create_on_execute'], mode=0755) + params.HdfsResource('/hive/warehouse', owner='hive', type='directory', action=['create_on_execute'], mode=0777) + params.HdfsResource('/user/hive', owner='hive', type='directory', action=['create_on_execute'], mode=0755) + params.HdfsResource('/tmp', mode=0777, action=['create_on_execute'], type='directory', owner='hdfs') + params.HdfsResource('/user/ambari-qa', type='directory', action=['create_on_execute'], mode=0770) + params.HdfsResource('/user/oozie', owner='oozie', type='directory', action=['create_on_execute'], mode=0775) + params.HdfsResource('/app-logs', recursive_chmod=True, owner='yarn', group='hadoop', type='directory', action=['create_on_execute'], mode=0777) + params.HdfsResource('/tmp/entity-file-history/active', owner='yarn', group='hadoop', type='directory', action=['create_on_execute']) + params.HdfsResource('/mapred', owner='mapred', type='directory', action=['create_on_execute']) + params.HdfsResource('/mapred/system', owner='hdfs', type='directory', action=['create_on_execute']) + params.HdfsResource('/mr-history/done', change_permissions_for_parents=True, owner='mapred', group='hadoop', type='directory', action=['create_on_execute'], mode=0777) + params.HdfsResource('/atshistory/done', owner='yarn', group='hadoop', type='directory', action=['create_on_execute'], mode=0700) + params.HdfsResource('/atshistory/active', owner='yarn', group='hadoop', type='directory', action=['create_on_execute'], mode=01777) + params.HdfsResource('/ams/hbase', owner='ams', type='directory', action=['create_on_execute'], mode=0775) + params.HdfsResource('/amshbase/staging', owner='ams', type='directory', action=['create_on_execute'], mode=0711) + params.HdfsResource('/user/ams/hbase', owner='ams', type='directory', action=['create_on_execute'], mode=0775) + + def putCreatedHdfsResourcesToIgnore(env): + if not 'hdfs_files' in env.config: + Logger.info("Not creating .hdfs_resource_ignore as no resources to use.") + return + + file_content = "" + for file in env.config['hdfs_files']: + file_content += file['target'] + file_content += "\n" + + with open("/var/lib/ambari-agent/data/.hdfs_resource_ignore", "a+") as fp: + fp.write(file_content) + env.set_params(params) hadoop_conf_dir = params.hadoop_conf_dir @@ -272,7 +306,7 @@ with Environment() as env: # DON'T CHANGE THE VALUE SINCE IT'S USED TO DETERMINE WHETHER TO RUN THE COMMAND OR NOT BY READING THE MARKER FILE. # Oozie tmp dir should be /var/tmp/oozie and is already created by a function above. command = format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war {oozie_secure} ") - command_to_file = format("cd {oozie_tmp_dir} && {oozie_setup_sh_current} prepare-war {oozie_secure} ") + command_to_file = format("cd {oozie_tmp_dir} && {oozie_setup_sh_current} prepare-war {oozie_secure} ").strip() run_prepare_war = False if os.path.exists(prepare_war_cmd_file): @@ -338,7 +372,9 @@ with Environment() as env: copy_tarballs_to_hdfs(format("/usr/hdp/{hdp_version}/pig/pig.tar.gz"), hdfs_path_prefix+"/hdp/apps/{{ hdp_stack_version }}/pig/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group) copy_tarballs_to_hdfs(format("/usr/hdp/{hdp_version}/hadoop-mapreduce/hadoop-streaming.jar"), hdfs_path_prefix+"/hdp/apps/{{ hdp_stack_version }}/mapreduce/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group) copy_tarballs_to_hdfs(format("/usr/hdp/{hdp_version}/sqoop/sqoop.tar.gz"), hdfs_path_prefix+"/hdp/apps/{{ hdp_stack_version }}/sqoop/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group) - + print "Creating hdfs directories..." + createHdfsResources() + putCreatedHdfsResourcesToIgnore(env) # jar shouldn't be used before (read comment below) File(format("{ambari_libs_dir}/fast-hdfs-resource.jar"), @@ -347,8 +383,12 @@ with Environment() as env: ) # Create everything in one jar call (this is fast). # (! Before everything should be executed with action="create_on_execute/delete_on_execute" for this time-optimization to work) - params.HdfsResource(None, - logoutput=True, - action="execute" - ) + try: + params.HdfsResource(None, + logoutput=True, + action="execute" + ) + except: + os.remove("/var/lib/ambari-agent/data/.hdfs_resource_ignore") + raise print "Completed tarball copy. Ambari preupload script completed." http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py index e774493..3066a07 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py +++ b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py @@ -319,7 +319,7 @@ class TestMetricsCollector(RMFTestCase): mode = 0775, hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', @@ -334,7 +334,7 @@ class TestMetricsCollector(RMFTestCase): mode = 0711, hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', @@ -346,7 +346,7 @@ class TestMetricsCollector(RMFTestCase): kinit_path_local = '/usr/bin/kinit', user = 'hdfs', hadoop_conf_dir = '/etc/hadoop/conf', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py index 80f1a15..8a6f2eb 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py @@ -332,7 +332,7 @@ class TestHBaseMaster(RMFTestCase): owner = 'hbase', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', ) self.assertResourceCalled('HdfsResource', '/apps/hbase/staging', security_enabled = False, @@ -344,7 +344,7 @@ class TestHBaseMaster(RMFTestCase): owner = 'hbase', hadoop_bin_dir = '/usr/bin', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', mode = 0711, ) self.assertResourceCalled('HdfsResource', None, @@ -354,7 +354,7 @@ class TestHBaseMaster(RMFTestCase): kinit_path_local = '/usr/bin/kinit', user = 'hdfs', - action = ['execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', hadoop_conf_dir = '/etc/hadoop/conf', ) @@ -467,7 +467,7 @@ class TestHBaseMaster(RMFTestCase): owner = 'hbase', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', ) self.assertResourceCalled('HdfsResource', '/apps/hbase/staging', security_enabled = True, @@ -479,7 +479,7 @@ class TestHBaseMaster(RMFTestCase): owner = 'hbase', hadoop_bin_dir = '/usr/bin', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', mode = 0711, ) self.assertResourceCalled('HdfsResource', None, @@ -489,7 +489,7 @@ class TestHBaseMaster(RMFTestCase): kinit_path_local = '/usr/bin/kinit', user = 'hdfs', - action = ['execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', hadoop_conf_dir = '/etc/hadoop/conf', ) @@ -613,7 +613,7 @@ class TestHBaseMaster(RMFTestCase): owner = 'hbase', hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', ) self.assertResourceCalled('HdfsResource', '/apps/hbase/staging', security_enabled = False, @@ -627,7 +627,7 @@ class TestHBaseMaster(RMFTestCase): owner = 'hbase', hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0711, ) self.assertResourceCalled('HdfsResource', None, @@ -639,7 +639,7 @@ class TestHBaseMaster(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = UnknownConfigurationMock(), user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', ) http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py index c79cd5f..2f1f66c 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py @@ -107,7 +107,7 @@ class TestNamenode(RMFTestCase): owner = 'hdfs', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0777, ) self.assertResourceCalled('HdfsResource', '/user/ambari-qa', @@ -123,7 +123,7 @@ class TestNamenode(RMFTestCase): owner = 'ambari-qa', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0770, ) self.assertResourceCalled('HdfsResource', None, @@ -136,7 +136,7 @@ class TestNamenode(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = None, user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertNoMoreResources() @@ -217,7 +217,7 @@ class TestNamenode(RMFTestCase): owner = 'hdfs', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0777, ) self.assertResourceCalled('HdfsResource', '/user/ambari-qa', @@ -233,7 +233,7 @@ class TestNamenode(RMFTestCase): owner = 'ambari-qa', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0770, ) self.assertResourceCalled('HdfsResource', None, @@ -246,7 +246,7 @@ class TestNamenode(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = None, user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertNoMoreResources() @@ -339,7 +339,7 @@ class TestNamenode(RMFTestCase): owner = 'hdfs', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', mode = 0777, only_if = True ) @@ -352,7 +352,7 @@ class TestNamenode(RMFTestCase): owner = 'ambari-qa', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', mode = 0770, only_if = True ) @@ -363,7 +363,7 @@ class TestNamenode(RMFTestCase): hadoop_bin_dir = '/usr/bin', kinit_path_local = '/usr/bin/kinit', user = 'hdfs', - action = ['execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertNoMoreResources() @@ -436,7 +436,7 @@ class TestNamenode(RMFTestCase): owner = 'hdfs', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0777, ) self.assertResourceCalled('HdfsResource', '/user/ambari-qa', @@ -452,7 +452,7 @@ class TestNamenode(RMFTestCase): owner = 'ambari-qa', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0770, ) self.assertResourceCalled('HdfsResource', None, @@ -465,7 +465,7 @@ class TestNamenode(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = None, user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertNoMoreResources() @@ -530,7 +530,7 @@ class TestNamenode(RMFTestCase): owner = 'hdfs', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0777, ) self.assertResourceCalled('HdfsResource', '/user/ambari-qa', @@ -546,7 +546,7 @@ class TestNamenode(RMFTestCase): owner = 'ambari-qa', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0770, ) self.assertResourceCalled('HdfsResource', None, @@ -559,7 +559,7 @@ class TestNamenode(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = None, user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertNoMoreResources() @@ -630,7 +630,7 @@ class TestNamenode(RMFTestCase): owner = 'hdfs', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0777, ) self.assertResourceCalled('HdfsResource', '/user/ambari-qa', @@ -646,7 +646,7 @@ class TestNamenode(RMFTestCase): owner = 'ambari-qa', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0770, ) self.assertResourceCalled('HdfsResource', None, @@ -659,7 +659,7 @@ class TestNamenode(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = 'hdfs', user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertNoMoreResources() @@ -730,7 +730,7 @@ class TestNamenode(RMFTestCase): owner = 'hdfs', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0777, ) self.assertResourceCalled('HdfsResource', '/user/ambari-qa', @@ -746,7 +746,7 @@ class TestNamenode(RMFTestCase): owner = 'ambari-qa', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0770, ) self.assertResourceCalled('HdfsResource', None, @@ -759,7 +759,7 @@ class TestNamenode(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = None, user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertNoMoreResources() @@ -829,7 +829,7 @@ class TestNamenode(RMFTestCase): owner = 'hdfs', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0777, ) self.assertResourceCalled('HdfsResource', '/user/ambari-qa', @@ -845,7 +845,7 @@ class TestNamenode(RMFTestCase): owner = 'ambari-qa', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0770, ) self.assertResourceCalled('HdfsResource', None, @@ -858,7 +858,7 @@ class TestNamenode(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = None, user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertNoMoreResources() @@ -936,7 +936,7 @@ class TestNamenode(RMFTestCase): owner = 'hdfs', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0777, ) self.assertResourceCalled('HdfsResource', '/user/ambari-qa', @@ -952,7 +952,7 @@ class TestNamenode(RMFTestCase): owner = 'ambari-qa', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0770, ) self.assertResourceCalled('HdfsResource', None, @@ -965,7 +965,7 @@ class TestNamenode(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = None, user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_service_check.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_service_check.py index 3084528..b28d991 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_service_check.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_service_check.py @@ -68,7 +68,7 @@ class TestServiceCheck(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = None, user = 'hdfs', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', mode = 0777, @@ -82,7 +82,7 @@ class TestServiceCheck(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = None, user = 'hdfs', - action = ['delete_on_execute'], + action = ['delete_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', type = 'file', ) @@ -96,7 +96,7 @@ class TestServiceCheck(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = None, user = 'hdfs', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', type = 'file', ) @@ -109,7 +109,7 @@ class TestServiceCheck(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = None, user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py index 43d27d7..14e88d6 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py @@ -339,7 +339,7 @@ class TestHiveServer(RMFTestCase): owner = 'hcat', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='missing_principal', default_fs=default_fs_default, + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='missing_principal', default_fs=default_fs_default, mode = 0755, ) self.assertResourceCalled('HdfsResource', '/user/hcat', @@ -351,7 +351,7 @@ class TestHiveServer(RMFTestCase): owner = 'hcat', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='missing_principal', default_fs=default_fs_default, + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='missing_principal', default_fs=default_fs_default, mode = 0755, ) @@ -368,7 +368,7 @@ class TestHiveServer(RMFTestCase): owner = 'hive', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='missing_principal', default_fs=default_fs_default, + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='missing_principal', default_fs=default_fs_default, mode = 0777, ) self.assertResourceCalled('HdfsResource', '/user/hive', @@ -380,7 +380,7 @@ class TestHiveServer(RMFTestCase): owner = 'hive', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='missing_principal', default_fs=default_fs_default, + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='missing_principal', default_fs=default_fs_default, mode = 0755, ) if not no_tmp: @@ -394,7 +394,7 @@ class TestHiveServer(RMFTestCase): group = 'hdfs', hadoop_bin_dir = '/usr/bin', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='missing_principal', default_fs=default_fs_default, + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='missing_principal', default_fs=default_fs_default, mode = 0777, ) self.assertResourceCalled('HdfsResource', None, @@ -403,7 +403,7 @@ class TestHiveServer(RMFTestCase): keytab = UnknownConfigurationMock(), kinit_path_local = '/usr/bin/kinit', user = 'hdfs', - action = ['execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='missing_principal', default_fs=default_fs_default, + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='missing_principal', default_fs=default_fs_default, hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertResourceCalled('Directory', '/etc/hive', @@ -521,7 +521,7 @@ class TestHiveServer(RMFTestCase): owner = 'hcat', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', mode = 0755, ) self.assertResourceCalled('HdfsResource', '/user/hcat', @@ -533,7 +533,7 @@ class TestHiveServer(RMFTestCase): owner = 'hcat', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', mode = 0755, ) @@ -546,7 +546,7 @@ class TestHiveServer(RMFTestCase): owner = 'hive', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', mode = 0777, ) self.assertResourceCalled('HdfsResource', '/user/hive', @@ -558,7 +558,7 @@ class TestHiveServer(RMFTestCase): owner = 'hive', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', mode = 0755, ) self.assertResourceCalled('HdfsResource', '/custompath/tmp/hive', @@ -571,7 +571,7 @@ class TestHiveServer(RMFTestCase): group = 'hdfs', hadoop_bin_dir = '/usr/bin', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', mode = 0777, ) self.assertResourceCalled('HdfsResource', None, @@ -580,7 +580,7 @@ class TestHiveServer(RMFTestCase): keytab = '/etc/security/keytabs/hdfs.headless.keytab', kinit_path_local = '/usr/bin/kinit', user = 'hdfs', - action = ['execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertResourceCalled('Directory', '/etc/hive', @@ -913,7 +913,7 @@ From source with checksum 150f554beae04f76f814f59549dead8b""" keytab = UnknownConfigurationMock(), kinit_path_local = '/usr/bin/kinit', user = 'hdfs', - action = ['execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='missing_principal', default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='missing_principal', default_fs='hdfs://c6401.ambari.apache.org:8020', hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', ) self.assertNoMoreResources() @@ -954,7 +954,7 @@ From source with checksum 150f554beae04f76f814f59549dead8b""" keytab = UnknownConfigurationMock(), kinit_path_local = '/usr/bin/kinit', user = 'hdfs', - action = ['execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='missing_principal', default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='missing_principal', default_fs='hdfs://c6401.ambari.apache.org:8020', hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', ) self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_service_check.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_service_check.py index 00439a8..2d790f8 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_service_check.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_service_check.py @@ -102,7 +102,7 @@ class TestServiceCheck(RMFTestCase): owner = 'ambari-qa', hadoop_conf_dir = '/etc/hadoop/conf', type = 'file', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', ) self.assertResourceCalled('HdfsResource', '/tmp/idtest.ambari-qa.1431110511.43.in', security_enabled = False, @@ -117,7 +117,7 @@ class TestServiceCheck(RMFTestCase): owner = 'ambari-qa', hadoop_conf_dir = '/etc/hadoop/conf', type = 'file', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', ) self.assertResourceCalled('HdfsResource', None, security_enabled = False, @@ -128,7 +128,7 @@ class TestServiceCheck(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = 'missing_principal', user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertResourceCalled('Execute', '/tmp/templetonSmoke.sh c6402.ambari.apache.org ambari-qa 50111 idtest.ambari-qa.1431110511.43.pig no_keytab false /usr/bin/kinit no_principal', @@ -212,7 +212,7 @@ class TestServiceCheck(RMFTestCase): owner = 'ambari-qa', hadoop_conf_dir = '/etc/hadoop/conf', type = 'file', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', ) self.assertResourceCalled('HdfsResource', '/tmp/idtest.ambari-qa.1431110511.43.in', security_enabled = True, @@ -227,7 +227,7 @@ class TestServiceCheck(RMFTestCase): owner = 'ambari-qa', hadoop_conf_dir = '/etc/hadoop/conf', type = 'file', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', ) self.assertResourceCalled('HdfsResource', None, security_enabled = True, @@ -238,7 +238,7 @@ class TestServiceCheck(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = 'hdfs', user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertResourceCalled('Execute', '/tmp/templetonSmoke.sh c6402.ambari.apache.org ambari-qa 50111 idtest.ambari-qa.1431110511.43.pig /etc/security/keytabs/smokeuser.headless.keytab true /usr/bin/kinit [email protected]', http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py index 3bb4b30..1c854b5 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py @@ -72,7 +72,7 @@ class TestOozieServer(RMFTestCase): owner = 'oozie', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', mode = 0775, ) self.assertResourceCalled('HdfsResource', None, @@ -81,7 +81,7 @@ class TestOozieServer(RMFTestCase): keytab = UnknownConfigurationMock(), kinit_path_local = '/usr/bin/kinit', user = 'hdfs', - action = ['execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertResourceCalled('Directory', '/etc/oozie/conf', @@ -284,7 +284,7 @@ class TestOozieServer(RMFTestCase): owner = 'oozie', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0775, ) self.assertResourceCalled('HdfsResource', None, @@ -296,7 +296,7 @@ class TestOozieServer(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = UnknownConfigurationMock(), user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertResourceCalled('Directory', '/etc/oozie/conf', @@ -508,7 +508,7 @@ class TestOozieServer(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = UnknownConfigurationMock(), recursive_chmod = True, - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', mode = 0755, @@ -522,7 +522,7 @@ class TestOozieServer(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = UnknownConfigurationMock(), user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-start.sh', @@ -601,7 +601,7 @@ class TestOozieServer(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = 'hdfs', recursive_chmod = True, - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', mode = 0755, @@ -615,7 +615,7 @@ class TestOozieServer(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = 'hdfs', user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-start.sh', @@ -653,7 +653,7 @@ class TestOozieServer(RMFTestCase): owner = 'oozie', hadoop_bin_dir = '/usr/bin', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', mode = 0775, ) self.assertResourceCalled('HdfsResource', None, @@ -662,7 +662,7 @@ class TestOozieServer(RMFTestCase): keytab = UnknownConfigurationMock(), kinit_path_local = '/usr/bin/kinit', user = 'hdfs', - action = ['execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertResourceCalled('Directory', '/etc/oozie/conf', @@ -835,7 +835,7 @@ class TestOozieServer(RMFTestCase): owner = 'oozie', hadoop_bin_dir = '/usr/bin', type = 'directory', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', mode = 0775, ) self.assertResourceCalled('HdfsResource', None, @@ -845,7 +845,7 @@ class TestOozieServer(RMFTestCase): kinit_path_local = '/usr/bin/kinit', user = 'hdfs', - action = ['execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertResourceCalled('Directory', '/etc/oozie/conf', @@ -1380,7 +1380,7 @@ class TestOozieServer(RMFTestCase): group = 'hadoop', hadoop_conf_dir = '/usr/hdp/2.3.0.0-1234/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0755 ) self.assertResourceCalled('HdfsResource', None, @@ -1392,7 +1392,7 @@ class TestOozieServer(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = UnknownConfigurationMock(), user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/usr/hdp/2.3.0.0-1234/hadoop/conf' ) self.assertResourceCalled('Execute', '/usr/hdp/2.3.0.0-1234/oozie/bin/oozie-setup.sh sharelib create -fs hdfs://c6401.ambari.apache.org:8020', @@ -1451,7 +1451,7 @@ class TestOozieServer(RMFTestCase): group = 'hadoop', hadoop_conf_dir = '/usr/hdp/2.3.0.0-1234/hadoop/conf', type = 'directory', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0755 ) self.assertResourceCalled('HdfsResource', None, @@ -1463,7 +1463,7 @@ class TestOozieServer(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = UnknownConfigurationMock(), user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/usr/hdp/2.3.0.0-1234/hadoop/conf' ) self.assertResourceCalled('Execute', '/usr/hdp/2.3.0.0-1234/oozie/bin/oozie-setup.sh sharelib create -fs hdfs://c6401.ambari.apache.org:8020', http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py index be807b2..ef187bd 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py @@ -88,7 +88,7 @@ class TestServiceCheck(RMFTestCase): keytab = UnknownConfigurationMock(), kinit_path_local = '/usr/bin/kinit', user = 'hdfs', - action = ['delete_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['delete_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', ) @@ -99,7 +99,7 @@ class TestServiceCheck(RMFTestCase): kinit_path_local = '/usr/bin/kinit', source = '//examples', user = 'hdfs', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', owner = 'ambari-qa', @@ -111,7 +111,7 @@ class TestServiceCheck(RMFTestCase): keytab = UnknownConfigurationMock(), kinit_path_local = '/usr/bin/kinit', user = 'hdfs', - action = ['delete_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['delete_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', ) @@ -122,7 +122,7 @@ class TestServiceCheck(RMFTestCase): kinit_path_local = '/usr/bin/kinit', source = '//examples/input-data', user = 'hdfs', - action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', owner = 'ambari-qa', @@ -134,7 +134,7 @@ class TestServiceCheck(RMFTestCase): keytab = UnknownConfigurationMock(), kinit_path_local = '/usr/bin/kinit', user = 'hdfs', - action = ['execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertResourceCalled('Execute', '/tmp/oozieSmoke2.sh suse /var/lib/oozie /etc/oozie/conf /usr/bin http://c6402.ambari.apache.org:11000/oozie / /etc/hadoop/conf /usr/bin ambari-qa False', http://git-wip-us.apache.org/repos/asf/ambari/blob/ad5ecf82/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_service_check.py b/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_service_check.py index 4a87f4b..386e5c9 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_service_check.py +++ b/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_service_check.py @@ -44,7 +44,7 @@ class TestPigServiceCheck(RMFTestCase): owner = 'ambari-qa', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['delete_on_execute'], + action = ['delete_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', ) self.assertResourceCalled('HdfsResource', '/user/ambari-qa/passwd', security_enabled = False, @@ -59,7 +59,7 @@ class TestPigServiceCheck(RMFTestCase): owner = 'ambari-qa', hadoop_conf_dir = '/etc/hadoop/conf', type = 'file', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', ) self.assertResourceCalled('HdfsResource', None, security_enabled = False, @@ -70,7 +70,7 @@ class TestPigServiceCheck(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = UnknownConfigurationMock(), user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', ) @@ -115,7 +115,7 @@ class TestPigServiceCheck(RMFTestCase): owner = 'ambari-qa', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', - action = ['delete_on_execute'], + action = ['delete_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', ) self.assertResourceCalled('HdfsResource', '/user/ambari-qa/passwd', security_enabled = True, @@ -130,7 +130,7 @@ class TestPigServiceCheck(RMFTestCase): owner = 'ambari-qa', hadoop_conf_dir = '/etc/hadoop/conf', type = 'file', - action = ['create_on_execute'], + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', ) self.assertResourceCalled('HdfsResource', None, security_enabled = True, @@ -141,7 +141,7 @@ class TestPigServiceCheck(RMFTestCase): kinit_path_local = '/usr/bin/kinit', principal_name = 'hdfs', user = 'hdfs', - action = ['execute'], + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab [email protected];',
