Repository: ambari Updated Branches: refs/heads/branch-2.2 673edd4dc -> c3e60d3f1
AMBARI-14660. HistoryServer upgrade times out when /app-logs is too large (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c3e60d3f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c3e60d3f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c3e60d3f Branch: refs/heads/branch-2.2 Commit: c3e60d3f136bb151dda21866ca597764773a11fd Parents: 673edd4 Author: Andrew Onishuk <[email protected]> Authored: Thu Jan 14 12:46:26 2016 +0200 Committer: Andrew Onishuk <[email protected]> Committed: Thu Jan 14 12:46:26 2016 +0200 ---------------------------------------------------------------------- .../libraries/providers/hdfs_resource.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c3e60d3f/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py index 59a04ece..116e1ee 100644 --- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py +++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py @@ -217,6 +217,12 @@ class HdfsResourceWebHDFS: Since it's not available on non-hdfs FS and also can be disabled in scope of HDFS. We should still have the other implementations for such a cases. """ + + # if we have more than this count of files to recursively chmod/chown + # webhdfs won't be used, but 'hadoop fs -chmod (or chown) -R ..' As it can really slow. + # (in one second ~17 files can be chmoded) + MAX_FILES_FOR_RECURSIVE_ACTION_VIA_WEBHDFS = 1000 + def action_execute(self, main_resource): pass @@ -343,6 +349,12 @@ class HdfsResourceWebHDFS: if self.main_resource.resource.recursive_chown: self._fill_directories_list(self.main_resource.resource.target, results) + + # if we don't do this, we can end up waiting real long, having a big result list. + if len(results) > HdfsResourceWebHDFS.MAX_FILES_FOR_RECURSIVE_ACTION_VIA_WEBHDFS: + shell.checked_call(["hadoop", "fs", "-chown", "-R", format("{owner}:{group}"), self.main_resource.resource.target], user=self.main_resource.resource.user) + results = [] + if self.main_resource.resource.change_permissions_for_parents: self._fill_in_parent_directories(self.main_resource.resource.target, results) @@ -360,6 +372,12 @@ class HdfsResourceWebHDFS: if self.main_resource.resource.recursive_chmod: self._fill_directories_list(self.main_resource.resource.target, results) + + # if we don't do this, we can end up waiting real long, having a big result list. + if len(results) > HdfsResourceWebHDFS.MAX_FILES_FOR_RECURSIVE_ACTION_VIA_WEBHDFS: + shell.checked_call(["hadoop", "fs", "-chmod", "-R", self.mode, self.main_resource.resource.target], user=self.main_resource.resource.user) + results = [] + if self.main_resource.resource.change_permissions_for_parents: self._fill_in_parent_directories(self.main_resource.resource.target, results)
