AMBARI-12900. Ambari overrides hot-fixed tez tarball on HDFS everytime HiveServer is restarted (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7184571c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7184571c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7184571c Branch: refs/heads/branch-2.1 Commit: 7184571c3dff60c85fd7983c27dc638f0eb80c84 Parents: 6930727 Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Thu Aug 27 15:45:59 2015 +0300 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Thu Aug 27 15:45:59 2015 +0300 ---------------------------------------------------------------------- .../resource_management/libraries/functions/copy_tarball.py | 5 +++-- .../resource_management/libraries/providers/hdfs_resource.py | 3 +++ .../resource_management/libraries/resources/hdfs_resource.py | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7184571c/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py index ad4aadc..792f019 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py @@ -96,7 +96,7 @@ def _get_single_version_from_hdp_select(): return hdp_version def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=None, custom_dest_file=None, force_execute=False, - use_ru_version_during_ru=True): + use_ru_version_during_ru=True, replace_existing_files=False): """ :param name: Tarball name, e.g., tez, hive, pig, sqoop. :param user_group: Group to own the directory. @@ -190,7 +190,8 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=Non source=source_file, group=user_group, owner=owner, - mode=0444 + mode=0444, + replace_existing_files=replace_existing_files, ) Logger.info("Will attempt to copy {0} tarball from {1} to DFS at {2}.".format(name, source_file, dest_file)) http://git-wip-us.apache.org/repos/asf/ambari/blob/7184571c/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py index cb6fb21..1d1aeba 100644 --- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py +++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py @@ -301,6 +301,9 @@ class HdfsResourceWebHDFS: if local_file_size == length: Logger.info(format("DFS file {target} is identical to {source}, skipping the copying")) return + elif not self.main_resource.resource.replace_existing_files: + Logger.info(format("Not replacing existing DFS file {target} which is different from {source}, due to replace_existing_files=False")) + return else: Logger.info(format("File {target} already exists in DFS, skipping the creation")) return http://git-wip-us.apache.org/repos/asf/ambari/blob/7184571c/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py index 3a991b0..69aeab0 100644 --- a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py +++ b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py @@ -63,6 +63,10 @@ class HdfsResource(Resource): recursive_chown = BooleanArgument(default=False) recursive_chmod = BooleanArgument(default=False) change_permissions_for_parents = BooleanArgument(default=False) + """ + If a file present in HDFS is different from source, should it be replaced? + """ + replace_existing_files = BooleanArgument(default=True) security_enabled = BooleanArgument(default=False) principal_name = ResourceArgument()