AMBARI-14396. Add support for recursive chown in resource management library. (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/92c054b1 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/92c054b1 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/92c054b1 Branch: refs/heads/trunk Commit: 92c054b1d6fcc0dc436ae4ff6b1cd9b8da68b459 Parents: 314bbc6 Author: Andrew Onishuk <[email protected]> Authored: Wed Dec 16 13:10:29 2015 +0200 Committer: Andrew Onishuk <[email protected]> Committed: Wed Dec 16 13:10:29 2015 +0200 ---------------------------------------------------------------------- .../TestDirectoryResource.py | 2 +- .../core/providers/system.py | 32 ++++++- .../core/resources/system.py | 79 ++++++++++++++++- .../python/resource_management/core/sudo.py | 35 +++++++- .../libraries/functions/conf_select.py | 2 +- .../functions/setup_ranger_plugin_xml.py | 2 +- .../libraries/script/script.py | 2 +- .../2.0.7/services/HIVE/package/scripts/hcat.py | 2 +- .../2.0.7/services/HIVE/package/scripts/hive.py | 4 +- .../package/scripts/accumulo_configuration.py | 8 +- .../package/scripts/accumulo_service.py | 12 ++- .../AMBARI_METRICS/0.1.0/package/scripts/ams.py | 38 ++++---- .../0.1.0/package/scripts/hbase.py | 37 +++----- .../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 10 +-- .../FALCON/0.5.0.2.1/package/scripts/falcon.py | 26 +++--- .../FLUME/1.4.0.2.0/package/scripts/flume.py | 2 +- .../GANGLIA/3.5.0/package/scripts/ganglia.py | 2 +- .../3.5.0/package/scripts/ganglia_monitor.py | 2 +- .../3.5.0/package/scripts/ganglia_server.py | 13 ++- .../HAWQ/2.0.0/package/scripts/common.py | 4 +- .../HAWQ/2.0.0/package/scripts/utils.py | 2 +- .../HBASE/0.96.0.2.0/package/scripts/hbase.py | 12 +-- .../HDFS/2.1.0.2.0/package/scripts/hdfs.py | 8 +- .../2.1.0.2.0/package/scripts/hdfs_datanode.py | 6 +- .../2.1.0.2.0/package/scripts/hdfs_namenode.py | 10 +-- .../2.1.0.2.0/package/scripts/hdfs_snamenode.py | 2 +- .../2.1.0.2.0/package/scripts/journalnode.py | 2 +- .../HDFS/2.1.0.2.0/package/scripts/utils.py | 4 +- .../HIVE/0.12.0.2.0/package/scripts/hcat.py | 6 +- .../HIVE/0.12.0.2.0/package/scripts/hive.py | 8 +- .../package/scripts/hive_metastore.py | 2 +- .../HIVE/0.12.0.2.0/package/scripts/webhcat.py | 8 +- .../KAFKA/0.8.1.2.2/package/scripts/kafka.py | 48 +++------- .../package/scripts/kerberos_common.py | 6 +- .../package/scripts/kerberos_server.py | 4 +- .../KNOX/0.5.0.2.2/package/scripts/knox.py | 40 +++------ .../MAHOUT/1.0.0.2.3/package/scripts/mahout.py | 2 +- .../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 26 ++++-- .../PIG/0.12.0.2.0/package/scripts/pig.py | 2 +- .../PXF/3.0.0/package/scripts/pxf.py | 2 +- .../0.4.0/package/scripts/ranger_service.py | 6 +- .../0.4.0/package/scripts/setup_ranger_xml.py | 12 ++- .../RANGER_KMS/0.5.0.2.3/package/scripts/kms.py | 14 +-- .../SLIDER/0.60.0.2.2/package/scripts/slider.py | 4 +- .../1.2.0.2.2/package/scripts/setup_spark.py | 2 +- .../SQOOP/1.4.4.2.0/package/scripts/sqoop.py | 2 +- .../STORM/0.9.1.2.1/package/scripts/storm.py | 8 +- .../0.9.1.2.1/package/scripts/storm_upgrade.py | 4 +- .../TEZ/0.4.0.2.1/package/scripts/tez.py | 2 +- .../YARN/2.1.0.2.0/package/scripts/yarn.py | 39 +++++---- .../3.4.5.2.0/package/scripts/zookeeper.py | 10 +-- .../custom_actions/scripts/check_host.py | 2 +- .../before-ANY/scripts/shared_initialization.py | 2 +- .../scripts/shared_initialization.py | 6 +- .../0.8/services/FLUME/package/scripts/flume.py | 2 +- .../services/GANGLIA/package/scripts/ganglia.py | 2 +- .../GANGLIA/package/scripts/ganglia_monitor.py | 2 +- .../GANGLIA/package/scripts/ganglia_server.py | 6 +- .../0.8/services/HBASE/package/scripts/hbase.py | 10 +-- .../0.8/services/HDFS/package/scripts/hdfs.py | 2 +- .../HDFS/package/scripts/hdfs_datanode.py | 4 +- .../HDFS/package/scripts/hdfs_namenode.py | 4 +- .../HDFS/package/scripts/hdfs_snamenode.py | 2 +- .../HDFS/package/scripts/journalnode.py | 2 +- .../0.8/services/HDFS/package/scripts/utils.py | 4 +- .../0.8/services/HIVE/package/scripts/hcat.py | 6 +- .../0.8/services/HIVE/package/scripts/hive.py | 4 +- .../services/HIVE/package/scripts/webhcat.py | 6 +- .../0.8/services/OOZIE/package/scripts/oozie.py | 4 +- .../0.8/services/PIG/package/scripts/pig.py | 2 +- .../package/scripts/mapred_service_check.py | 2 +- .../0.8/services/YARN/package/scripts/yarn.py | 10 +-- .../ZOOKEEPER/package/scripts/zookeeper.py | 8 +- .../before-ANY/scripts/shared_initialization.py | 18 ++-- .../before-START/scripts/rack_awareness.py | 4 +- .../scripts/shared_initialization.py | 8 +- .../services/FALCON/package/scripts/falcon.py | 4 +- .../services/STORM/package/scripts/storm.py | 2 +- .../services/TEZ/package/scripts/tez.py | 2 +- .../services/YARN/package/scripts/yarn.py | 10 +-- .../AMBARI_METRICS/test_metrics_collector.py | 43 ++++----- .../python/stacks/2.0.6/FLUME/test_flume.py | 8 +- .../2.0.6/GANGLIA/test_ganglia_monitor.py | 4 +- .../stacks/2.0.6/GANGLIA/test_ganglia_server.py | 15 ++-- .../stacks/2.0.6/HBASE/test_hbase_client.py | 16 ++-- .../stacks/2.0.6/HBASE/test_hbase_master.py | 36 ++++---- .../2.0.6/HBASE/test_hbase_regionserver.py | 48 +++++----- .../2.0.6/HBASE/test_phoenix_queryserver.py | 32 +++---- .../python/stacks/2.0.6/HDFS/test_datanode.py | 40 ++++----- .../stacks/2.0.6/HDFS/test_hdfs_client.py | 2 +- .../stacks/2.0.6/HDFS/test_journalnode.py | 24 ++--- .../python/stacks/2.0.6/HDFS/test_namenode.py | 60 ++++++------- .../python/stacks/2.0.6/HDFS/test_nfsgateway.py | 16 ++-- .../python/stacks/2.0.6/HDFS/test_snamenode.py | 26 +++--- .../test/python/stacks/2.0.6/HDFS/test_zkfc.py | 40 ++++----- .../stacks/2.0.6/HIVE/test_hcat_client.py | 12 +-- .../stacks/2.0.6/HIVE/test_hive_client.py | 8 +- .../stacks/2.0.6/HIVE/test_hive_metastore.py | 30 +++---- .../stacks/2.0.6/HIVE/test_hive_server.py | 20 ++--- .../stacks/2.0.6/HIVE/test_webhcat_server.py | 24 ++--- .../stacks/2.0.6/OOZIE/test_oozie_client.py | 6 +- .../stacks/2.0.6/OOZIE/test_oozie_server.py | 92 ++++++++++---------- .../python/stacks/2.0.6/PIG/test_pig_client.py | 6 +- .../python/stacks/2.0.6/SQOOP/test_sqoop.py | 4 +- .../stacks/2.0.6/YARN/test_historyserver.py | 48 +++++----- .../stacks/2.0.6/YARN/test_mapreduce2_client.py | 36 ++++---- .../stacks/2.0.6/YARN/test_nodemanager.py | 82 +++++++++-------- .../stacks/2.0.6/YARN/test_resourcemanager.py | 36 ++++---- .../stacks/2.0.6/YARN/test_yarn_client.py | 54 ++++++------ .../2.0.6/ZOOKEEPER/test_zookeeper_client.py | 16 ++-- .../2.0.6/ZOOKEEPER/test_zookeeper_server.py | 16 ++-- .../2.0.6/hooks/before-ANY/test_before_any.py | 17 ++-- .../hooks/before-START/test_before_start.py | 24 ++--- .../stacks/2.1/FALCON/test_falcon_client.py | 12 +-- .../stacks/2.1/FALCON/test_falcon_server.py | 48 +++++----- .../stacks/2.1/HIVE/test_hive_metastore.py | 30 +++---- .../python/stacks/2.1/STORM/test_storm_base.py | 16 ++-- .../python/stacks/2.1/TEZ/test_tez_client.py | 2 +- .../stacks/2.1/YARN/test_apptimelineserver.py | 20 ++--- .../stacks/2.2/KAFKA/test_kafka_broker.py | 62 +++++-------- .../stacks/2.2/KERBEROS/test_kerberos_client.py | 14 +-- .../stacks/2.2/KERBEROS/test_kerberos_server.py | 24 ++--- .../python/stacks/2.2/KNOX/test_knox_gateway.py | 55 +++++------- .../stacks/2.2/SLIDER/test_slider_client.py | 4 +- .../stacks/2.2/SPARK/test_job_history_server.py | 8 +- .../stacks/2.2/SPARK/test_spark_client.py | 8 +- .../stacks/2.3/ATLAS/test_metadata_server.py | 10 +-- .../stacks/2.3/MAHOUT/test_mahout_client.py | 2 +- .../src/test/python/stacks/2.3/PXF/test_pxf.py | 2 +- .../2.3/SPARK/test_spark_thrift_server.py | 4 +- .../python/stacks/2.3/STORM/test_storm_base.py | 16 ++-- .../test/python/stacks/2.3/YARN/test_ats_1_5.py | 20 ++--- .../HIVE/0.11.0.2.0.5.0/package/scripts/hcat.py | 2 +- .../HIVE/0.11.0.2.0.5.0/package/scripts/hive.py | 4 +- .../dummy_stack/HIVE/package/scripts/hcat.py | 2 +- .../dummy_stack/HIVE/package/scripts/hive.py | 4 +- .../2.0.7/services/HIVE/package/scripts/hcat.py | 2 +- .../2.0.7/services/HIVE/package/scripts/hive.py | 4 +- 138 files changed, 1064 insertions(+), 1009 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-agent/src/test/python/resource_management/TestDirectoryResource.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestDirectoryResource.py b/ambari-agent/src/test/python/resource_management/TestDirectoryResource.py index 011be6d..867fbc8 100644 --- a/ambari-agent/src/test/python/resource_management/TestDirectoryResource.py +++ b/ambari-agent/src/test/python/resource_management/TestDirectoryResource.py @@ -63,7 +63,7 @@ class TestDirectoryResource(TestCase): mode=0777, owner="hdfs", group="hadoop", - recursive=True + create_parents = True ) os_makedirs_mock.assert_called_with('/a/b/c/d', 0777) http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-common/src/main/python/resource_management/core/providers/system.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/system.py b/ambari-common/src/main/python/resource_management/core/providers/system.py index 4feacee..5066fa8 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/system.py +++ b/ambari-common/src/main/python/resource_management/core/providers/system.py @@ -34,7 +34,12 @@ from resource_management.core import ExecuteTimeoutException from resource_management.core.providers import Provider from resource_management.core.logger import Logger -def _ensure_metadata(path, user, group, mode=None, cd_access=None): +def assert_not_safemode_folder(path, safemode_folders): + if os.path.abspath(path) in safemode_folders: + raise Fail(("Not performing recursive operation ('recursive_ownership' or 'recursive_mode_flags') on folder '%s'" + + " as this can damage the system. Please pass changed safemode_folders parameter to Directory resource if you really intend to do this.") % (path)) + +def _ensure_metadata(path, user, group, mode=None, cd_access=None, recursive_ownership=False, recursive_mode_flags=None, recursion_follow_links=False, safemode_folders=[]): user_entity = group_entity = None if user or group: @@ -62,7 +67,26 @@ def _ensure_metadata(path, user, group, mode=None, cd_access=None): Logger.info( "Changing group for %s from %d to %s" % (path, stat.st_gid, group)) + if recursive_ownership: + assert_not_safemode_folder(path, safemode_folders) + sudo.chown_recursive(path, _user_entity, _group_entity, recursion_follow_links) + sudo.chown(path, user_entity, group_entity) + + if recursive_mode_flags: + if not isinstance(recursive_mode_flags, dict): + raise Fail("'recursion_follow_links' value should be a dictionary with 'f' and(or) 'd' key (for file and directory permission flags)") + + regexp_to_match = "^({0},)*({0})$".format("[ugoa]+[+=-][rwx]+" ) + for key, flags in recursive_mode_flags.iteritems(): + if key != 'd' and key != 'f': + raise Fail("'recursive_mode_flags' with value '%s' has unknown key '%s', only keys 'f' and 'd' are valid" % (str(recursive_mode_flags), str(key))) + + if not re.match(regexp_to_match, flags): + raise Fail("'recursive_mode_flags' found '%s', but should value format have the following format: [ugoa...][[+-=][perms...]...]." % (str(flags))) + + assert_not_safemode_folder(path, safemode_folders) + sudo.chmod_recursive(path, recursive_mode_flags, recursion_follow_links) if mode: stat = sudo.stat(path) @@ -156,7 +180,7 @@ class DirectoryProvider(Provider): if path != self.resource.path: Logger.info("Following the link {0} to {1} to create the directory".format(self.resource.path, path)) - if self.resource.recursive: + if self.resource.create_parents: sudo.makedirs(path, self.resource.mode or 0755) else: dirname = os.path.dirname(path) @@ -169,7 +193,9 @@ class DirectoryProvider(Provider): raise Fail("Applying %s failed, file %s already exists" % (self.resource, path)) _ensure_metadata(path, self.resource.owner, self.resource.group, - mode=self.resource.mode, cd_access=self.resource.cd_access) + mode=self.resource.mode, cd_access=self.resource.cd_access, + recursive_ownership=self.resource.recursive_ownership, recursive_mode_flags=self.resource.recursive_mode_flags, + recursion_follow_links=self.resource.recursion_follow_links, safemode_folders=self.resource.safemode_folders) def action_delete(self): path = self.resource.path http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-common/src/main/python/resource_management/core/resources/system.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/resources/system.py b/ambari-common/src/main/python/resource_management/core/resources/system.py index 558bb2b..6860735 100644 --- a/ambari-common/src/main/python/resource_management/core/resources/system.py +++ b/ambari-common/src/main/python/resource_management/core/resources/system.py @@ -60,10 +60,11 @@ class Directory(Resource): group = ResourceArgument() follow = BooleanArgument(default=True) # follow links? """ - this works for 'create', 'delete' is anyway recursive - recursive means only "mkdir -p", it does NOT perform recursive chown/chmod + Example: + Directory('/a/b/c/d', create_parents=False) # will fail unless /a/b/c exists + Directory('/a/b/c/d', create_parents=True) # will succeed if /a/b/c doesn't exists """ - recursive = BooleanArgument(default=False) + create_parents = BooleanArgument(default=False) """ Grants x-bit for all the folders up-to the directory @@ -75,6 +76,78 @@ class Directory(Resource): The letters can be combined together. """ cd_access = ResourceArgument() + """ + If True sets the user and group mentioned in arguments, for all the contents of folder and its subfolder. + + CAUNTION: THIS IS NO RECOMENDED TO USE THIS, please treat such a usages as last resort hacks. + The problem with using recursive permissions setting is that the system can be damaged badly by doing this, + imagine recursively setting for '/' or '/etc' (it can break the system) or other important for user folder. + This is partially fixed by 'safemode_folders' feature, but still is risky. + + See also: safemode_folders, recursion_follow_links + """ + recursive_ownership = BooleanArgument(default=False) + + """ + A dictionary, which gives the mode flags which should be set for files in key 'f', and for + directories in key 'd'. + + The format of those flags should be symbolic format used in chmod. From chmod man page: + the format of a symbolic mode is [ugoa...][[+-=][perms...]...], + where perms is either zero or more letters from the set rwxXst, or a single letter from the set ugo. + Multiple symbolic modes can be given, separated by commas. + + u - user who is owner + g - user from group + o - other users + a - all + + Example: + recursive_mode_flags = { + 'd': 'u+rwx,g+rx' # for subfolders, enforce 'rwx' (read,write,execute) for owner of the file, and 'rx' (read,execute) for group, don't change 'others' permissions + 'f': 'u+rw,go+r' # for files in the directory, enforce 'rw' (read, write) for owner of the file, and 'r' (read) for group and others. + } + + recursive_mode_flags = { + 'd': 'a+rwx' # for subfolders, enforce rwxrwxrwx (777) permisons. + 'f': 'a+rw' # for files in the directory, enforce adding 'rw' (read,write) to original permissions. If file had 'x' bit before it will stay there. + } + + CAUNTION: THIS IS NO RECOMENDED TO USE THIS, please treat such a usages as last resort hacks. + The problem with using recursive permissions setting is that the system can be damaged badly by doing this, + imagine recursively setting permissions for '/' or '/etc' (it can break the system) or other important for user folder. + This is partially fixed by 'safemode_folders' feature, but still is risky. + + See also: safemode_folders, recursion_follow_links + """ + recursive_mode_flags = ResourceArgument(default=None) + + """ + This is the list folder which are not allowed to be recursively chmod-ed or chown-ed. (recursive_ownership and recursive_mode_flags). + Fail exception will appear if tried. + + Example of a dangerous code, which will not be succeed: + Directory("/", + owner="my_user", + recursive_ownership=True + ) + + This aims to the resolve the problem of mistakenly doing recursive actions for system necessary folders. + which results in damaging the operating system. + """ + safemode_folders = ForcedListArgument(default=["/", "/bin", "/sbin", "/etc", "/dev", + "/proc", "/var", "/usr", "/home", "/boot", "/lib", "/opt", + "/mnt", "/media", "/srv", "/root", "/sys" ]) + + """ + If True while recursive chown/chmod is done (recursive_ownership or recursive_mode_flags), + symlinks will be followed, duing recursion walking, also + this will also make chmod/chown to set permissions for symlink targets, not for symlink itself. + + Note: if recursion_follow_links=False chmod will not set permissions nor on symlink neither on targets. + As according to chmod man: 'This is not a problem since the permissions of symbolic links are never used'. + """ + recursion_follow_links = BooleanArgument(default=False) actions = Resource.actions + ["create", "delete"] http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-common/src/main/python/resource_management/core/sudo.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/sudo.py b/ambari-common/src/main/python/resource_management/core/sudo.py index 46fa9d0..bb28d9f 100644 --- a/ambari-common/src/main/python/resource_management/core/sudo.py +++ b/ambari-common/src/main/python/resource_management/core/sudo.py @@ -36,6 +36,21 @@ if os.geteuid() == 0: gid = group.gr_gid if group else -1 if uid != -1 or gid != -1: return os.chown(path, uid, gid) + + def chown_recursive(path, owner, group, follow_links=False): + uid = owner.pw_uid if owner else -1 + gid = group.gr_gid if group else -1 + + if uid == -1 and gid == -1: + return + + for root, dirs, files in os.walk(path, followlinks=follow_links): + for name in files + dirs: + if follow_links: + os.chown(os.path.join(root, name), uid, gid) + else: + os.lchown(os.path.join(root, name), uid, gid) + def chmod(path, mode): return os.chmod(path, mode) @@ -112,7 +127,6 @@ if os.geteuid() == 0: else: - # os.chown replacement def chown(path, owner, group): owner = owner.pw_name if owner else "" @@ -120,6 +134,15 @@ else: if owner or group: shell.checked_call(["chown", owner+":"+group, path], sudo=True) + def chown_recursive(path, owner, group, follow_links=False): + owner = owner.pw_name if owner else "" + group = group.gr_name if group else "" + if owner or group: + flags = ["-R"] + if follow_links: + flags.append("-L") + shell.checked_call(["chown"] + flags + [owner+":"+group, path], sudo=True) + # os.chmod replacement def chmod(path, mode): shell.checked_call(["chmod", oct(mode), path], sudo=True) @@ -226,4 +249,12 @@ else: # shutil.copy replacement def copy(src, dst): - shell.checked_call(["sudo", "cp", "-r", src, dst], sudo=True) \ No newline at end of file + shell.checked_call(["sudo", "cp", "-r", src, dst], sudo=True) + +def chmod_recursive(path, recursive_mode_flags, recursion_follow_links): + find_flags = [] + if recursion_follow_links: + find_flags.append('-L') + + for key, flags in recursive_mode_flags.iteritems(): + shell.checked_call(["find"] + find_flags + [path, "-type", key, "-exec" , "chmod", flags ,"{}" ,";"]) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py index a3f64bf..2bc01ee 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py @@ -228,7 +228,7 @@ def create(stack_name, package, version, dry_run = False): Directory(d, mode=0755, cd_access='a', - recursive=True) + create_parents=True) return dirs http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py index 29ffe0d..d6f6deb 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py @@ -90,7 +90,7 @@ def setup_ranger_plugin(component_select_name, service_name, owner = component_user, group = component_group, mode=0775, - recursive = True, + create_parents = True, cd_access = 'a' ) http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 09ca118..c954a15 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -610,7 +610,7 @@ class Script(object): env_configs_list = config['commandParams']['env_configs_list'] properties_configs_list = config['commandParams']['properties_configs_list'] - Directory(self.get_tmp_dir(), recursive=True) + Directory(self.get_tmp_dir(), create_parents = True) conf_tmp_dir = tempfile.mkdtemp(dir=self.get_tmp_dir()) output_filename = os.path.join(self.get_tmp_dir(), config['commandParams']['output_file']) http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-funtest/src/test/resources/stacks/HDP/2.0.7/services/HIVE/package/scripts/hcat.py ---------------------------------------------------------------------- diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.0.7/services/HIVE/package/scripts/hcat.py b/ambari-funtest/src/test/resources/stacks/HDP/2.0.7/services/HIVE/package/scripts/hcat.py index 2993d3a..29de1c0 100644 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.0.7/services/HIVE/package/scripts/hcat.py +++ b/ambari-funtest/src/test/resources/stacks/HDP/2.0.7/services/HIVE/package/scripts/hcat.py @@ -32,7 +32,7 @@ def hcat(): Directory(params.hcat_pid_dir, owner=params.webhcat_user, - recursive=True + create_parents = True ) hcat_TemplateConfig('hcat-env.sh') http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-funtest/src/test/resources/stacks/HDP/2.0.7/services/HIVE/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.0.7/services/HIVE/package/scripts/hive.py b/ambari-funtest/src/test/resources/stacks/HDP/2.0.7/services/HIVE/package/scripts/hive.py index 63e7fa3..f813755 100644 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.0.7/services/HIVE/package/scripts/hive.py +++ b/ambari-funtest/src/test/resources/stacks/HDP/2.0.7/services/HIVE/package/scripts/hive.py @@ -36,7 +36,7 @@ def hive(name=None): Directory(hive_config_dir, owner=params.hive_user, group=params.user_group, - recursive=True + create_parents = True ) XmlConfig("hive-site.xml", @@ -87,7 +87,7 @@ def crt_directory(name): import params Directory(name, - recursive=True, + create_parents = True, owner=params.hive_user, group=params.user_group, mode=0755) http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_configuration.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_configuration.py b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_configuration.py index acc340f..09a3d90 100644 --- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_configuration.py +++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_configuration.py @@ -29,7 +29,7 @@ def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or ' mode=0755, owner = params.accumulo_user, group = params.user_group, - recursive = True + create_parents = True ) if name == 'client': @@ -65,7 +65,7 @@ def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or ' mode=0700, owner = params.accumulo_user, group = params.user_group, - recursive = True + create_parents = True ) # create a site file for server processes configs = {} @@ -85,7 +85,7 @@ def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or ' Directory( params.pid_dir, owner = params.accumulo_user, group = params.user_group, - recursive = True, + create_parents = True, cd_access = "a", mode = 0755, ) @@ -94,7 +94,7 @@ def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or ' Directory (params.log_dir, owner = params.accumulo_user, group = params.user_group, - recursive = True, + create_parents = True, cd_access = "a", mode = 0755, ) http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_service.py b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_service.py index a4e9c54..0d9687b 100644 --- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_service.py +++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_service.py @@ -20,6 +20,7 @@ limitations under the License. from resource_management import * import time +import os def accumulo_service( name, action = 'start'): # 'start' or 'stop' or 'status' @@ -31,10 +32,13 @@ def accumulo_service( name, pid_exists = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1") if action == 'start': - Execute(as_sudo(['chown','-R',params.accumulo_user+":"+params.user_group, - format("$(getent passwd {accumulo_user} | cut -d: -f6)")], - auto_escape=False), - ignore_failures=True) + Directory(os.path.expanduser("~"), + user = params.accumulo_user, + group = params.user_group, + recursive_ownership = True, + ignore_failures=True + ) + if name != 'tserver': Execute(format("{daemon_script} org.apache.accumulo.master.state.SetGoalState NORMAL"), not_if=as_user(pid_exists, params.accumulo_user), http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py index 8fdf4ba..94b9fc9 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py @@ -34,12 +34,12 @@ def ams(name=None): Directory(params.ams_collector_conf_dir, owner=params.ams_user, - recursive=True + create_parents = True ) Directory(params.ams_checkpoint_dir, owner=params.ams_user, - recursive=True + create_parents = True ) XmlConfig("ams-site.xml", @@ -155,7 +155,7 @@ def ams(name=None): Directory(params.ams_monitor_conf_dir, owner=params.ams_user, - recursive=True + create_parents = True ) TemplateConfig( @@ -184,24 +184,18 @@ def ams(name=None): Directory(params.ams_collector_conf_dir, owner=params.ams_user, group=params.user_group, - recursive=True + create_parents = True, + recursive_ownership = True, ) - - Execute(('chown', '-R', params.ams_user, params.ams_collector_conf_dir), - sudo=True - ) - + Directory(params.ams_checkpoint_dir, owner=params.ams_user, group=params.user_group, cd_access="a", - recursive=True + create_parents = True, + recursive_ownership = True ) - Execute(('chown', '-R', params.ams_user, params.ams_checkpoint_dir), - sudo=True - ) - XmlConfig("ams-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=params.config['configurations']['ams-site'], @@ -249,7 +243,7 @@ def ams(name=None): owner=params.ams_user, group=params.user_group, cd_access="a", - recursive=True, + create_parents = True, mode=0755, ) @@ -257,7 +251,7 @@ def ams(name=None): owner=params.ams_user, group=params.user_group, cd_access="a", - recursive=True, + create_parents = True, mode=0755, ) @@ -269,7 +263,7 @@ def ams(name=None): # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, - recursive=True, + create_parents = True, owner='root', group='root' ) @@ -289,7 +283,7 @@ def ams(name=None): mode = 0755, group=params.user_group, cd_access="a", - recursive=True + create_parents = True ) pass @@ -337,28 +331,28 @@ def ams(name=None): Directory(params.ams_monitor_conf_dir, owner=params.ams_user, group=params.user_group, - recursive=True + create_parents = True ) Directory(params.ams_monitor_log_dir, owner=params.ams_user, group=params.user_group, mode=0755, - recursive=True + create_parents = True ) Directory(params.ams_monitor_pid_dir, owner=params.ams_user, group=params.user_group, mode=0755, - recursive=True + create_parents = True ) Directory(format("{ams_monitor_dir}/psutil/build"), owner=params.ams_user, group=params.user_group, cd_access="a", - recursive=True) + create_parents = True) Execute(format("{sudo} chown -R {ams_user}:{user_group} {ams_monitor_dir}") ) http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py index 2ddb8fe..556380e 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py @@ -27,16 +27,16 @@ def hbase(name=None, action = None): import params Directory(params.hbase_conf_dir, owner = params.hadoop_user, - recursive = True + create_parents = True ) Directory(params.hbase_tmp_dir, - recursive = True, + create_parents = True, owner = params.hadoop_user ) Directory (os.path.join(params.local_dir, "jars"), owner = params.hadoop_user, - recursive = True + create_parents = True ) XmlConfig("hbase-site.xml", @@ -73,7 +73,7 @@ def hbase(name=None, action = None): if name != "client": Directory (params.hbase_log_dir, owner = params.hadoop_user, - recursive = True + create_parents = True ) if (params.hbase_log4j_props != None): @@ -94,29 +94,23 @@ def hbase(name=None # 'master' or 'regionserver' or 'client' Directory(params.hbase_conf_dir, owner = params.hbase_user, group = params.user_group, - recursive = True + create_parents = True, + recursive_ownership = True, ) - Execute(('chown', '-R', params.hbase_user, params.hbase_conf_dir), - sudo=True - ) - Directory (params.hbase_tmp_dir, owner = params.hbase_user, cd_access="a", - recursive = True + create_parents = True, + recursive_ownership = True, ) - Execute(('chown', '-R', params.hbase_user, params.hbase_tmp_dir), - sudo=True - ) - Directory (os.path.join(params.local_dir, "jars"), owner = params.hbase_user, group = params.user_group, cd_access="a", mode=0775, - recursive = True + create_parents = True ) merged_ams_hbase_site = {} @@ -139,7 +133,7 @@ def hbase(name=None # 'master' or 'regionserver' or 'client' mode = 0755, group=params.user_group, cd_access="a", - recursive=True + create_parents = True ) pass @@ -184,14 +178,14 @@ def hbase(name=None # 'master' or 'regionserver' or 'client' if name != "client": Directory( params.hbase_pid_dir, owner = params.hbase_user, - recursive = True, + create_parents = True, cd_access = "a", mode = 0755, ) Directory (params.hbase_log_dir, owner = params.hbase_user, - recursive = True, + create_parents = True, cd_access = "a", mode = 0755, ) @@ -237,13 +231,10 @@ def hbase(name=None # 'master' or 'regionserver' or 'client' Directory(local_root_dir, owner = params.hbase_user, cd_access="a", - recursive = True + create_parents = True, + recursive_ownership = True ) - Execute(('chown', '-R', params.hbase_user, local_root_dir), - sudo=True - ) - File(format("{params.hbase_pid_dir}/distributed_mode"), action="delete", owner=params.hbase_user) if params.hbase_log4j_props is not None: http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py index 2dbb273..8c17214 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py @@ -30,7 +30,7 @@ def metadata(): cd_access='a', owner=params.metadata_user, group=params.user_group, - recursive=True + create_parents = True ) Directory(params.conf_dir, @@ -38,7 +38,7 @@ def metadata(): cd_access='a', owner=params.metadata_user, group=params.user_group, - recursive=True + create_parents = True ) Directory(params.log_dir, @@ -46,7 +46,7 @@ def metadata(): cd_access='a', owner=params.metadata_user, group=params.user_group, - recursive=True + create_parents = True ) Directory(params.data_dir, @@ -54,7 +54,7 @@ def metadata(): cd_access='a', owner=params.metadata_user, group=params.user_group, - recursive=True + create_parents = True ) Directory(params.expanded_war_dir, @@ -62,7 +62,7 @@ def metadata(): cd_access='a', owner=params.metadata_user, group=params.user_group, - recursive=True + create_parents = True ) File(format("{expanded_war_dir}/atlas.war"), http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py index 6c8d5ff..81ea39e 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py @@ -41,33 +41,33 @@ def falcon(type, action = None, upgrade_type=None): if action == 'config': Directory(params.falcon_pid_dir, owner = params.falcon_user, - recursive = True, + create_parents = True, mode = 0755, cd_access = "a", ) Directory(params.falcon_log_dir, owner = params.falcon_user, - recursive = True, + create_parents = True, mode = 0755, cd_access = "a", ) Directory(params.falcon_webapp_dir, owner = params.falcon_user, - recursive = True) + create_parents = True) Directory(params.falcon_home, owner = params.falcon_user, - recursive = True) + create_parents = True) Directory(params.etc_prefix_dir, mode = 0755, - recursive = True) + create_parents = True) Directory(params.falcon_conf_dir, owner = params.falcon_user, - recursive = True) + create_parents = True) File(params.falcon_conf_dir + '/falcon-env.sh', content = InlineTemplate(params.falcon_env_sh_template), @@ -95,7 +95,7 @@ def falcon(type, action = None, upgrade_type=None): owner = params.falcon_user, group = params.user_group, mode = 0775, - recursive = True, + create_parents = True, cd_access = "a") if params.falcon_graph_serialize_path: @@ -103,7 +103,7 @@ def falcon(type, action = None, upgrade_type=None): owner = params.falcon_user, group = params.user_group, mode = 0775, - recursive = True, + create_parents = True, cd_access = "a") if type == 'server': @@ -117,7 +117,7 @@ def falcon(type, action = None, upgrade_type=None): elif params.store_uri[0:4] == "file": Directory(params.store_uri[7:], owner = params.falcon_user, - recursive = True) + create_parents = True) # TODO change to proper mode params.HdfsResource(params.flacon_apps_dir, @@ -135,7 +135,7 @@ def falcon(type, action = None, upgrade_type=None): elif params.falcon_store_uri[0:4] == "file": Directory(params.falcon_store_uri[7:], owner = params.falcon_user, - recursive = True) + create_parents = True) if params.supports_hive_dr: params.HdfsResource(params.dfs_data_mirroring_dir, @@ -151,18 +151,18 @@ def falcon(type, action = None, upgrade_type=None): params.HdfsResource(None, action = "execute") Directory(params.falcon_local_dir, owner = params.falcon_user, - recursive = True, + create_parents = True, cd_access = "a") if params.falcon_embeddedmq_enabled == True: Directory( os.path.abspath(os.path.join(params.falcon_embeddedmq_data, "..")), owner = params.falcon_user, - recursive = True) + create_parents = True) Directory(params.falcon_embeddedmq_data, owner = params.falcon_user, - recursive = True) + create_parents = True) # although Falcon's falcon-config.sh will use 'which hadoop' to figure # this out, in an upgraded cluster, it's possible that 'which hadoop' http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume.py b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume.py index 293df58..b30b122 100644 --- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume.py +++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume.py @@ -96,7 +96,7 @@ def flume(action = None): ) Directory(params.flume_conf_dir, - recursive=True, + create_parents = True, owner=params.flume_user, ) Directory(params.flume_log_dir, http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/GANGLIA/3.5.0/package/scripts/ganglia.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/GANGLIA/3.5.0/package/scripts/ganglia.py b/ambari-server/src/main/resources/common-services/GANGLIA/3.5.0/package/scripts/ganglia.py index 69fde27..9a69b72 100644 --- a/ambari-server/src/main/resources/common-services/GANGLIA/3.5.0/package/scripts/ganglia.py +++ b/ambari-server/src/main/resources/common-services/GANGLIA/3.5.0/package/scripts/ganglia.py @@ -35,7 +35,7 @@ def config(): Directory(shell_cmds_dir, owner="root", group="root", - recursive=True + create_parents = True ) init_file("gmetad") init_file("gmond") http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/GANGLIA/3.5.0/package/scripts/ganglia_monitor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/GANGLIA/3.5.0/package/scripts/ganglia_monitor.py b/ambari-server/src/main/resources/common-services/GANGLIA/3.5.0/package/scripts/ganglia_monitor.py index 05ceaff..feb0af3 100644 --- a/ambari-server/src/main/resources/common-services/GANGLIA/3.5.0/package/scripts/ganglia_monitor.py +++ b/ambari-server/src/main/resources/common-services/GANGLIA/3.5.0/package/scripts/ganglia_monitor.py @@ -71,7 +71,7 @@ class GangliaMonitor(Script): Directory(params.ganglia_conf_dir, owner="root", group=params.user_group, - recursive=True + create_parents = True ) ganglia.config() http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/GANGLIA/3.5.0/package/scripts/ganglia_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/GANGLIA/3.5.0/package/scripts/ganglia_server.py b/ambari-server/src/main/resources/common-services/GANGLIA/3.5.0/package/scripts/ganglia_server.py index 1893483..8d75d7f 100644 --- a/ambari-server/src/main/resources/common-services/GANGLIA/3.5.0/package/scripts/ganglia_server.py +++ b/ambari-server/src/main/resources/common-services/GANGLIA/3.5.0/package/scripts/ganglia_server.py @@ -83,14 +83,13 @@ def change_permission(): Directory(os.path.abspath(os.path.join(params.ganglia_runtime_dir, "..")), mode=0755, - recursive=True + create_parents = True ) Directory(params.dwoo_path, mode=0755, - recursive=True - ) - Execute(('chown', '-R', params.web_user, params.dwoo_path), - sudo = True, + create_parents = True, + owner = params.web_user, + recursive_ownership = True, ) def server_files(): @@ -98,7 +97,7 @@ def server_files(): rrd_py_path = params.rrd_py_path Directory(rrd_py_path, - recursive=True + create_parents = True ) rrd_py_file_path = path.join(rrd_py_path, "rrd.py") TemplateConfig(rrd_py_file_path, @@ -112,7 +111,7 @@ def server_files(): owner=rrd_file_owner, group=rrd_file_owner, mode=0755, - recursive=True + create_parents = True ) if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family(): http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py index ac81b11..5ca2049 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py @@ -223,7 +223,7 @@ def __update_limits_file(): """ import params # Ensure limits directory exists - Directory(hawq_constants.limits_conf_dir, recursive=True, owner=hawq_constants.root_user, group=hawq_constants.root_user) + Directory(hawq_constants.limits_conf_dir, create_parents = True, owner=hawq_constants.root_user, group=hawq_constants.root_user) # Generate limits for hawq user limits_file_content = "#### HAWQ Limits Parameters ###########\n" @@ -266,7 +266,7 @@ def __update_sysctl_file(): Updates /etc/sysctl.d/hawq_sysctl.conf file with the HAWQ parameters on CentOS/RHEL. """ # Ensure sys ctl sub-directory exists - Directory(hawq_constants.sysctl_conf_dir, recursive=True, owner=hawq_constants.root_user, group=hawq_constants.root_user) + Directory(hawq_constants.sysctl_conf_dir, create_parents = True, owner=hawq_constants.root_user, group=hawq_constants.root_user) # Generate temporary file with kernel parameters needed by hawq File(hawq_constants.hawq_sysctl_tmp_file, content=__convert_sysctl_dict_to_text(), owner=hawq_constants.hawq_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/utils.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/utils.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/utils.py index 316c7f2..c1a38bb 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/utils.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/utils.py @@ -35,7 +35,7 @@ def create_dir_as_hawq_user(directory): """ Creates directories with hawq_user and hawq_group (defaults to gpadmin:gpadmin) """ - Directory(directory, recursive=True, owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group) + Directory(directory, create_parents = True, owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group) def exec_hawq_operation(operation, option, not_if=None, only_if=None, logoutput=True): http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py index 22b67e1..c407a93 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py @@ -53,13 +53,13 @@ def hbase(name=None): Directory( params.hbase_conf_dir, owner = params.hbase_user, group = params.user_group, - recursive = True + create_parents = True ) Directory(params.java_io_tmpdir, owner = params.hbase_user, group = params.user_group, - recursive = True, + create_parents = True, mode=0777 ) parent_dir = os.path.dirname(params.tmp_dir) @@ -68,7 +68,7 @@ def hbase(name=None): parent_dir = os.path.dirname(parent_dir) if parent_dir != os.path.abspath(os.sep) : Directory (parent_dir, - recursive = True, + create_parents = True, cd_access="a", ) Execute(("chmod", "1777", parent_dir), sudo=True) @@ -129,7 +129,7 @@ def hbase(name=None): # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, - recursive=True, + create_parents = True, owner='root', group='root' ) @@ -153,14 +153,14 @@ def hbase(name=None): if name != "client": Directory( params.pid_dir, owner = params.hbase_user, - recursive = True, + create_parents = True, cd_access = "a", mode = 0755, ) Directory (params.log_dir, owner = params.hbase_user, - recursive = True, + create_parents = True, cd_access = "a", mode = 0755, ) http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py index 1a34a24..246bf07 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py @@ -33,7 +33,7 @@ def hdfs(name=None): # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, - recursive=True, + create_parents = True, owner='root', group='root' ) @@ -71,7 +71,7 @@ def hdfs(name=None): ) Directory(params.hadoop_conf_secure_dir, - recursive=True, + create_parents = True, owner='root', group=params.user_group, cd_access='a', @@ -122,7 +122,7 @@ def hdfs(name=None): def install_snappy(): import params Directory([params.so_target_dir_x86, params.so_target_dir_x64], - recursive=True, + create_parents = True, ) Link(params.so_target_x86, to=params.so_src_x86, @@ -139,7 +139,7 @@ def hdfs(component=None): Directory(directories, owner=params.hdfs_user, mode="(OI)(CI)F", - recursive=True + create_parents = True ) File(params.exclude_file_path, content=Template("exclude_hosts_list.j2"), http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_datanode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_datanode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_datanode.py index e225927..a22fb99 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_datanode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_datanode.py @@ -30,7 +30,7 @@ def create_dirs(data_dir, params): :param params: parameters """ Directory(data_dir, - recursive=True, + create_parents = True, cd_access="a", mode=0755, owner=params.hdfs_user, @@ -43,14 +43,14 @@ def datanode(action=None): if action == "configure": import params Directory(params.dfs_domain_socket_dir, - recursive=True, + create_parents = True, mode=0751, owner=params.hdfs_user, group=params.user_group) if not os.path.isdir(os.path.dirname(params.data_dir_mount_file)): Directory(os.path.dirname(params.data_dir_mount_file), - recursive=True, + create_parents = True, mode=0755, owner=params.hdfs_user, group=params.user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py index 004caff..b91fdb5 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py @@ -228,7 +228,7 @@ def create_name_dirs(directories): mode=0755, owner=params.hdfs_user, group=params.user_group, - recursive=True, + create_parents = True, cd_access="a", ) @@ -278,7 +278,7 @@ def format_namenode(force=None): ) for m_dir in mark_dir: Directory(m_dir, - recursive = True + create_parents = True ) else: if params.dfs_ha_namenode_active is not None and \ @@ -298,7 +298,7 @@ def format_namenode(force=None): ) for m_dir in mark_dir: Directory(m_dir, - recursive = True + create_parents = True ) def is_namenode_formatted(params): @@ -316,7 +316,7 @@ def is_namenode_formatted(params): if marked: for mark_dir in mark_dirs: Directory(mark_dir, - recursive = True + create_parents = True ) return marked @@ -334,7 +334,7 @@ def is_namenode_formatted(params): elif os.path.isfile(old_mark_dir): for mark_dir in mark_dirs: Directory(mark_dir, - recursive = True, + create_parents = True, ) Directory(old_mark_dir, action = "delete" http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_snamenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_snamenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_snamenode.py index aded211..500ed15 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_snamenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_snamenode.py @@ -28,7 +28,7 @@ def snamenode(action=None, format=False): import params for fs_checkpoint_dir in params.fs_checkpoint_dirs: Directory(fs_checkpoint_dir, - recursive=True, + create_parents = True, cd_access="a", mode=0755, owner=params.hdfs_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py index 2ef1b69..ad4283c 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py @@ -88,7 +88,7 @@ class JournalNodeDefault(JournalNode): import params Directory(params.jn_edits_dir, - recursive=True, + create_parents = True, cd_access="a", owner=params.hdfs_user, group=params.user_group http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py index aa97af0..e59dd78 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py @@ -206,7 +206,7 @@ def service(action=None, name=None, user=None, options="", create_pid_dir=False, if create_pid_dir: Directory(pid_dir, owner=user, - recursive=True) + create_parents = True) if create_log_dir: if name == "nfs3": Directory(log_dir, @@ -216,7 +216,7 @@ def service(action=None, name=None, user=None, options="", create_pid_dir=False, else: Directory(log_dir, owner=user, - recursive=True) + create_parents = True) if params.security_enabled and name == "datanode": ## The directory where pid files are stored in the secure data environment. http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat.py index 5385099..45dcbe7 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat.py @@ -43,21 +43,21 @@ def hcat(): from setup_atlas_hive import setup_atlas_hive Directory(params.hive_conf_dir, - recursive=True, + create_parents = True, owner=params.hcat_user, group=params.user_group, ) Directory(params.hcat_conf_dir, - recursive=True, + create_parents = True, owner=params.hcat_user, group=params.user_group, ) Directory(params.hcat_pid_dir, owner=params.webhcat_user, - recursive=True + create_parents = True ) XmlConfig("hive-site.xml", http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py index 39b3433..a141185 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py @@ -240,7 +240,7 @@ def hive(name=None): # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, - recursive=True, + create_parents = True, owner='root', group='root' ) @@ -308,7 +308,7 @@ def fill_conf_dir(component_conf_dir): Directory(component_conf_dir, owner=params.hive_user, group=params.user_group, - recursive=True + create_parents = True ) XmlConfig("mapred-site.xml", @@ -359,7 +359,7 @@ def crt_directory(name): import params Directory(name, - recursive=True, + create_parents = True, cd_access='a', owner=params.hive_user, group=params.user_group, @@ -398,7 +398,7 @@ def jdbc_connector(): Execute(format("yes | {sudo} cp {jars_path_in_archive} {hive_lib}")) Directory(params.jdbc_libs_dir, - recursive=True) + create_parents = True) Execute(format("yes | {sudo} cp {libs_path_in_archive} {jdbc_libs_dir}")) http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py index e347e25..5866cd0 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py @@ -207,7 +207,7 @@ class HiveMetastoreDefault(HiveMetastore): Execute(format("yes | {sudo} cp {jars_in_hive_lib} {target_directory}")) - Directory(target_native_libs_directory, recursive=True) + Directory(target_native_libs_directory, create_parents = True) Execute(format("yes | {sudo} cp {libs_in_hive_lib} {target_native_libs_directory}")) http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py index b597852..401debc 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py @@ -50,16 +50,16 @@ def webhcat(): owner=params.webhcat_user, mode=0755, group=params.user_group, - recursive=True) + create_parents = True) Directory(params.templeton_log_dir, owner=params.webhcat_user, mode=0755, group=params.user_group, - recursive=True) + create_parents = True) Directory(params.config_dir, - recursive=True, + create_parents = True, owner=params.webhcat_user, group=params.user_group, cd_access="a") @@ -116,7 +116,7 @@ def webhcat(): Directory(params.webhcat_conf_dir, cd_access='a', - recursive=True + create_parents = True ) log4j_webhcat_filename = 'webhcat-log4j.properties' http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py index faabee0..1c01174 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py @@ -82,8 +82,9 @@ def kafka(upgrade_type=None): cd_access='a', owner=params.kafka_user, group=params.user_group, - recursive=True) - set_dir_ownership(kafka_data_dirs) + create_parents = True, + recursive_ownership = True, + ) PropertiesFile("server.properties", dir=params.conf_dir, @@ -114,7 +115,7 @@ def kafka(upgrade_type=None): # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, - recursive=True, + create_parents = True, owner='root', group='root' ) @@ -150,7 +151,7 @@ def setup_symlink(kafka_managed_dir, kafka_ambari_managed_dir): Directory(kafka_managed_dir, action="delete", - recursive=True) + create_parents = True) elif os.path.islink(kafka_managed_dir) and os.path.realpath(kafka_managed_dir) != kafka_ambari_managed_dir: Link(kafka_managed_dir, @@ -169,8 +170,9 @@ def setup_symlink(kafka_managed_dir, kafka_ambari_managed_dir): cd_access='a', owner=params.kafka_user, group=params.user_group, - recursive=True) - set_dir_ownership(kafka_managed_dir) + create_parents = True, + recursive_ownership = True, + ) if backup_folder_path: # Restore backed up files to current relevant dirs if needed - will be triggered only when changing to/from default path; @@ -182,7 +184,7 @@ def setup_symlink(kafka_managed_dir, kafka_ambari_managed_dir): # Clean up backed up folder Directory(backup_folder_path, action="delete", - recursive=True) + create_parents = True) # Uses agent temp dir to store backup files @@ -194,9 +196,9 @@ def backup_dir_contents(dir_path, backup_folder_suffix): cd_access='a', owner=params.kafka_user, group=params.user_group, - recursive=True + create_parents = True, + recursive_ownership = True, ) - set_dir_ownership(backup_destination_path) # Safely copy top-level contents to backup folder for file in os.listdir(dir_path): File(os.path.join(backup_destination_path, file), @@ -205,35 +207,13 @@ def backup_dir_contents(dir_path, backup_folder_suffix): return backup_destination_path - def ensure_base_directories(): - """ - Make basic Kafka directories, and make sure that their ownership is correct - """ import params - base_dirs = [params.kafka_log_dir, params.kafka_pid_dir, params.conf_dir] - Directory(base_dirs, + Directory([params.kafka_log_dir, params.kafka_pid_dir, params.conf_dir], mode=0755, cd_access='a', owner=params.kafka_user, group=params.user_group, - recursive=True + create_parents = True, + recursive_ownership = True, ) - set_dir_ownership(base_dirs) - - -def set_dir_ownership(targets): - import params - if isinstance(targets, collections.Iterable): - directories = targets - else: # If target is a single object, convert it to list - directories = [targets] - for directory in directories: - # If path is empty or a single slash, - # may corrupt filesystem permissions - if len(directory) > 1: - Execute(('chown', '-R', format("{kafka_user}:{user_group}"), directory), - sudo=True) - else: - Logger.warning("Permissions for the folder \"%s\" were not updated due to " - "empty path passed: " % directory) http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/package/scripts/kerberos_common.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/package/scripts/kerberos_common.py b/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/package/scripts/kerberos_common.py index d40627a..b72c4bf 100644 --- a/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/package/scripts/kerberos_common.py +++ b/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/package/scripts/kerberos_common.py @@ -102,7 +102,7 @@ class KerberosScript(Script): Directory(params.krb5_conf_dir, owner='root', - recursive=True, + create_parents = True, group='root', mode=0755 ) @@ -366,7 +366,7 @@ class KerberosScript(Script): if (keytab_file_path is not None) and (len(keytab_file_path) > 0): head, tail = os.path.split(keytab_file_path) if head: - Directory(head, recursive=True, mode=0755, owner="root", group="root") + Directory(head, create_parents = True, mode=0755, owner="root", group="root") owner = get_property_value(item, 'keytab_file_owner_name') owner_access = get_property_value(item, 'keytab_file_owner_access') @@ -438,7 +438,7 @@ class KerberosScript(Script): if params.jce_policy_zip is not None: jce_curl_target = format("{artifact_dir}/{jce_policy_zip}") Directory(params.artifact_dir, - recursive = True, + create_parents = True, ) File(jce_curl_target, content = DownloadSource(format("{jce_location}/{jce_policy_zip}")), http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/package/scripts/kerberos_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/package/scripts/kerberos_server.py b/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/package/scripts/kerberos_server.py index 8cc4312..7fe02ab 100644 --- a/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/package/scripts/kerberos_server.py +++ b/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/package/scripts/kerberos_server.py @@ -27,7 +27,7 @@ class KerberosServer(KerberosScript): Directory(params.kadm5_acl_dir, owner='root', - recursive=True, + create_parents = True, group='root', mode=0700 ) @@ -50,7 +50,7 @@ class KerberosServer(KerberosScript): Directory(params.kdc_conf_dir, owner='root', - recursive=True, + create_parents = True, group='root', mode=0700 ) http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox.py index 806cec8..5436e47 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox.py @@ -88,16 +88,14 @@ def knox(): @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def knox(): import params - - directories = [params.knox_data_dir, params.knox_logs_dir, params.knox_pid_dir, params.knox_conf_dir, os.path.join(params.knox_conf_dir, "topologies")] - for directory in directories: - Directory(directory, - owner = params.knox_user, - group = params.knox_group, - recursive = True, - cd_access = "a", - mode = 0755, - ) + Directory([params.knox_data_dir, params.knox_logs_dir, params.knox_pid_dir, params.knox_conf_dir, os.path.join(params.knox_conf_dir, "topologies")], + owner = params.knox_user, + group = params.knox_group, + create_parents = True, + cd_access = "a", + mode = 0755, + recursive_ownership = True, + ) XmlConfig("gateway-site.xml", conf_dir=params.knox_conf_dir, @@ -139,12 +137,6 @@ def knox(): template_tag = None ) - dirs_to_chown = tuple(directories) - cmd = ('chown','-R',format('{knox_user}:{knox_group}')) + dirs_to_chown - Execute(cmd, - sudo = True, - ) - cmd = format('{knox_client_bin} create-master --master {knox_master_secret!p}') master_secret_exist = as_user(format('test -f {knox_master_secret_path}'), params.knox_user) @@ -181,20 +173,12 @@ def update_knox_logfolder_permissions(): cluster in non-working state """ import params - knox_dirs = [params.knox_logs_dir] - + Directory(params.knox_logs_dir, owner = params.knox_user, group = params.knox_group, - recursive = True, + create_parents = True, cd_access = "a", mode = 0755, - ) - - for d in knox_dirs: - if len(d) > 1: # If path is empty or a single slash, may corrupt filesystem permissions - Execute(('chown', '-R', format("{knox_user}:{knox_group}"), d), - sudo=True - ) - else: - Logger.warning("Permissions for the Knox folder \"%s\" was not updated due to empty path passed" % d) + recursive_ownership = True, + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/mahout.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/mahout.py b/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/mahout.py index 0fa937b..f2c3c18 100644 --- a/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/mahout.py +++ b/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/mahout.py @@ -26,7 +26,7 @@ def mahout(): import params Directory( params.mahout_conf_dir, - recursive = True, + create_parents = True, owner = params.mahout_user, group = params.user_group ) http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py index ba56c26..b35eec0 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py @@ -57,7 +57,7 @@ def oozie(is_server=False): Directory(params.oozie_tmp_dir, owner=params.oozie_user, - recursive = True, + create_parents = True, ) if is_server: @@ -96,7 +96,7 @@ def oozie(is_server=False): ) params.HdfsResource(None, action="execute") Directory(params.conf_dir, - recursive = True, + create_parents = True, owner = params.oozie_user, group = params.user_group ) @@ -193,12 +193,12 @@ def oozie_server_specific(): owner = params.oozie_user, group = params.user_group, mode = 0755, - recursive = True, + create_parents = True, cd_access="a", ) Directory(params.oozie_libext_dir, - recursive=True, + create_parents = True, ) hashcode_file = format("{oozie_home}/.hashcode") @@ -215,12 +215,18 @@ def oozie_server_specific(): configure_cmds = [] configure_cmds.append(('cp', params.ext_js_path, params.oozie_libext_dir)) configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}'))) - configure_cmds.append(('chown', '-RL', format('{oozie_user}:{user_group}'), params.oozie_webapps_conf_dir)) Execute( configure_cmds, not_if = no_op_test, sudo = True, ) + + Directory(params.oozie_webapps_conf_dir, + user = params.oozie_user, + group = params.user_group, + recursive_ownership = True, + recursion_follow_links = True, + ) # download the database JAR download_database_library_if_needed() @@ -259,7 +265,7 @@ def oozie_server_specific(): if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0: # Create hive-site and tez-site configs for oozie Directory(params.hive_conf_dir, - recursive = True, + create_parents = True, owner = params.oozie_user, group = params.user_group ) @@ -281,8 +287,10 @@ def oozie_server_specific(): group = params.user_group, mode = 0664 ) - Execute(('chown', '-R', format("{oozie_user}:{user_group}"), params.oozie_server_dir), - sudo=True + Directory(params.oozie_server_dir, + owner = params.oozie_user, + group = params.user_group, + recursive_ownership = True, ) def download_database_library_if_needed(target_directory = None): @@ -323,7 +331,7 @@ def download_database_library_if_needed(target_directory = None): Execute(format("yes | {sudo} cp {jars_path_in_archive} {oozie_libext_dir}")) Directory(params.jdbc_libs_dir, - recursive=True) + create_parents = True) Execute(format("yes | {sudo} cp {libs_path_in_archive} {jdbc_libs_dir}")) http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py index c5f095f..83f7048 100644 --- a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py +++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py @@ -28,7 +28,7 @@ def pig(): import params Directory( params.pig_conf_dir, - recursive = True, + create_parents = True, owner = params.hdfs_user, group = params.user_group ) http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf.py b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf.py index 08475fd..caae9bf 100644 --- a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf.py +++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf.py @@ -121,7 +121,7 @@ class Pxf(Script): Directory(params.pxf_instance_dir, owner=params.pxf_user, group=params.pxf_group, - recursive=True) + create_parents = True) if __name__ == "__main__": http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_service.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_service.py index 4f711c3..9270ea3 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_service.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_service.py @@ -34,7 +34,11 @@ def ranger_service(name, action=None): if params.stack_is_hdp23_or_further: - Execute(('chown','-R', format('{unix_user}:{unix_group}'), format('{usersync_log_dir}/')), sudo=True) + Directory(format('{usersync_log_dir}/'), + user = params.unix_user, + group = params.unix_group, + recursive_ownership = True, + ) Execute(params.usersync_start, environment=env_dict, not_if=no_op_test, http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py index 8de0d80..81f93a1 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py @@ -52,7 +52,7 @@ def setup_ranger_admin(upgrade_type=None): Directory(ranger_conf, owner = params.unix_user, group = params.unix_group, - recursive = True + create_parents = True ) if upgrade_type is not None: @@ -95,7 +95,11 @@ def setup_ranger_admin(upgrade_type=None): Execute(('cp', '-f', src_file, dst_file), sudo=True) - Execute(('chown','-R',format('{unix_user}:{unix_group}'), format('{ranger_home}/')), sudo=True) + Directory(format('{ranger_home}/'), + user = params.unix_user, + group = params.unix_group, + recursive_ownership = True, + ) Directory(params.admin_log_dir, owner = params.unix_user, @@ -140,7 +144,7 @@ def setup_ranger_db(upgrade_type=None): Directory(params.java_share_dir, mode=0755, - recursive=True, + create_parents = True, cd_access="a" ) @@ -164,7 +168,7 @@ def setup_ranger_db(upgrade_type=None): Directory(params.jdbc_libs_dir, cd_access="a", - recursive=True) + create_parents = True) Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False), path=["/bin", "/usr/bin/"]) http://git-wip-us.apache.org/repos/asf/ambari/blob/92c054b1/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py index 25134df..a611f8e 100755 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py @@ -61,7 +61,7 @@ def setup_kms_db(): Directory(params.java_share_dir, mode=0755, - recursive=True, + create_parents = True, cd_access="a" ) @@ -85,7 +85,7 @@ def setup_kms_db(): Directory(params.jdbc_libs_dir, cd_access="a", - recursive=True) + create_parents = True) Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False), path=["/bin", "/usr/bin/"]) @@ -171,7 +171,7 @@ def kms(): Directory(params.kms_conf_dir, owner = params.kms_user, group = params.kms_group, - recursive = True + create_parents = True ) if params.xa_audit_db_is_enabled: @@ -201,7 +201,11 @@ def kms(): mode = 0755 ) - Execute(('chown','-R',format('{kms_user}:{kms_group}'), format('{kms_home}/')), sudo=True) + Directory(format('{kms_home}/'), + user = params.kms_user, + group = params.kms_group, + recursive_ownership = True, + ) Directory(params.kms_log_dir, owner = params.kms_user, @@ -301,7 +305,7 @@ def enable_kms_plugin(): owner = params.kms_user, group = params.kms_group, mode=0775, - recursive = True + create_parents = True ) File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',format('kms_{repo_name}.json')),
