Repository: ambari Updated Branches: refs/heads/trunk 4bcf111f1 -> c09edb729
AMBARI-7333. Tez deployment changes for Champlain (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c09edb72 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c09edb72 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c09edb72 Branch: refs/heads/trunk Commit: c09edb729fce602f0943b048df7810353ecd4975 Parents: 4bcf111 Author: Andrew Onishuk <[email protected]> Authored: Wed Sep 17 18:11:06 2014 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Wed Sep 17 18:11:06 2014 +0300 ---------------------------------------------------------------------- .../libraries/providers/copy_from_local.py | 12 +++++++++--- .../libraries/resources/copy_from_local.py | 3 ++- .../HIVE/package/scripts/install_jars.py | 19 +++++++++++++++---- .../2.1/services/TEZ/configuration/tez-site.xml | 2 +- .../python/stacks/2.0.6/HIVE/test_hive_server.py | 10 ++++++++++ 5 files changed, 37 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c09edb72/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py b/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py index 9031a77..79d3e35 100644 --- a/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py +++ b/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py @@ -27,6 +27,7 @@ class CopyFromLocalProvider(Provider): def action_run(self): path = self.resource.path dest_dir = self.resource.dest_dir + dest_file = self.resource.dest_file kinnit_if_needed = self.resource.kinnit_if_needed owner = self.resource.owner group = self.resource.group @@ -34,9 +35,14 @@ class CopyFromLocalProvider(Provider): hdfs_usr=self.resource.hdfs_user hadoop_conf_path = self.resource.hadoop_conf_dir - copy_cmd = format("fs -copyFromLocal {path} {dest_dir}") - dest_file_name = os.path.split(path)[1] - dest_path = dest_dir + dest_file_name if dest_dir.endswith(os.sep) else dest_dir + os.sep + dest_file_name + + if dest_file: + copy_cmd = format("fs -copyFromLocal {path} {dest_dir}/{dest_file}") + dest_path = dest_dir + dest_file if dest_dir.endswith(os.sep) else dest_dir + os.sep + dest_file + else: + dest_file_name = os.path.split(path)[1] + copy_cmd = format("fs -copyFromLocal {path} {dest_dir}") + dest_path = dest_dir + os.sep + dest_file_name # Need to run unless as resource user su_cmd = 'su - {0} -c'.format(owner) unless_cmd = format("{su_cmd} '{kinnit_if_needed} hadoop fs -ls {dest_path}' >/dev/null 2>&1") http://git-wip-us.apache.org/repos/asf/ambari/blob/c09edb72/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py b/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py index 328d9c2..eaaeab5 100644 --- a/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py +++ b/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py @@ -28,6 +28,7 @@ class CopyFromLocal(Resource): path = ResourceArgument(default=lambda obj: obj.name) dest_dir = ResourceArgument(required=True) + dest_file = ResourceArgument() owner = ResourceArgument(required=True) group = ResourceArgument() mode = ResourceArgument() @@ -35,4 +36,4 @@ class CopyFromLocal(Resource): hadoop_conf_dir = ResourceArgument(default='/etc/hadoop/conf') hdfs_user = ResourceArgument(default='hdfs') - actions = Resource.actions + ["run"] \ No newline at end of file + actions = Resource.actions + ["run"] http://git-wip-us.apache.org/repos/asf/ambari/blob/c09edb72/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py index 3548de7..0045fee 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py @@ -72,6 +72,16 @@ def install_tez_jars(): hdfs_user=params.hdfs_user, hadoop_conf_dir=params.hadoop_conf_dir ) + + CopyFromLocal(params.tez_tar_file, + owner=params.tez_user, + mode=0755, + dest_dir=app_dir_path, + dest_file="tez.tar.gz", + kinnit_if_needed=kinit_if_needed, + hdfs_user=params.hdfs_user, + hadoop_conf_dir=params.hadoop_conf_dir + ) pass if lib_dir_path: @@ -92,10 +102,11 @@ def get_tez_hdfs_dir_paths(tez_lib_uris = None): if tez_lib_uris and tez_lib_uris.strip().find(hdfs_path_prefix, 0) != -1: dir_paths = tez_lib_uris.split(',') for path in dir_paths: - lib_dir_path = path.replace(hdfs_path_prefix, '') - lib_dir_path = lib_dir_path if lib_dir_path.endswith(os.sep) else lib_dir_path + os.sep - lib_dir_paths.append(lib_dir_path) + if not "tez.tar.gz" in path: + lib_dir_path = path.replace(hdfs_path_prefix, '') + lib_dir_path = lib_dir_path if lib_dir_path.endswith(os.sep) else lib_dir_path + os.sep + lib_dir_paths.append(lib_dir_path) pass pass - return lib_dir_paths \ No newline at end of file + return lib_dir_paths http://git-wip-us.apache.org/repos/asf/ambari/blob/c09edb72/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/configuration/tez-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/configuration/tez-site.xml index 218c508..7b34d3b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/configuration/tez-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/configuration/tez-site.xml @@ -21,7 +21,7 @@ <property> <name>tez.lib.uris</name> - <value>hdfs:///apps/tez/,hdfs:///apps/tez/lib/</value> + <value>hdfs:///apps/tez/,hdfs:///apps/tez/lib/,hdfs:///apps/tez/tez.tar.gz</value> <description>The location of the Tez libraries which will be localized for DAGs</description> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/c09edb72/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py index bc723ab..e0dc3d8 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py @@ -88,6 +88,16 @@ class TestHiveServer(RMFTestCase): hdfs_user='hdfs' ) + self.assertResourceCalled('CopyFromLocal', '/usr/lib/tez/tez*.tar.gz', + mode=0755, + owner='tez', + dest_dir='/apps/tez/', + dest_file='tez.tar.gz', + kinnit_if_needed='', + hadoop_conf_dir='/etc/hadoop/conf', + hdfs_user='hdfs' + ) + self.assertResourceCalled('CopyFromLocal', '/usr/lib/tez/lib/*.jar', mode=0755, owner='tez',
