Repository: ambari Updated Branches: refs/heads/branch-1.7.0 1b3cfd6c4 -> ee9be26cc
AMBARI-8016 Cannot run hive tez query using HDP 2.1 stack (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ee9be26c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ee9be26c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ee9be26c Branch: refs/heads/branch-1.7.0 Commit: ee9be26cc7168b607242cab8f0956ff5e21cf349 Parents: 1b3cfd6 Author: Dmytro Sen <[email protected]> Authored: Wed Oct 29 19:14:34 2014 +0200 Committer: Dmytro Sen <[email protected]> Committed: Wed Oct 29 19:14:34 2014 +0200 ---------------------------------------------------------------------- .../HIVE/package/scripts/hive_server.py | 4 +- .../HIVE/package/scripts/install_jars.py | 107 +++++++++++++++++++ .../services/HIVE/package/scripts/params.py | 8 ++ .../stacks/2.0.6/HIVE/test_hive_server.py | 53 +++++++++ 4 files changed, 171 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ee9be26c/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py index 4e55cfb..1110544 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py @@ -22,6 +22,7 @@ from resource_management import * from hive import hive from hive_service import hive_service from resource_management.libraries.functions.dynamic_variable_interpretation import copy_tarballs_to_hdfs +from install_jars import install_tez_jars class HiveServer(Script): @@ -32,7 +33,8 @@ class HiveServer(Script): def configure(self, env): import params env.set_params(params) - + if not params.stack_is_hdp22_or_further: + install_tez_jars() hive(name='hiveserver2') def start(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/ee9be26c/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py new file mode 100644 index 0000000..a18ca72 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +import os + +def install_tez_jars(): + import params + + destination_hdfs_dirs = get_tez_hdfs_dir_paths(params.tez_lib_uris) + + # If tez libraries are to be stored in hdfs + if destination_hdfs_dirs: + for hdfs_dir in destination_hdfs_dirs: + params.HdfsDirectory(hdfs_dir, + action="create_delayed", + owner=params.tez_user, + mode=0755 + ) + pass + params.HdfsDirectory(None, action="create") + + if params.security_enabled: + kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};") + else: + kinit_if_needed = "" + + if kinit_if_needed: + Execute(kinit_if_needed, + user=params.tez_user, + path='/bin' + ) + pass + + app_dir_path = None + lib_dir_path = None + + if len(destination_hdfs_dirs) > 0: + for path in destination_hdfs_dirs: + if 'lib' in path: + lib_dir_path = path + else: + app_dir_path = path + pass + pass + pass + + if app_dir_path: + for scr_file, dest_file in params.app_dir_files.iteritems(): + CopyFromLocal(scr_file, + mode=0755, + owner=params.tez_user, + dest_dir=app_dir_path, + dest_file=dest_file, + kinnit_if_needed=kinit_if_needed, + hdfs_user=params.hdfs_user, + hadoop_bin_dir=params.hadoop_bin_dir, + hadoop_conf_dir=params.hadoop_conf_dir + ) + + if lib_dir_path: + CopyFromLocal(params.tez_local_lib_jars, + mode=0755, + owner=params.tez_user, + dest_dir=lib_dir_path, + kinnit_if_needed=kinit_if_needed, + hdfs_user=params.hdfs_user, + hadoop_bin_dir=params.hadoop_bin_dir, + hadoop_conf_dir=params.hadoop_conf_dir + ) + pass + + +def get_tez_hdfs_dir_paths(tez_lib_uris = None): + hdfs_path_prefix = 'hdfs://' + lib_dir_paths = [] + if tez_lib_uris and tez_lib_uris.strip().find(hdfs_path_prefix, 0) != -1: + dir_paths = tez_lib_uris.split(',') + for path in dir_paths: + if not "tez.tar.gz" in path: + lib_dir_path = path.replace(hdfs_path_prefix, '') + lib_dir_path = lib_dir_path if lib_dir_path.endswith(os.sep) else lib_dir_path + os.sep + lib_dir_paths.append(lib_dir_path) + else: + lib_dir_path = path.replace(hdfs_path_prefix, '') + lib_dir_paths.append(os.path.dirname(lib_dir_path)) + pass + pass + + return lib_dir_paths http://git-wip-us.apache.org/repos/asf/ambari/blob/ee9be26c/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py index 2bfbdc1..41c5d1a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py @@ -211,6 +211,14 @@ hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_nam # Tez-related properties tez_user = config['configurations']['tez-env']['tez_user'] +# Tez jars +tez_local_api_jars = '/usr/lib/tez/tez*.jar' +tez_local_lib_jars = '/usr/lib/tez/lib/*.jar' +app_dir_files = {tez_local_api_jars:None} + +# Tez libraries +tez_lib_uris = default("/configurations/tez-site/tez.lib.uris", None) + if System.get_instance().os_family == "ubuntu": mysql_configname = '/etc/mysql/my.cnf' else: http://git-wip-us.apache.org/repos/asf/ambari/blob/ee9be26c/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py index 0abc438..f28e821 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py @@ -137,6 +137,59 @@ class TestHiveServer(RMFTestCase): self.assertFalse(socket_mock.called) def assert_configure_default(self): + self.assertResourceCalled('HdfsDirectory', '/apps/tez/', + action = ['create_delayed'], + mode = 0755, + owner = 'tez', + security_enabled = False, + keytab = UnknownConfigurationMock(), + conf_dir = '/etc/hadoop/conf', + hdfs_user = 'hdfs', + bin_dir = '/usr/bin', + kinit_path_local = "/usr/bin/kinit" + ) + + self.assertResourceCalled('HdfsDirectory', '/apps/tez/lib/', + action = ['create_delayed'], + mode = 0755, + owner = 'tez', + security_enabled = False, + keytab = UnknownConfigurationMock(), + conf_dir = '/etc/hadoop/conf', + hdfs_user = 'hdfs', + bin_dir = '/usr/bin', + kinit_path_local = "/usr/bin/kinit" + ) + self.assertResourceCalled('HdfsDirectory', None, + security_enabled = False, + keytab = UnknownConfigurationMock(), + conf_dir = '/etc/hadoop/conf', + hdfs_user = 'hdfs', + kinit_path_local = '/usr/bin/kinit', + bin_dir = '/usr/bin', + action = ['create'] + ) + + self.assertResourceCalled('CopyFromLocal', '/usr/lib/tez/tez*.jar', + mode=0755, + owner='tez', + dest_dir='/apps/tez/', + kinnit_if_needed='', + hadoop_conf_dir='/etc/hadoop/conf', + hadoop_bin_dir='/usr/bin', + hdfs_user='hdfs', + dest_file=None + ) + + self.assertResourceCalled('CopyFromLocal', '/usr/lib/tez/lib/*.jar', + mode=0755, + owner='tez', + dest_dir='/apps/tez/lib/', + kinnit_if_needed='', + hadoop_bin_dir='/usr/bin', + hadoop_conf_dir='/etc/hadoop/conf', + hdfs_user='hdfs' + ) self.assertResourceCalled('HdfsDirectory', '/apps/hive/warehouse', security_enabled = False, keytab = UnknownConfigurationMock(),
