Add hadoop LZO jars if installed in the cluster. (Slim Bouguerra via Swapan Shridhar).
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bbec0d37 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bbec0d37 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bbec0d37 Branch: refs/heads/feature-branch-AMBARI-21307 Commit: bbec0d3751b9aad709d8fbf86c2744b4bb412e2d Parents: ca8ae51 Author: Swapan Shridhar <sshrid...@hortonworks.com> Authored: Fri Aug 18 14:10:55 2017 -0700 Committer: Swapan Shridhar <sshrid...@hortonworks.com> Committed: Fri Aug 18 14:10:55 2017 -0700 ---------------------------------------------------------------------- .../DRUID/0.9.2/package/scripts/druid.py | 12 ++++++++++ .../DRUID/0.9.2/package/scripts/params.py | 13 +++++++++++ .../test/python/stacks/2.6/DRUID/test_druid.py | 24 +++++++++++++------- 3 files changed, 41 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/bbec0d37/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py index cc02519..fa52247 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py @@ -18,6 +18,7 @@ limitations under the License. """ import json import os +from resource_management import Fail from resource_management.libraries.resources.properties_file import PropertiesFile from resource_management.core.resources.system import Directory, Execute, File from resource_management.core.source import DownloadSource @@ -113,6 +114,17 @@ def druid(upgrade_type=None, nodeType=None): node_jvm_opts=druid_env_config[format('druid.{node_type_lowercase}.jvm.opts')]) ) Logger.info(format("Created druid-{node_type_lowercase} jvm.config")) + # Handling hadoop Lzo jars if enable and node type is hadoop related eg Overlords and MMs + if ['middleManager', 'overlord'].__contains__(node_type_lowercase) and params.lzo_enabled and len( + params.lzo_packages) > 0: + try: + Logger.info( + format( + "Copying hadoop lzo jars from {hadoop_lib_home} to {druid_hadoop_dependencies_dir}/hadoop-client/*/")) + Execute( + format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {druid_hadoop_dependencies_dir}/hadoop-client/*/')) + except Fail as ex: + Logger.info(format("No Hadoop LZO found at {hadoop_lib_home}/hadoop-lzo*.jar")) # All druid nodes have dependency on hdfs_client ensure_hadoop_directories() http://git-wip-us.apache.org/repos/asf/ambari/blob/bbec0d37/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py index d47e604..7e90475 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py @@ -18,6 +18,7 @@ limitations under the License. """ from ambari_commons import OSCheck +from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select from resource_management.libraries.resources.hdfs_resource import HdfsResource @@ -26,6 +27,7 @@ from resource_management.libraries.script.script import Script from resource_management.libraries.functions import format from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources from resource_management.libraries.functions.default import default +from ambari_commons.constants import AMBARI_SUDO_BINARY import status_params @@ -50,10 +52,14 @@ stack_name = default("/hostLevelParams/stack_name", None) # stack version stack_version = default("/commandParams/version", None) +# un-formatted stack version +stack_version_unformatted = str(config['hostLevelParams']['stack_version']) + # default role to coordinator needed for service checks component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "DRUID_COORDINATOR") hostname = config['hostname'] +sudo = AMBARI_SUDO_BINARY # default druid parameters druid_home = format("{stack_root}/current/{component_directory}") @@ -230,3 +236,10 @@ if not len(druid_router_hosts) == 0: druid_router_host = druid_router_hosts[0] druid_coordinator_port = config['configurations']['druid-coordinator']['druid.port'] druid_router_port = config['configurations']['druid-router']['druid.port'] + +# Create current Hadoop Clients Libs +stack_version_unformatted = str(config['hostLevelParams']['stack_version']) +io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) +lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() +lzo_packages = get_lzo_packages(stack_version_unformatted) +hadoop_lib_home = stack_root + '/' + stack_version + '/hadoop/lib' http://git-wip-us.apache.org/repos/asf/ambari/blob/bbec0d37/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py index 41b0bf5..1d49f9a 100644 --- a/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py +++ b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py @@ -48,7 +48,8 @@ class TestDruid(RMFTestCase): config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", config_overrides = { 'role' : 'DRUID_OVERLORD' }, stack_version=self.STACK_VERSION, - target=RMFTestCase.TARGET_COMMON_SERVICES + target=RMFTestCase.TARGET_COMMON_SERVICES, + checked_call_mocks=[(0, "OK.", ""), (0, "OK.", "")] ) self.assert_configure_default('druid-overlord') self.assertNoMoreResources() @@ -60,7 +61,8 @@ class TestDruid(RMFTestCase): config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", stack_version=self.STACK_VERSION, config_overrides = { 'role' : 'DRUID_OVERLORD' }, - target=RMFTestCase.TARGET_COMMON_SERVICES + target=RMFTestCase.TARGET_COMMON_SERVICES, + checked_call_mocks=[(0, "OK.", ""), (0, "OK.", "")] ) self.assert_configure_default('druid-overlord') self.assertResourceCalled('Execute', format("/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/hdp/current/druid-overlord/extensions/mysql-metadata-storage/* org.apache.ambari.server.DBConnectionVerification 'jdbc:mysql://my-db-host:3306/druid?createDatabaseIfNotExist=true' druid diurd com.mysql.jdbc.Driver"), @@ -80,7 +82,8 @@ class TestDruid(RMFTestCase): config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", stack_version=self.STACK_VERSION, config_overrides = { 'role' : 'DRUID_OVERLORD' }, - target=RMFTestCase.TARGET_COMMON_SERVICES + target=RMFTestCase.TARGET_COMMON_SERVICES, + checked_call_mocks=[(0, "OK.", ""), (0, "OK.", "")] ) self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-overlord/conf/druid-env.sh ; /usr/hdp/current/druid-overlord/bin/node.sh overlord stop'), user='druid' @@ -94,7 +97,8 @@ class TestDruid(RMFTestCase): config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", config_overrides = { 'role' : 'DRUID_COORDINATOR' }, stack_version=self.STACK_VERSION, - target=RMFTestCase.TARGET_COMMON_SERVICES + target=RMFTestCase.TARGET_COMMON_SERVICES, + checked_call_mocks=[(0, "OK.", ""), (0, "OK.", "")] ) self.assert_configure_default('druid-coordinator') self.assertNoMoreResources() @@ -140,7 +144,8 @@ class TestDruid(RMFTestCase): config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", config_overrides = { 'role' : 'DRUID_BROKER' }, stack_version=self.STACK_VERSION, - target=RMFTestCase.TARGET_COMMON_SERVICES + target=RMFTestCase.TARGET_COMMON_SERVICES, + checked_call_mocks=[(0, "OK.", ""), (0, "OK.", "")] ) self.assert_configure_default('druid-broker') self.assertNoMoreResources() @@ -152,7 +157,8 @@ class TestDruid(RMFTestCase): config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", stack_version=self.STACK_VERSION, config_overrides = { 'role' : 'DRUID_BROKER' }, - target=RMFTestCase.TARGET_COMMON_SERVICES + target=RMFTestCase.TARGET_COMMON_SERVICES, + checked_call_mocks=[(0, "OK.", ""), (0, "OK.", "")] ) self.assert_configure_default('druid-broker') self.assertResourceCalled('Execute', format("/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/hdp/current/druid-broker/extensions/mysql-metadata-storage/* org.apache.ambari.server.DBConnectionVerification 'jdbc:mysql://my-db-host:3306/druid?createDatabaseIfNotExist=true' druid diurd com.mysql.jdbc.Driver"), @@ -232,7 +238,8 @@ class TestDruid(RMFTestCase): config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", config_overrides = { 'role' : 'DRUID_HISTORICAL' }, stack_version=self.STACK_VERSION, - target=RMFTestCase.TARGET_COMMON_SERVICES + target=RMFTestCase.TARGET_COMMON_SERVICES, + checked_call_mocks=[(0, "OK.", ""), (0, "OK.", "")] ) self.assert_configure_default('druid-historical') self.assertNoMoreResources() @@ -278,7 +285,8 @@ class TestDruid(RMFTestCase): config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", config_overrides = { 'role' : 'DRUID_MIDDLEMANAGER' }, stack_version=self.STACK_VERSION, - target=RMFTestCase.TARGET_COMMON_SERVICES + target=RMFTestCase.TARGET_COMMON_SERVICES, + checked_call_mocks=[(0, "OK.", ""), (0, "OK.", "")] ) self.assert_configure_default('druid-middlemanager') self.assertNoMoreResources()