http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py deleted file mode 100644 index ec98c3c..0000000 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py +++ /dev/null @@ -1,307 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import json -import os -from resource_management import Fail -from resource_management.libraries.resources.properties_file import PropertiesFile -from resource_management.core.resources.system import Directory, Execute, File -from resource_management.core.source import DownloadSource -from resource_management.core.source import InlineTemplate -from resource_management.libraries.functions import format -from resource_management.libraries.functions.show_logs import show_logs -from resource_management.core.logger import Logger - - -def druid(upgrade_type=None, nodeType=None): - import params - ensure_base_directories() - - # Environment Variables - File(format("{params.druid_conf_dir}/druid-env.sh"), - owner=params.druid_user, - content=InlineTemplate(params.druid_env_sh_template), - mode = 0700 - ) - - # common config - druid_common_config = mutable_config_dict(params.config['configurations']['druid-common']) - # User cannot override below configs - druid_common_config['druid.host'] = params.hostname - druid_common_config['druid.extensions.directory'] = params.druid_extensions_dir - druid_common_config['druid.extensions.hadoopDependenciesDir'] = params.druid_hadoop_dependencies_dir - druid_common_config['druid.selectors.indexing.serviceName'] = params.config['configurations']['druid-overlord'][ - 'druid.service'] - druid_common_config['druid.selectors.coordinator.serviceName'] = \ - params.config['configurations']['druid-coordinator']['druid.service'] - druid_common_config['druid.extensions.loadList'] = json.dumps(eval(params.druid_extensions_load_list) + - eval(params.druid_security_extensions_load_list)) - - # delete the password and user if empty otherwiswe derby will fail. - if 'derby' == druid_common_config['druid.metadata.storage.type']: - del druid_common_config['druid.metadata.storage.connector.user'] - del druid_common_config['druid.metadata.storage.connector.password'] - - druid_env_config = mutable_config_dict(params.config['configurations']['druid-env']) - - PropertiesFile("common.runtime.properties", - dir=params.druid_common_conf_dir, - properties=druid_common_config, - owner=params.druid_user, - group=params.user_group, - mode = 0600 - ) - Logger.info("Created common.runtime.properties") - - File(format("{params.druid_common_conf_dir}/druid-log4j.xml"), - mode=0644, - owner=params.druid_user, - group=params.user_group, - content=InlineTemplate(params.log4j_props) - ) - Logger.info("Created log4j file") - - File("/etc/logrotate.d/druid", - mode=0644, - owner='root', - group='root', - content=InlineTemplate(params.logrotate_props) - ) - - Logger.info("Created log rotate file") - - # node specific configs - for node_type in ['coordinator', 'overlord', 'historical', 'broker', 'middleManager', 'router']: - node_config_dir = format('{params.druid_conf_dir}/{node_type}') - node_type_lowercase = node_type.lower() - - # Write runtime.properties file - node_config = mutable_config_dict(params.config['configurations'][format('druid-{node_type_lowercase}')]) - PropertiesFile("runtime.properties", - dir=node_config_dir, - properties=node_config, - owner=params.druid_user, - group=params.user_group, - mode = 0600 - ) - Logger.info(format("Created druid-{node_type_lowercase} runtime.properties")) - - # Write jvm configs - File(format('{node_config_dir}/jvm.config'), - owner=params.druid_user, - group=params.user_group, - content=InlineTemplate( - "-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}", - node_heap_memory=druid_env_config[format('druid.{node_type_lowercase}.jvm.heap.memory')], - log4j_config_file=format("{params.druid_common_conf_dir}/druid-log4j.xml"), - node_direct_memory=druid_env_config[ - format('druid.{node_type_lowercase}.jvm.direct.memory')], - node_jvm_opts=druid_env_config[format('druid.{node_type_lowercase}.jvm.opts')]) - ) - Logger.info(format("Created druid-{node_type_lowercase} jvm.config")) - # Handling hadoop Lzo jars if enable and node type is hadoop related eg Overlords and MMs - if ['middleManager', 'overlord'].__contains__(node_type_lowercase) and params.lzo_enabled and len( - params.lzo_packages) > 0: - try: - Logger.info( - format( - "Copying hadoop lzo jars from {hadoop_lib_home} to {druid_hadoop_dependencies_dir}/hadoop-client/*/")) - Execute( - format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {druid_hadoop_dependencies_dir}/hadoop-client/*/')) - except Fail as ex: - Logger.info(format("No Hadoop LZO found at {hadoop_lib_home}/hadoop-lzo*.jar")) - - # All druid nodes have dependency on hdfs_client - ensure_hadoop_directories() - download_database_connector_if_needed() - # Pull all required dependencies - pulldeps() - - -def mutable_config_dict(config): - rv = {} - for key, value in config.iteritems(): - rv[key] = value - return rv - - -def ensure_hadoop_directories(): - import params - if 'hdfs-site' not in params.config['configurations']: - # HDFS Not Installed nothing to do. - Logger.info("Skipping HDFS directory creation as HDFS not installed") - return - - druid_common_config = params.config['configurations']['druid-common'] - # final overlord config contains both common and overlord config - druid_middlemanager_config = params.config['configurations']['druid-middlemanager'] - - # If user is using HDFS as deep storage create HDFS Directory for storing segments - deep_storage = druid_common_config["druid.storage.type"] - storage_dir = druid_common_config["druid.storage.storageDirectory"] - - if deep_storage == 'hdfs': - # create the home dir for druid - params.HdfsResource(format("/user/{params.druid_user}"), - type="directory", - action="create_on_execute", - owner=params.druid_user, - group='hadoop', - recursive_chown=True, - recursive_chmod=True - ) - - # create the segment storage dir, users like hive from group hadoop need to write to this directory - create_hadoop_directory(storage_dir, mode=0775) - - # Create HadoopIndexTask hadoopWorkingPath - hadoop_working_path = druid_middlemanager_config['druid.indexer.task.hadoopWorkingPath'] - if hadoop_working_path is not None: - if hadoop_working_path.startswith(params.hdfs_tmp_dir): - params.HdfsResource(params.hdfs_tmp_dir, - type="directory", - action="create_on_execute", - owner=params.hdfs_user, - mode=0777, - ) - create_hadoop_directory(hadoop_working_path, mode=0775) - - # If HDFS is used for storing logs, create Index Task log directory - indexer_logs_type = druid_common_config['druid.indexer.logs.type'] - indexer_logs_directory = druid_common_config['druid.indexer.logs.directory'] - if indexer_logs_type == 'hdfs' and indexer_logs_directory is not None: - create_hadoop_directory(indexer_logs_directory) - - -def create_hadoop_directory(hadoop_dir, mode=0755): - import params - params.HdfsResource(hadoop_dir, - type="directory", - action="create_on_execute", - owner=params.druid_user, - group='hadoop', - mode=mode - ) - Logger.info(format("Created Hadoop Directory [{hadoop_dir}], with mode [{mode}]")) - - -def ensure_base_directories(): - import params - Directory( - [params.druid_log_dir, params.druid_pid_dir], - mode=0755, - owner=params.druid_user, - group=params.user_group, - create_parents=True, - recursive_ownership=True, - ) - - Directory( - [params.druid_conf_dir, params.druid_common_conf_dir, params.druid_coordinator_conf_dir, - params.druid_broker_conf_dir, params.druid_middlemanager_conf_dir, params.druid_historical_conf_dir, - params.druid_overlord_conf_dir, params.druid_router_conf_dir, params.druid_segment_infoDir, - params.druid_tasks_dir], - mode=0700, - cd_access='a', - owner=params.druid_user, - group=params.user_group, - create_parents=True, - recursive_ownership=True, - ) - - segment_cache_locations = json.loads(params.druid_segment_cache_locations) - for segment_cache_location in segment_cache_locations: - Directory( - segment_cache_location["path"], - mode=0700, - owner=params.druid_user, - group=params.user_group, - create_parents=True, - recursive_ownership=True, - cd_access='a' - ) - - - -def get_daemon_cmd(params=None, node_type=None, command=None): - return format('source {params.druid_conf_dir}/druid-env.sh ; {params.druid_home}/bin/node.sh {node_type} {command}') - - -def getPid(params=None, nodeType=None): - return format('{params.druid_pid_dir}/{nodeType}.pid') - - -def pulldeps(): - import params - extensions_list = eval(params.druid_extensions) - extensions_string = '{0}'.format("-c ".join(extensions_list)) - repository_list = eval(params.druid_repo_list) - repository_string = '{0}'.format("-r ".join(repository_list)) - if len(extensions_list) > 0: - try: - # Make sure druid user has permissions to write dependencies - Directory( - [params.druid_extensions_dir, params.druid_hadoop_dependencies_dir], - mode=0755, - cd_access='a', - owner=params.druid_user, - group=params.user_group, - create_parents=True, - recursive_ownership=True, - ) - pull_deps_command = format( - "source {params.druid_conf_dir}/druid-env.sh ; java -classpath '{params.druid_home}/lib/*' -Ddruid.extensions.loadList=[] " - "-Ddruid.extensions.directory={params.druid_extensions_dir} -Ddruid.extensions.hadoopDependenciesDir={params.druid_hadoop_dependencies_dir} " - "io.druid.cli.Main tools pull-deps -c {extensions_string} --no-default-hadoop") - - if len(repository_list) > 0: - pull_deps_command = format("{pull_deps_command} -r {repository_string}") - - Execute(pull_deps_command, - user=params.druid_user - ) - Logger.info(format("Pull Dependencies Complete")) - except: - show_logs(params.druid_log_dir, params.druid_user) - raise - - -def download_database_connector_if_needed(): - """ - Downloads the database connector to use when connecting to the metadata storage - """ - import params - if params.metadata_storage_type != 'mysql' or not params.jdbc_driver_jar: - return - - File(params.check_db_connection_jar, - content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")) - ) - - target_jar_with_directory = params.connector_download_dir + os.path.sep + params.jdbc_driver_jar - - if not os.path.exists(target_jar_with_directory): - File(params.downloaded_custom_connector, - content=DownloadSource(params.connector_curl_source)) - - Execute(('cp', '--remove-destination', params.downloaded_custom_connector, target_jar_with_directory), - path=["/bin", "/usr/bin/"], - sudo=True) - - File(target_jar_with_directory, owner=params.druid_user, - group=params.user_group)
http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid_node.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid_node.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid_node.py deleted file mode 100644 index 8053dcb..0000000 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid_node.py +++ /dev/null @@ -1,114 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import os -from resource_management.core import sudo -from resource_management import Script -from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute -from resource_management.libraries.functions.format import format -from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions import StackFeature -from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions.check_process_status import check_process_status -from resource_management.libraries.functions.show_logs import show_logs -from druid import druid, get_daemon_cmd, getPid - - -class DruidBase(Script): - def __init__(self, nodeType=None): - self.nodeType = nodeType - - def install(self, env): - self.install_packages(env) - - def configure(self, env, upgrade_type=None): - import params - env.set_params(params) - druid(upgrade_type=upgrade_type, nodeType=self.nodeType) - - def pre_upgrade_restart(self, env, upgrade_type=None): - node_type_lower = self.nodeType.lower() - Logger.info(format("Executing druid-{node_type_lower} Upgrade pre-restart")) - import params - - env.set_params(params) - - if params.stack_version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version): - stack_select.select_packages(params.stack_version) - - def start(self, env, upgrade_type=None): - import params - env.set_params(params) - self.configure(env, upgrade_type=upgrade_type) - daemon_cmd = get_daemon_cmd(params, self.nodeType, "start") - # Verify Database connection on Druid start - if params.metadata_storage_type == 'mysql': - if not params.jdbc_driver_jar or not os.path.isfile(params.connector_download_dir + os.path.sep + params.jdbc_driver_jar): - path_to_jdbc = params.connector_download_dir + os.path.sep + "*" - error_message = "Error! Sorry, but we can't find jdbc driver for mysql.So, db connection check can fail." + \ - "Please run 'ambari-server setup --jdbc-db=mysql --jdbc-driver={path_to_jdbc} on server host.'" - Logger.error(error_message) - else: - path_to_jdbc = params.connector_download_dir + os.path.sep + params.jdbc_driver_jar - db_connection_check_command = format("{params.java8_home}/bin/java -cp {params.check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{params.metadata_storage_url}' {params.metadata_storage_user} {params.metadata_storage_password!p} com.mysql.jdbc.Driver") - else: - db_connection_check_command = None - - if db_connection_check_command: - sudo.chmod(params.check_db_connection_jar, 0755) - Execute( db_connection_check_command, - tries=5, - try_sleep=10, - user=params.druid_user - ) - - try: - Execute(daemon_cmd, - user=params.druid_user - ) - except: - show_logs(params.druid_log_dir, params.druid_user) - raise - - def stop(self, env, upgrade_type=None): - import params - env.set_params(params) - - daemon_cmd = get_daemon_cmd(params, self.nodeType, "stop") - try: - Execute(daemon_cmd, - user=params.druid_user - ) - except: - show_logs(params.druid_log_dir, params.druid_user) - raise - - def status(self, env): - import status_params - env.set_params(status_params) - pid_file = getPid(status_params, self.nodeType) - check_process_status(pid_file) - - def get_log_folder(self): - import params - return params.druid_log_dir - - def get_user(self): - import params - return params.druid_user http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/historical.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/historical.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/historical.py deleted file mode 100644 index 22390a6..0000000 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/historical.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from druid_node import DruidBase - - -class DruidHistorical(DruidBase): - def __init__(self): - DruidBase.__init__(self, nodeType="historical") - - -if __name__ == "__main__": - DruidHistorical().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/middlemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/middlemanager.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/middlemanager.py deleted file mode 100644 index 20df89c..0000000 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/middlemanager.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from druid_node import DruidBase - - -class DruidMiddleManager(DruidBase): - def __init__(self): - DruidBase.__init__(self, nodeType="middleManager") - - -if __name__ == "__main__": - DruidMiddleManager().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/overlord.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/overlord.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/overlord.py deleted file mode 100644 index e4d7fcc..0000000 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/overlord.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from druid_node import DruidBase - - -class DruidOverlord(DruidBase): - def __init__(self): - DruidBase.__init__(self, nodeType="overlord") - - -if __name__ == "__main__": - DruidOverlord().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py deleted file mode 100644 index fd1cde6..0000000 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from ambari_commons import OSCheck -from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages -from resource_management.libraries.functions import conf_select -from resource_management.libraries.functions import stack_select -from resource_management.libraries.resources.hdfs_resource import HdfsResource -from resource_management.libraries.functions import get_kinit_path -from resource_management.libraries.script.script import Script -from resource_management.libraries.functions import format -from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources -from resource_management.libraries.functions.default import default -from ambari_commons.constants import AMBARI_SUDO_BINARY - -import status_params - -# a map of the Ambari role to the component name -# for use with <stack-root>/current/<component> -SERVER_ROLE_DIRECTORY_MAP = { - 'DRUID_BROKER': 'druid-broker', - 'DRUID_COORDINATOR': 'druid-coordinator', - 'DRUID_HISTORICAL': 'druid-historical', - 'DRUID_MIDDLEMANAGER': 'druid-middlemanager', - 'DRUID_OVERLORD': 'druid-overlord', - 'DRUID_ROUTER': 'druid-router' -} - -# server configurations -config = Script.get_config() -stack_root = Script.get_stack_root() -tmp_dir = Script.get_tmp_dir() - -stack_name = default("/hostLevelParams/stack_name", None) - -# stack version -stack_version = default("/commandParams/version", None) - -# un-formatted stack version -stack_version_unformatted = str(config['hostLevelParams']['stack_version']) - -# default role to coordinator needed for service checks -component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "DRUID_COORDINATOR") - -hostname = config['hostname'] -sudo = AMBARI_SUDO_BINARY - -# default druid parameters -druid_home = format("{stack_root}/current/{component_directory}") -druid_conf_dir = format("{stack_root}/current/{component_directory}/conf") - -druid_common_conf_dir = druid_conf_dir + "/_common" -druid_coordinator_conf_dir = druid_conf_dir + "/coordinator" -druid_overlord_conf_dir = druid_conf_dir + "/overlord" -druid_broker_conf_dir = druid_conf_dir + "/broker" -druid_historical_conf_dir = druid_conf_dir + "/historical" -druid_middlemanager_conf_dir = druid_conf_dir + "/middleManager" -druid_router_conf_dir = druid_conf_dir + "/router" -druid_extensions_dir = druid_home + "/extensions" -druid_hadoop_dependencies_dir = druid_home + "/hadoop-dependencies" -druid_segment_infoDir = config['configurations']['druid-historical']['druid.segmentCache.infoDir'] -druid_segment_cache_locations = config['configurations']['druid-historical']['druid.segmentCache.locations'] -druid_tasks_dir = config['configurations']['druid-middlemanager']['druid.indexer.task.baseTaskDir'] -druid_user = config['configurations']['druid-env']['druid_user'] -druid_log_dir = config['configurations']['druid-env']['druid_log_dir'] -druid_classpath = config['configurations']['druid-env']['druid_classpath'] -druid_extensions = config['configurations']['druid-common']['druid.extensions.pullList'] -druid_repo_list = config['configurations']['druid-common']['druid.extensions.repositoryList'] -druid_extensions_load_list = config['configurations']['druid-common']['druid.extensions.loadList'] -druid_security_extensions_load_list = config['configurations']['druid-common']['druid.security.extensions.loadList'] - - -# status params -druid_pid_dir = status_params.druid_pid_dir -user_group = config['configurations']['cluster-env']['user_group'] -java8_home = config['hostLevelParams']['java_home'] -druid_env_sh_template = config['configurations']['druid-env']['content'] - -# log4j params -log4j_props = config['configurations']['druid-log4j']['content'] -druid_log_level = config['configurations']['druid-log4j']['druid_log_level'] -metamx_log_level = config['configurations']['druid-log4j']['metamx_log_level'] -root_log_level = config['configurations']['druid-log4j']['root_log_level'] - -druid_log_maxbackupindex = default('/configurations/druid-logrotate/druid_log_maxbackupindex', 7) -druid_log_maxfilesize = default('/configurations/druid-logrotate/druid_log_maxfilesize', 256) -logrotate_props = config['configurations']['druid-logrotate']['content'] - -# Metadata storage -metadata_storage_user = config['configurations']['druid-common']['druid.metadata.storage.connector.user'] -metadata_storage_password = config['configurations']['druid-common']['druid.metadata.storage.connector.password'] -metadata_storage_db_name = config['configurations']['druid-common']['database_name'] -metadata_storage_db_name = config['configurations']['druid-common']['database_name'] -metadata_storage_type = config['configurations']['druid-common']['druid.metadata.storage.type'] -metadata_storage_url = config['configurations']['druid-common']['druid.metadata.storage.connector.connectURI'] -jdk_location = config['hostLevelParams']['jdk_location'] -if 'mysql' == metadata_storage_type: - jdbc_driver_jar = default("/hostLevelParams/custom_mysql_jdbc_name", None) - connector_curl_source = format("{jdk_location}/{jdbc_driver_jar}") - connector_download_dir=format("{druid_extensions_dir}/mysql-metadata-storage") - downloaded_custom_connector = format("{tmp_dir}/{jdbc_driver_jar}") - -check_db_connection_jar_name = "DBConnectionVerification.jar" -check_db_connection_jar = format("/usr/lib/ambari-agent/{check_db_connection_jar_name}") - -# HDFS -security_enabled = config['configurations']['cluster-env']['security_enabled'] -hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) -hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] -hadoop_bin_dir = stack_select.get_hadoop_dir("bin") -hadoop_conf_dir = conf_select.get_hadoop_conf_dir() -hdfs_principal_name = default('/configurations/hadoop-env/hdfs_principal_name', 'missing_principal').replace("_HOST", - hostname) -hdfs_site = config['configurations']['hdfs-site'] -default_fs = config['configurations']['core-site']['fs.defaultFS'] -dfs_type = default("/commandParams/dfs_type", "") -hdfs_tmp_dir = config['configurations']['hadoop-env']['hdfs_tmp_dir'] - -# Kerberos -druid_principal_name = default('/configurations/druid-common/druid.hadoop.security.kerberos.principal', - 'missing_principal') -druid_user_keytab = default('/configurations/druid-common/druid.hadoop.security.kerberos.keytab', 'missing_keytab') - -import functools - -# create partial functions with common arguments for every HdfsResource call -# to create hdfs directory we need to call params.HdfsResource in code -HdfsResource = functools.partial( - HdfsResource, - user=hdfs_user, - hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore", - security_enabled=security_enabled, - keytab=hdfs_user_keytab, - kinit_path_local=kinit_path_local, - hadoop_bin_dir=hadoop_bin_dir, - hadoop_conf_dir=hadoop_conf_dir, - principal_name=hdfs_principal_name, - hdfs_site=hdfs_site, - default_fs=default_fs, - immutable_paths=get_not_managed_resources(), - dfs_type=dfs_type -) - -# Ambari Metrics -metric_emitter_type = "noop" -metric_collector_host = "" -metric_collector_port = "" -metric_collector_protocol = "" -metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "") -metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "") -metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "") - -ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", []) -has_metric_collector = not len(ams_collector_hosts) == 0 - -if has_metric_collector: - metric_emitter_type = "ambari-metrics" - if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_vip_host' in config['configurations']['cluster-env']: - metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host'] - else: - metric_collector_host = ams_collector_hosts[0] - if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_vip_port' in config['configurations']['cluster-env']: - metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port'] - else: - metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "localhost:6188") - if metric_collector_web_address.find(':') != -1: - metric_collector_port = metric_collector_web_address.split(':')[1] - else: - metric_collector_port = '6188' - if default("/configurations/ams-site/timeline.metrics.service.http.policy", "HTTP_ONLY") == "HTTPS_ONLY": - metric_collector_protocol = 'https' - else: - metric_collector_protocol = 'http' - pass - -# Create current Hadoop Clients Libs -stack_version_unformatted = str(config['hostLevelParams']['stack_version']) -io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) -lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() -lzo_packages = get_lzo_packages(stack_version_unformatted) -hadoop_lib_home = stack_root + '/' + stack_version + '/hadoop/lib' http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/router.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/router.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/router.py deleted file mode 100644 index 1731a2a..0000000 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/router.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from druid_node import DruidBase - - -class DruidRouter(DruidBase): - def __init__(self): - DruidBase.__init__(self, nodeType="router") - - -if __name__ == "__main__": - DruidRouter().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/service_check.py deleted file mode 100644 index 139b727..0000000 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/service_check.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from resource_management.libraries.script.script import Script -from resource_management.libraries.functions.format import format -from resource_management.core.resources.system import Execute - - -class ServiceCheck(Script): - def service_check(self, env): - import params - env.set_params(params) - - self.checkComponent(params, "druid_coordinator", "druid-coordinator") - self.checkComponent(params, "druid_overlord", "druid-overlord") - - def checkComponent(self, params, component_name, config_name): - component_port = params.config['configurations'][format('{config_name}')]['druid.port'] - for component_host in params.config['clusterHostInfo'][format('{component_name}_hosts')]: - Execute(format( - "curl -s -o /dev/null -w'%{{http_code}}' --negotiate -u: -k {component_host}:{component_port}/status | grep 200"), - tries=10, - try_sleep=3, - logoutput=True) - - -if __name__ == "__main__": - ServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py deleted file mode 100644 index ee1d61c..0000000 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from resource_management.libraries.script.script import Script - -config = Script.get_config() - -druid_pid_dir = config['configurations']['druid-env']['druid_pid_dir'] http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json deleted file mode 100644 index c68b9b9..0000000 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "name": "default", - "description": "default quick links configuration", - "configuration": { - "protocol": { - "type": "HTTP_ONLY" - }, - "links": [ - { - "name": "coordinator_console", - "label": "Druid Coordinator Console", - "component_name": "DRUID_COORDINATOR", - "requires_user_name": "false", - "url": "%@://%@:%@", - "port": { - "http_property": "druid.port", - "http_default_port": "8081", - "regex": "^(\\d+)$", - "site": "druid-coordinator" - } - }, - { - "name": "overlord_console", - "label": "Druid Overlord Console", - "component_name": "DRUID_OVERLORD", - "requires_user_name": "false", - "url": "%@://%@:%@", - "port": { - "http_property": "druid.port", - "http_default_port": "8090", - "regex": "^(\\d+)$", - "site": "druid-overlord" - } - } - ] - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/role_command_order.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/role_command_order.json deleted file mode 100644 index 4d697fe..0000000 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/role_command_order.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "general_deps" : { - "_comment" : "dependencies for Druid", - "DRUID_HISTORICAL-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], - "DRUID_OVERLORD-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], - "DRUID_MIDDLEMANAGER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], - "DRUID_BROKER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], - "DRUID_ROUTER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], - "DRUID_COORDINATOR-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], - "DRUID_OVERLORD-RESTART" : ["DRUID_HISTORICAL-RESTART"], - "DRUID_MIDDLEMANAGER-RESTART" : ["DRUID_OVERLORD-RESTART"], - "DRUID_BROKER-RESTART" : ["DRUID_MIDDLEMANAGER-RESTART"], - "DRUID_ROUTER-RESTART" : ["DRUID_BROKER-RESTART"], - "DRUID_COORDINATOR-RESTART" : ["DRUID_ROUTER-RESTART"], - "DRUID_SERVICE_CHECK-SERVICE_CHECK" : ["DRUID_HISTORICAL-START", "DRUID_COORDINATOR-START", "DRUID_OVERLORD-START", "DRUID_MIDDLEMANAGER-START", "DRUID_BROKER-START", "DRUID_ROUTER-START"] - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json deleted file mode 100644 index 7033e19..0000000 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json +++ /dev/null @@ -1,120 +0,0 @@ -{ - "name": "default", - "description": "Default theme for Druid service", - "configuration": { - "layouts": [ - { - "name": "default", - "tabs": [ - { - "name": "metadata_storage", - "display-name": "META DATA STORAGE CONFIG", - "layout": { - "tab-columns": "", - "tab-rows": "1", - "sections": [ - { - "name": "section-metadata-storage", - "display-name": "", - "row-index": "0", - "column-index": "0", - "row-span": "2", - "column-span": "1", - "section-columns": "1", - "section-rows": "1", - "subsections": [ - { - "name": "subsection-metadata-storage-row1-col1", - "display-name": "DRUID META DATA STORAGE", - "row-index": "0", - "column-index": "0", - "row-span": "1", - "column-span": "1" - } - ] - } - ] - } - } - ] - } - ], - "placement": { - "configuration-layout": "default", - "configs": [ - { - "config": "druid-common/database_name", - "subsection-name": "subsection-metadata-storage-row1-col1" - }, - { - "config": "druid-common/druid.metadata.storage.type", - "subsection-name": "subsection-metadata-storage-row1-col1" - }, - { - "config": "druid-common/druid.metadata.storage.connector.user", - "subsection-name": "subsection-metadata-storage-row1-col1" - }, - { - "config": "druid-common/druid.metadata.storage.connector.password", - "subsection-name": "subsection-metadata-storage-row1-col1" - }, - { - "config": "druid-common/metastore_hostname", - "subsection-name": "subsection-metadata-storage-row1-col1" - }, - { - "config": "druid-common/druid.metadata.storage.connector.port", - "subsection-name": "subsection-metadata-storage-row1-col1" - }, - { - "config": "druid-common/druid.metadata.storage.connector.connectURI", - "subsection-name": "subsection-metadata-storage-row1-col1" - } - ] - }, - "widgets": [ - { - "config": "druid-common/database_name", - "widget": { - "type": "text-field" - } - }, - { - "config": "druid-common/druid.metadata.storage.type", - "widget": { - "type": "combo" - } - }, - { - "config": "druid-common/druid.metadata.storage.connector.user", - "widget": { - "type": "text-field" - } - }, - { - "config": "druid-common/druid.metadata.storage.connector.password", - "widget": { - "type": "password" - } - }, - { - "config": "druid-common/metastore_hostname", - "widget": { - "type": "text-field" - } - }, - { - "config": "druid-common/druid.metadata.storage.connector.port", - "widget": { - "type": "text-field" - } - }, - { - "config": "druid-common/druid.metadata.storage.connector.connectURI", - "widget": { - "type": "text-field" - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml index f1f099e..07dd6d6 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml @@ -20,9 +20,8 @@ <services> <service> <name>DRUID</name> - <version>0.9.2</version> - <extends>common-services/DRUID/0.9.2</extends> - <selection>TECH_PREVIEW</selection> + <version>0.10.1</version> + <extends>common-services/DRUID/0.10.1</extends> </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py index d8b9fe2..80e9d54 100644 --- a/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py +++ b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py @@ -31,7 +31,7 @@ from resource_management.core.logger import Logger @patch("resource_management.libraries.Script.get_tmp_dir", new=MagicMock(return_value=('/var/lib/ambari-agent/tmp'))) @patch.object(functions, "get_stack_version", new=MagicMock(return_value="2.0.0.0-1234")) class TestDruid(RMFTestCase): - COMMON_SERVICES_PACKAGE_DIR = "DRUID/0.9.2/package" + COMMON_SERVICES_PACKAGE_DIR = "DRUID/0.10.1/package" STACK_VERSION = "2.6" DEFAULT_IMMUTABLE_PATHS = ['/apps/hive/warehouse', '/apps/falcon', '/mr-history/done', '/app-logs', '/tmp']
