http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/files/task-log4j.properties ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/files/task-log4j.properties b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/files/task-log4j.properties new file mode 100755 index 0000000..7e12962 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/files/task-log4j.properties @@ -0,0 +1,134 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + + +# Define some default values that can be overridden by system properties +hadoop.root.logger=INFO,console +hadoop.log.dir=. +hadoop.log.file=hadoop.log + +# +# Job Summary Appender +# +# Use following logger to send summary to separate file defined by +# hadoop.mapreduce.jobsummary.log.file rolled daily: +# hadoop.mapreduce.jobsummary.logger=INFO,JSA +# +hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} +hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log + +# Define the root logger to the system property "hadoop.root.logger". +log4j.rootLogger=${hadoop.root.logger}, EventCounter + +# Logging Threshold +log4j.threshhold=ALL + +# +# Daily Rolling File Appender +# + +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +# +# TaskLog Appender +# + +#Default values +hadoop.tasklog.taskid=null +hadoop.tasklog.iscleanup=false +hadoop.tasklog.noKeepSplits=4 +hadoop.tasklog.totalLogFileSize=100 +hadoop.tasklog.purgeLogSplits=true +hadoop.tasklog.logsRetainHours=12 + +log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender +log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} +log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} +log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} + +log4j.appender.TLA.layout=org.apache.log4j.PatternLayout +log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n + +# +# Rolling File Appender +# + +#log4j.appender.RFA=org.apache.log4j.RollingFileAppender +#log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} + +# Logfile size and and 30-day backups +#log4j.appender.RFA.MaxFileSize=1MB +#log4j.appender.RFA.MaxBackupIndex=30 + +#log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n +#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# Custom Logging levels + +hadoop.metrics.log.level=INFO +#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG +#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG +log4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level} + +# Jets3t library +log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR + +# +# Null Appender +# Trap security logger on the hadoop client side +# +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# Event Counter Appender +# Sends counts of logging messages at different severity levels to Hadoop Metrics. +# +log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter + +# Removes "deprecated" messages +log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN
http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/files/topology_script.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/files/topology_script.py b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/files/topology_script.py new file mode 100755 index 0000000..0f7a55c --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/files/topology_script.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import sys, os +from string import join +import ConfigParser + + +DEFAULT_RACK = "/default-rack" +DATA_FILE_NAME = os.path.dirname(os.path.abspath(__file__)) + "/topology_mappings.data" +SECTION_NAME = "network_topology" + +class TopologyScript(): + + def load_rack_map(self): + try: + #RACK_MAP contains both host name vs rack and ip vs rack mappings + mappings = ConfigParser.ConfigParser() + mappings.read(DATA_FILE_NAME) + return dict(mappings.items(SECTION_NAME)) + except ConfigParser.NoSectionError: + return {} + + def get_racks(self, rack_map, args): + if len(args) == 1: + return DEFAULT_RACK + else: + return join([self.lookup_by_hostname_or_ip(input_argument, rack_map) for input_argument in args[1:]],) + + def lookup_by_hostname_or_ip(self, hostname_or_ip, rack_map): + #try looking up by hostname + rack = rack_map.get(hostname_or_ip) + if rack is not None: + return rack + #try looking up by ip + rack = rack_map.get(self.extract_ip(hostname_or_ip)) + #try by localhost since hadoop could be passing in 127.0.0.1 which might not be mapped + return rack if rack is not None else rack_map.get("localhost.localdomain", DEFAULT_RACK) + + #strips out port and slashes in case hadoop passes in something like 127.0.0.1/127.0.0.1:50010 + def extract_ip(self, container_string): + return container_string.split("/")[0].split(":")[0] + + def execute(self, args): + rack_map = self.load_rack_map() + rack = self.get_racks(rack_map, args) + print rack + +if __name__ == "__main__": + TopologyScript().execute(sys.argv) http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/hook.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/hook.py b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/hook.py new file mode 100755 index 0000000..f21e4b1 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/hook.py @@ -0,0 +1,39 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * +from rack_awareness import create_topology_script_and_mapping +from shared_initialization import setup_hadoop, setup_configs, create_javahome_symlink + +class BeforeStartHook(Hook): + + def hook(self, env): + import params + + self.run_custom_hook('before-ANY') + env.set_params(params) + + setup_hadoop() + setup_configs() + create_javahome_symlink() + create_topology_script_and_mapping() + +if __name__ == "__main__": + BeforeStartHook().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/params.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/params.py b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/params.py new file mode 100755 index 0000000..fc2c61f --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/params.py @@ -0,0 +1,318 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os + +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import default +from resource_management.libraries.functions import format_jvm_option +from resource_management.libraries.functions import format +from resource_management.libraries.functions.version import format_stack_version, compare_versions +from ambari_commons.os_check import OSCheck +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources +from resource_management.libraries.resources.hdfs_resource import HdfsResource + +config = Script.get_config() + +host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) + +stack_version_unformatted = config['hostLevelParams']['stack_version'] +stack_version_formatted = format_stack_version(stack_version_unformatted) + +dfs_type = default("/commandParams/dfs_type", "") +hadoop_conf_dir = "/etc/hadoop/conf" + +component_list = default("/localComponents", []) + +hdfs_tmp_dir = config['configurations']['hadoop-env']['hdfs_tmp_dir'] + +# hadoop default params +mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*" + +hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec") +hadoop_lib_home = stack_select.get_hadoop_dir("lib") +hadoop_bin = stack_select.get_hadoop_dir("sbin") +hadoop_home = '/usr' +create_lib_snappy_symlinks = True + +# HDP 2.2+ params +if Script.is_stack_greater_or_equal("2.2"): + mapreduce_libs_path = "/usr/hdp/current/hadoop-mapreduce-client/*" + hadoop_home = stack_select.get_hadoop_dir("home") + create_lib_snappy_symlinks = False + +current_service = config['serviceName'] + +#security params +security_enabled = config['configurations']['cluster-env']['security_enabled'] + +#users and groups +has_hadoop_env = 'hadoop-env' in config['configurations'] +mapred_user = config['configurations']['mapred-env']['mapred_user'] +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +yarn_user = config['configurations']['yarn-env']['yarn_user'] + +user_group = config['configurations']['cluster-env']['user_group'] + +#hosts +hostname = config["hostname"] +ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] +rm_host = default("/clusterHostInfo/rm_host", []) +slave_hosts = default("/clusterHostInfo/slave_hosts", []) +oozie_servers = default("/clusterHostInfo/oozie_server", []) +hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", []) +hive_server_host = default("/clusterHostInfo/hive_server_host", []) +hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", []) +hs_host = default("/clusterHostInfo/hs_host", []) +jtnode_host = default("/clusterHostInfo/jtnode_host", []) +namenode_host = default("/clusterHostInfo/namenode_host", []) +zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) +ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", []) +ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", []) + +has_namenode = not len(namenode_host) == 0 +has_resourcemanager = not len(rm_host) == 0 +has_slaves = not len(slave_hosts) == 0 +has_oozie_server = not len(oozie_servers) == 0 +has_hcat_server_host = not len(hcat_server_hosts) == 0 +has_hive_server_host = not len(hive_server_host) == 0 +has_hbase_masters = not len(hbase_master_hosts) == 0 +has_zk_host = not len(zk_hosts) == 0 +has_ganglia_server = not len(ganglia_server_hosts) == 0 +has_metric_collector = not len(ams_collector_hosts) == 0 + +is_namenode_master = hostname in namenode_host +is_jtnode_master = hostname in jtnode_host +is_rmnode_master = hostname in rm_host +is_hsnode_master = hostname in hs_host +is_hbase_master = hostname in hbase_master_hosts +is_slave = hostname in slave_hosts +if has_ganglia_server: + ganglia_server_host = ganglia_server_hosts[0] +if has_metric_collector: + if 'cluster-env' in config['configurations'] and \ + 'metrics_collector_vip_host' in config['configurations']['cluster-env']: + metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host'] + else: + metric_collector_host = ams_collector_hosts[0] + if 'cluster-env' in config['configurations'] and \ + 'metrics_collector_vip_port' in config['configurations']['cluster-env']: + metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port'] + else: + metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "localhost:6188") + if metric_collector_web_address.find(':') != -1: + metric_collector_port = metric_collector_web_address.split(':')[1] + else: + metric_collector_port = '6188' + if default("/configurations/ams-site/timeline.metrics.service.http.policy", "HTTP_ONLY") == "HTTPS_ONLY": + metric_collector_protocol = 'https' + else: + metric_collector_protocol = 'http' + metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "") + metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "") + metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "") + + pass +metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60) +metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10) + +#hadoop params + +if has_namenode or dfs_type == 'HCFS': + hadoop_tmp_dir = format("/tmp/hadoop-{hdfs_user}") + hadoop_conf_dir = conf_select.get_hadoop_conf_dir(force_latest_on_upgrade=True) + task_log4j_properties_location = os.path.join(hadoop_conf_dir, "task-log4j.properties") + +hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix'] +hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] +hbase_tmp_dir = "/tmp/hbase-hbase" +#db params +server_db_name = config['hostLevelParams']['db_name'] +db_driver_filename = config['hostLevelParams']['db_driver_filename'] +oracle_driver_url = config['hostLevelParams']['oracle_jdbc_url'] +mysql_driver_url = config['hostLevelParams']['mysql_jdbc_url'] +ambari_server_resources = config['hostLevelParams']['jdk_location'] +oracle_driver_symlink_url = format("{ambari_server_resources}oracle-jdbc-driver.jar") +mysql_driver_symlink_url = format("{ambari_server_resources}mysql-jdbc-driver.jar") + +ambari_db_rca_url = config['hostLevelParams']['ambari_db_rca_url'][0] +ambari_db_rca_driver = config['hostLevelParams']['ambari_db_rca_driver'][0] +ambari_db_rca_username = config['hostLevelParams']['ambari_db_rca_username'][0] +ambari_db_rca_password = config['hostLevelParams']['ambari_db_rca_password'][0] + +if has_namenode and 'rca_enabled' in config['configurations']['hadoop-env']: + rca_enabled = config['configurations']['hadoop-env']['rca_enabled'] +else: + rca_enabled = False +rca_disabled_prefix = "###" +if rca_enabled == True: + rca_prefix = "" +else: + rca_prefix = rca_disabled_prefix + +#hadoop-env.sh +java_home = config['hostLevelParams']['java_home'] + +jsvc_path = "/usr/lib/bigtop-utils" + +hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize'] +namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize'] +namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize'] +namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize'] +namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m") +namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m") + +jtnode_opt_newsize = "200m" +jtnode_opt_maxnewsize = "200m" +jtnode_heapsize = "1024m" +ttnode_heapsize = "1024m" + +dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize'] +mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce") +mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce") + +#log4j.properties + +yarn_log_dir_prefix = default("/configurations/yarn-env/yarn_log_dir_prefix","/var/log/hadoop-yarn") + +dfs_hosts = default('/configurations/hdfs-site/dfs.hosts', None) + +#log4j.properties +if (('hdfs-log4j' in config['configurations']) and ('content' in config['configurations']['hdfs-log4j'])): + log4j_props = config['configurations']['hdfs-log4j']['content'] + if (('yarn-log4j' in config['configurations']) and ('content' in config['configurations']['yarn-log4j'])): + log4j_props += config['configurations']['yarn-log4j']['content'] +else: + log4j_props = None + +refresh_topology = False +command_params = config["commandParams"] if "commandParams" in config else None +if command_params is not None: + refresh_topology = bool(command_params["refresh_topology"]) if "refresh_topology" in command_params else False + +ambari_libs_dir = "/var/lib/ambari-agent/lib" +is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled'] +default_fs = config['configurations']['core-site']['fs.defaultFS'] + +#host info +all_hosts = default("/clusterHostInfo/all_hosts", []) +all_racks = default("/clusterHostInfo/all_racks", []) +all_ipv4_ips = default("/clusterHostInfo/all_ipv4_ips", []) +slave_hosts = default("/clusterHostInfo/slave_hosts", []) + +#topology files +net_topology_script_file_path = "/etc/hadoop/conf/topology_script.py" +net_topology_script_dir = os.path.dirname(net_topology_script_file_path) +net_topology_mapping_data_file_name = 'topology_mappings.data' +net_topology_mapping_data_file_path = os.path.join(net_topology_script_dir, net_topology_mapping_data_file_name) + +#Added logic to create /tmp and /user directory for HCFS stack. +has_core_site = 'core-site' in config['configurations'] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +kinit_path_local = get_kinit_path() +stack_version_unformatted = config['hostLevelParams']['stack_version'] +stack_version_formatted = format_stack_version(stack_version_unformatted) +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") +hdfs_principal_name = default('/configurations/hadoop-env/hdfs_principal_name', None) +hdfs_site = config['configurations']['hdfs-site'] +default_fs = config['configurations']['core-site']['fs.defaultFS'] +smoke_user = config['configurations']['cluster-env']['smokeuser'] +smoke_hdfs_user_dir = format("/user/{smoke_user}") +smoke_hdfs_user_mode = 0770 + + +##### Namenode RPC ports - metrics config section start ##### + +# Figure out the rpc ports for current namenode +nn_rpc_client_port = None +nn_rpc_dn_port = None +nn_rpc_healthcheck_port = None + +namenode_id = None +namenode_rpc = None + +dfs_ha_enabled = False +dfs_ha_nameservices = default('/configurations/hdfs-site/dfs.internal.nameservices', None) +if dfs_ha_nameservices is None: + dfs_ha_nameservices = default('/configurations/hdfs-site/dfs.nameservices', None) +dfs_ha_namenode_ids = default(format("/configurations/hdfs-site/dfs.ha.namenodes.{dfs_ha_nameservices}"), None) + +dfs_ha_namemodes_ids_list = [] +other_namenode_id = None + +if dfs_ha_namenode_ids: + dfs_ha_namemodes_ids_list = dfs_ha_namenode_ids.split(",") + dfs_ha_namenode_ids_array_len = len(dfs_ha_namemodes_ids_list) + if dfs_ha_namenode_ids_array_len > 1: + dfs_ha_enabled = True + +if dfs_ha_enabled: + for nn_id in dfs_ha_namemodes_ids_list: + nn_host = config['configurations']['hdfs-site'][format('dfs.namenode.rpc-address.{dfs_ha_nameservices}.{nn_id}')] + if hostname in nn_host: + namenode_id = nn_id + namenode_rpc = nn_host + pass + pass +else: + namenode_rpc = default('/configurations/hdfs-site/dfs.namenode.rpc-address', None) + +if namenode_rpc: + nn_rpc_client_port = namenode_rpc.split(':')[1].strip() + +if dfs_ha_enabled: + dfs_service_rpc_address = default(format('/configurations/hdfs-site/dfs.namenode.servicerpc-address.{dfs_ha_nameservices}.{namenode_id}'), None) + dfs_lifeline_rpc_address = default(format('/configurations/hdfs-site/dfs.namenode.lifeline.rpc-address.{dfs_ha_nameservices}.{namenode_id}'), None) +else: + dfs_service_rpc_address = default('/configurations/hdfs-site/dfs.namenode.servicerpc-address', None) + dfs_lifeline_rpc_address = default(format('/configurations/hdfs-site/dfs.namenode.lifeline.rpc-address'), None) + +if dfs_service_rpc_address: + nn_rpc_dn_port = dfs_service_rpc_address.split(':')[1].strip() + +if dfs_lifeline_rpc_address: + nn_rpc_healthcheck_port = dfs_lifeline_rpc_address.split(':')[1].strip() + +is_nn_client_port_configured = False if nn_rpc_client_port is None else True +is_nn_dn_port_configured = False if nn_rpc_dn_port is None else True +is_nn_healthcheck_port_configured = False if nn_rpc_healthcheck_port is None else True + +##### end ##### + +import functools +#create partial functions with common arguments for every HdfsResource call +#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_principal_name, + hdfs_site = hdfs_site, + default_fs = default_fs, + immutable_paths = get_not_managed_resources(), + dfs_type = dfs_type +) http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/rack_awareness.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/rack_awareness.py b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/rack_awareness.py new file mode 100755 index 0000000..548f051 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/rack_awareness.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from resource_management.core.resources import File +from resource_management.core.source import StaticFile, Template +from resource_management.libraries.functions import format + + +def create_topology_mapping(): + import params + + File(params.net_topology_mapping_data_file_path, + content=Template("topology_mappings.data.j2"), + owner=params.hdfs_user, + group=params.user_group, + only_if=format("test -d {net_topology_script_dir}")) + +def create_topology_script(): + import params + + File(params.net_topology_script_file_path, + content=StaticFile('topology_script.py'), + mode=0755, + only_if=format("test -d {net_topology_script_dir}")) + +def create_topology_script_and_mapping(): + import params + if params.has_hadoop_env: + create_topology_mapping() + create_topology_script() http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/shared_initialization.py b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/shared_initialization.py new file mode 100755 index 0000000..ba9c8fb --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/shared_initialization.py @@ -0,0 +1,175 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os +from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil + +from resource_management import * + +def setup_hadoop(): + """ + Setup hadoop files and directories + """ + import params + + Execute(("setenforce","0"), + only_if="test -f /selinux/enforce", + not_if="(! which getenforce ) || (which getenforce && getenforce | grep -q Disabled)", + sudo=True, + ) + + #directories + if params.has_namenode or params.dfs_type == 'HCFS': + Directory(params.hdfs_log_dir_prefix, + create_parents = True, + owner='root', + group=params.user_group, + mode=0775, + cd_access='a', + ) + if params.has_namenode: + Directory(params.hadoop_pid_dir_prefix, + create_parents = True, + owner='root', + group='root', + cd_access='a', + ) + Directory(params.hadoop_tmp_dir, + create_parents = True, + owner=params.hdfs_user, + cd_access='a', + ) + #files + if params.security_enabled: + tc_owner = "root" + else: + tc_owner = params.hdfs_user + + # if WebHDFS is not enabled we need this jar to create hadoop folders. + if params.host_sys_prepped: + print "Skipping copying of fast-hdfs-resource.jar as host is sys prepped" + elif params.dfs_type == 'HCFS' or not WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs): + # for source-code of jar goto contrib/fast-hdfs-resource + File(format("{ambari_libs_dir}/fast-hdfs-resource.jar"), + mode=0644, + content=StaticFile("fast-hdfs-resource.jar") + ) + + if os.path.exists(params.hadoop_conf_dir): + File(os.path.join(params.hadoop_conf_dir, 'commons-logging.properties'), + owner=tc_owner, + content=Template('commons-logging.properties.j2') + ) + + health_check_template_name = "health_check" + File(os.path.join(params.hadoop_conf_dir, health_check_template_name), + owner=tc_owner, + content=Template(health_check_template_name + ".j2") + ) + + log4j_filename = os.path.join(params.hadoop_conf_dir, "log4j.properties") + if (params.log4j_props != None): + File(log4j_filename, + mode=0644, + group=params.user_group, + owner=params.hdfs_user, + content=params.log4j_props + ) + elif (os.path.exists(format("{params.hadoop_conf_dir}/log4j.properties"))): + File(log4j_filename, + mode=0644, + group=params.user_group, + owner=params.hdfs_user, + ) + + File(os.path.join(params.hadoop_conf_dir, "hadoop-metrics2.properties"), + owner=params.hdfs_user, + group=params.user_group, + content=Template("hadoop-metrics2.properties.j2") + ) + + if params.dfs_type == 'HCFS' and params.has_core_site and 'ECS_CLIENT' in params.component_list: + create_dirs() + + +def setup_configs(): + """ + Creates configs for services HDFS mapred + """ + import params + + if params.has_namenode or params.dfs_type == 'HCFS': + if os.path.exists(params.hadoop_conf_dir): + File(params.task_log4j_properties_location, + content=StaticFile("task-log4j.properties"), + mode=0755 + ) + + if os.path.exists(os.path.join(params.hadoop_conf_dir, 'configuration.xsl')): + File(os.path.join(params.hadoop_conf_dir, 'configuration.xsl'), + owner=params.hdfs_user, + group=params.user_group + ) + if os.path.exists(os.path.join(params.hadoop_conf_dir, 'masters')): + File(os.path.join(params.hadoop_conf_dir, 'masters'), + owner=params.hdfs_user, + group=params.user_group + ) + + generate_include_file() + + +def generate_include_file(): + import params + + if params.has_namenode and params.dfs_hosts and params.has_slaves: + include_hosts_list = params.slave_hosts + File(params.dfs_hosts, + content=Template("include_hosts_list.j2"), + owner=params.hdfs_user, + group=params.user_group + ) + +def create_javahome_symlink(): + if os.path.exists("/usr/jdk/jdk1.6.0_31") and not os.path.exists("/usr/jdk64/jdk1.6.0_31"): + Directory("/usr/jdk64/", + create_parents = True, + ) + Link("/usr/jdk/jdk1.6.0_31", + to="/usr/jdk64/jdk1.6.0_31", + ) + +def create_dirs(): + import params + params.HdfsResource(params.hdfs_tmp_dir, + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + mode=0777 + ) + params.HdfsResource(params.smoke_hdfs_user_dir, + type="directory", + action="create_on_execute", + owner=params.smoke_user, + mode=params.smoke_hdfs_user_mode + ) + params.HdfsResource(None, + action="execute" + ) + http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/commons-logging.properties.j2 ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/commons-logging.properties.j2 b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/commons-logging.properties.j2 new file mode 100755 index 0000000..2197ba5 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/commons-logging.properties.j2 @@ -0,0 +1,43 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +#/* +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +#Logging Implementation + +#Log4J +org.apache.commons.logging.Log=org.apache.commons.logging.impl.Log4JLogger + +#JDK Logger +#org.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/exclude_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/exclude_hosts_list.j2 b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/exclude_hosts_list.j2 new file mode 100755 index 0000000..1adba80 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/exclude_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in hdfs_exclude_file %} +{{host}} +{% endfor %} http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 new file mode 100755 index 0000000..fcd9b23 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 @@ -0,0 +1,104 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# syntax: [prefix].[source|sink|jmx].[instance].[options] +# See package.html for org.apache.hadoop.metrics2 for details + +{% if has_ganglia_server %} +*.period=60 + +*.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31 +*.sink.ganglia.period=10 + +# default for supportsparse is false +*.sink.ganglia.supportsparse=true + +.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both +.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40 + +# Hook up to the server +namenode.sink.ganglia.servers={{ganglia_server_host}}:8661 +datanode.sink.ganglia.servers={{ganglia_server_host}}:8659 +jobtracker.sink.ganglia.servers={{ganglia_server_host}}:8662 +tasktracker.sink.ganglia.servers={{ganglia_server_host}}:8658 +maptask.sink.ganglia.servers={{ganglia_server_host}}:8660 +reducetask.sink.ganglia.servers={{ganglia_server_host}}:8660 +resourcemanager.sink.ganglia.servers={{ganglia_server_host}}:8664 +nodemanager.sink.ganglia.servers={{ganglia_server_host}}:8657 +historyserver.sink.ganglia.servers={{ganglia_server_host}}:8666 +journalnode.sink.ganglia.servers={{ganglia_server_host}}:8654 +nimbus.sink.ganglia.servers={{ganglia_server_host}}:8649 +supervisor.sink.ganglia.servers={{ganglia_server_host}}:8650 + +resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue + +{% endif %} + +{% if has_metric_collector %} + +*.period={{metrics_collection_period}} +*.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar +*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +*.sink.timeline.period={{metrics_collection_period}} +*.sink.timeline.sendInterval={{metrics_report_interval}}000 +*.sink.timeline.slave.host.name = {{hostname}} + +# HTTPS properties +*.sink.timeline.truststore.path = {{metric_truststore_path}} +*.sink.timeline.truststore.type = {{metric_truststore_type}} +*.sink.timeline.truststore.password = {{metric_truststore_password}} + +datanode.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}} +namenode.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}} +resourcemanager.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}} +nodemanager.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}} +jobhistoryserver.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}} +journalnode.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}} +maptask.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}} +reducetask.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}} +applicationhistoryserver.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}} + +resourcemanager.sink.timeline.tagsForPrefix.yarn=Queue + +{% if is_nn_client_port_configured %} +# Namenode rpc ports customization +namenode.sink.timeline.metric.rpc.client.port={{nn_rpc_client_port}} +{% endif %} +{% if is_nn_dn_port_configured %} +namenode.sink.timeline.metric.rpc.datanode.port={{nn_rpc_dn_port}} +{% endif %} +{% if is_nn_healthcheck_port_configured %} +namenode.sink.timeline.metric.rpc.healthcheck.port={{nn_rpc_healthcheck_port}} +{% endif %} + +{% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/health_check.j2 ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/health_check.j2 b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/health_check.j2 new file mode 100755 index 0000000..0a03d17 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/health_check.j2 @@ -0,0 +1,81 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +#!/bin/bash +# +#/* +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +err=0; + +function check_disks { + + for m in `awk '$3~/ext3/ {printf" %s ",$2}' /etc/fstab` ; do + fsdev="" + fsdev=`awk -v m=$m '$2==m {print $1}' /proc/mounts`; + if [ -z "$fsdev" -a "$m" != "/mnt" ] ; then + msg_="$msg_ $m(u)" + else + msg_="$msg_`awk -v m=$m '$2==m { if ( $4 ~ /^ro,/ ) {printf"%s(ro)",$2 } ; }' /proc/mounts`" + fi + done + + if [ -z "$msg_" ] ; then + echo "disks ok" ; exit 0 + else + echo "$msg_" ; exit 2 + fi + +} + +# Run all checks +for check in disks ; do + msg=`check_${check}` ; + if [ $? -eq 0 ] ; then + ok_msg="$ok_msg$msg," + else + err_msg="$err_msg$msg," + fi +done + +if [ ! -z "$err_msg" ] ; then + echo -n "ERROR $err_msg " +fi +if [ ! -z "$ok_msg" ] ; then + echo -n "OK: $ok_msg" +fi + +echo + +# Success! +exit 0 http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/include_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/include_hosts_list.j2 b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/include_hosts_list.j2 new file mode 100755 index 0000000..4a9e713 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/include_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in slave_hosts %} +{{host}} +{% endfor %} http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/topology_mappings.data.j2 ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/topology_mappings.data.j2 b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/topology_mappings.data.j2 new file mode 100755 index 0000000..15034d6 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/templates/topology_mappings.data.j2 @@ -0,0 +1,24 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + # +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +[network_topology] +{% for host in all_hosts %} +{% if host in slave_hosts %} +{{host}}={{all_racks[loop.index-1]}} +{{all_ipv4_ips[loop.index-1]}}={{all_racks[loop.index-1]}} +{% endif %} +{% endfor %} http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/kerberos.json ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/kerberos.json b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/kerberos.json new file mode 100644 index 0000000..3aad080 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/kerberos.json @@ -0,0 +1,60 @@ +{ + "properties": { + "realm": "${kerberos-env/realm}", + "keytab_dir": "/etc/security/keytabs", + "additional_realms": "" + }, + "identities": [ + { + "name": "spnego", + "principal": { + "value": "HTTP/_HOST@${realm}", + "type" : "service" + }, + "keytab": { + "file": "${keytab_dir}/spnego.service.keytab", + "owner": { + "name": "root", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "r" + } + } + }, + { + "name": "smokeuser", + "principal": { + "value": "${cluster-env/smokeuser}-${cluster_name|toLower()}@${realm}", + "type" : "user", + "configuration": "cluster-env/smokeuser_principal_name", + "local_username" : "${cluster-env/smokeuser}" + }, + "keytab": { + "file": "${keytab_dir}/smokeuser.headless.keytab", + "owner": { + "name": "${cluster-env/smokeuser}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "r" + }, + "configuration": "cluster-env/smokeuser_keytab" + } + }, + { + "name": "ambari-server", + "principal": { + "value": "ambari-server-${cluster_name|toLower()}@${realm}", + "type" : "user", + "configuration": "cluster-env/ambari_principal_name" + }, + "keytab": { + "file": "${keytab_dir}/ambari.server.keytab" + } + } + ] + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/metainfo.xml b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/metainfo.xml new file mode 100755 index 0000000..ca45822 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/metainfo.xml @@ -0,0 +1,22 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <versions> + <active>true</active> + </versions> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/properties/stack_features.json b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/properties/stack_features.json new file mode 100755 index 0000000..0c3e305 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/properties/stack_features.json @@ -0,0 +1,51 @@ +{ + "stack_features": [ + { + "name": "snappy", + "description": "Snappy compressor/decompressor support", + "min_version": "2.0.0.0", + "max_version": "2.2.0.0" + }, + { + "name": "lzo", + "description": "LZO libraries support", + "min_version": "2.2.1.0" + }, + { + "name": "copy_tarball_to_hdfs", + "description": "Copy tarball to HDFS support (AMBARI-12113)", + "min_version": "2.2.0.0" + }, + { + "name": "hive_metastore_upgrade_schema", + "description": "Hive metastore upgrade schema support (AMBARI-11176)", + "min_version": "2.3.0.0" + }, + { + "name": "hive_webhcat_specific_configs", + "description": "Hive webhcat specific configurations support (AMBARI-12364)", + "min_version": "2.3.0.0" + }, + { + "name": "hive_purge_table", + "description": "Hive purge table support (AMBARI-12260)", + "min_version": "2.3.0.0" + }, + { + "name": "hive_server2_kerberized_env", + "description": "Hive server2 working on kerberized environment (AMBARI-13749)", + "min_version": "2.2.3.0", + "max_version": "2.2.5.0" + }, + { + "name": "hive_env_heapsize", + "description": "Hive heapsize property defined in hive-env (AMBARI-12801)", + "min_version": "2.2.0.0" + }, + { + "name": "hive_metastore_site_support", + "description": "Hive Metastore site support", + "min_version": "2.5.0.0" + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/properties/stack_tools.json ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/properties/stack_tools.json b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/properties/stack_tools.json new file mode 100755 index 0000000..c3df235 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/properties/stack_tools.json @@ -0,0 +1,4 @@ +{ + "stack_selector": ["distro-select", "/usr/bin/distro-select", "distro-select"], + "conf_selector": ["conf-select", "/usr/bin/conf-select", "conf-select"] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/repos/repoinfo.xml b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/repos/repoinfo.xml new file mode 100755 index 0000000..06a16b0 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/repos/repoinfo.xml @@ -0,0 +1,33 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<reposinfo> + <os family="redhat6"> + <repo> + <baseurl>http://repo.odpi.org/ODPi/2.0/centos-6/</baseurl> + <repoid>ODPi-2.0</repoid> + <reponame>ODPi</reponame> + </repo> + </os> + <os family="ubuntu14"> + <repo> + <baseurl>http://repo.odpi.org/ODPi/2.0/ubuntu-14.04/apt</baseurl> + <repoid>ODPi-2.0</repoid> + <reponame>odpi</reponame> + </repo> + </os> +</reposinfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/role_command_order.json ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/role_command_order.json b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/role_command_order.json new file mode 100755 index 0000000..05beb76c --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/role_command_order.json @@ -0,0 +1,75 @@ +{ + "_comment" : "Record format:", + "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", + "general_deps" : { + "_comment" : "dependencies for all cases", + "HBASE_MASTER-START": ["ZOOKEEPER_SERVER-START"], + "HBASE_REGIONSERVER-START": ["HBASE_MASTER-START"], + "APP_TIMELINE_SERVER-START": ["NAMENODE-START", "DATANODE-START"], + "OOZIE_SERVER-START": ["NODEMANAGER-START", "RESOURCEMANAGER-START"], + "WEBHCAT_SERVER-START": ["NODEMANAGER-START", "HIVE_SERVER-START"], + "WEBHCAT_SERVER-RESTART": ["NODEMANAGER-RESTART", "HIVE_SERVER-RESTART"], + "HIVE_METASTORE-START": ["MYSQL_SERVER-START", "NAMENODE-START"], + "HIVE_METASTORE-RESTART": ["MYSQL_SERVER-RESTART", "NAMENODE-RESTART"], + "HIVE_SERVER-START": ["NODEMANAGER-START", "MYSQL_SERVER-START"], + "HIVE_SERVER-RESTART": ["NODEMANAGER-RESTART", "MYSQL_SERVER-RESTART", "ZOOKEEPER_SERVER-RESTART"], + "HUE_SERVER-START": ["HIVE_SERVER-START", "HCAT-START", "OOZIE_SERVER-START"], + "FLUME_HANDLER-START": ["OOZIE_SERVER-START"], + "MAPREDUCE_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"], + "OOZIE_SERVICE_CHECK-SERVICE_CHECK": ["OOZIE_SERVER-START", "MAPREDUCE2_SERVICE_CHECK-SERVICE_CHECK"], + "HBASE_SERVICE_CHECK-SERVICE_CHECK": ["HBASE_MASTER-START", "HBASE_REGIONSERVER-START"], + "HIVE_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START", "HIVE_METASTORE-START", "WEBHCAT_SERVER-START"], + "PIG_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"], + "SQOOP_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"], + "ZOOKEEPER_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"], + "ZOOKEEPER_QUORUM_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"], + "ZOOKEEPER_SERVER-STOP" : ["HBASE_MASTER-STOP", "HBASE_REGIONSERVER-STOP", "METRICS_COLLECTOR-STOP"], + "HBASE_MASTER-STOP": ["HBASE_REGIONSERVER-STOP"] + }, + "_comment" : "GLUSTERFS-specific dependencies", + "optional_glusterfs": { + "HBASE_MASTER-START": ["PEERSTATUS-START"], + "GLUSTERFS_SERVICE_CHECK-SERVICE_CHECK": ["PEERSTATUS-START"] + }, + "_comment" : "Dependencies that are used when GLUSTERFS is not present in cluster", + "optional_no_glusterfs": { + "METRICS_COLLECTOR-START": ["NAMENODE-START", "DATANODE-START", "SECONDARY_NAMENODE-START", "ZOOKEEPER_SERVER-START"], + "AMBARI_METRICS_SERVICE_CHECK-SERVICE_CHECK": ["METRICS_COLLECTOR-START", "HDFS_SERVICE_CHECK-SERVICE_CHECK"], + "SECONDARY_NAMENODE-START": ["NAMENODE-START"], + "SECONDARY_NAMENODE-RESTART": ["NAMENODE-RESTART"], + "RESOURCEMANAGER-START": ["NAMENODE-START", "DATANODE-START"], + "NODEMANAGER-START": ["NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START"], + "HISTORYSERVER-START": ["NAMENODE-START", "DATANODE-START"], + "HBASE_MASTER-START": ["NAMENODE-START", "DATANODE-START"], + "HIVE_SERVER-START": ["DATANODE-START"], + "WEBHCAT_SERVER-START": ["DATANODE-START"], + "HISTORYSERVER-RESTART": ["NAMENODE-RESTART"], + "RESOURCEMANAGER-RESTART": ["NAMENODE-RESTART"], + "NODEMANAGER-RESTART": ["NAMENODE-RESTART"], + "OOZIE_SERVER-RESTART": ["NAMENODE-RESTART"], + "HDFS_SERVICE_CHECK-SERVICE_CHECK": ["NAMENODE-START", "DATANODE-START", + "SECONDARY_NAMENODE-START"], + "MAPREDUCE2_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", + "RESOURCEMANAGER-START", "HISTORYSERVER-START", "YARN_SERVICE_CHECK-SERVICE_CHECK"], + "YARN_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"], + "RESOURCEMANAGER_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START"], + "PIG_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START", "NODEMANAGER-START"], + "NAMENODE-STOP": ["RESOURCEMANAGER-STOP", "NODEMANAGER-STOP", + "HISTORYSERVER-STOP", "HBASE_MASTER-STOP", "METRICS_COLLECTOR-STOP"], + "DATANODE-STOP": ["RESOURCEMANAGER-STOP", "NODEMANAGER-STOP", + "HISTORYSERVER-STOP", "HBASE_MASTER-STOP"], + "METRICS_GRAFANA-START": ["METRICS_COLLECTOR-START"], + "METRICS_COLLECTOR-STOP": ["METRICS_GRAFANA-STOP"] + }, + "_comment" : "Dependencies that are used in HA NameNode cluster", + "namenode_optional_ha": { + "NAMENODE-START": ["ZKFC-START", "JOURNALNODE-START", "ZOOKEEPER_SERVER-START"], + "ZKFC-START": ["ZOOKEEPER_SERVER-START"], + "ZKFC-STOP": ["NAMENODE-STOP"], + "JOURNALNODE-STOP": ["NAMENODE-STOP"] + }, + "_comment" : "Dependencies that are used in ResourceManager HA cluster", + "resourcemanager_optional_ha" : { + "RESOURCEMANAGER-START": ["ZOOKEEPER_SERVER-START"] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HDFS/metainfo.xml b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HDFS/metainfo.xml new file mode 100755 index 0000000..d6e30b7 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HDFS/metainfo.xml @@ -0,0 +1,27 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>HDFS</name> + <version>2.7.1+odpi</version> + <extends>common-services/HDFS/2.1.0.2.0</extends> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/alerts.json ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/alerts.json b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/alerts.json new file mode 100755 index 0000000..e2431c3 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/alerts.json @@ -0,0 +1,232 @@ +{ + "HIVE": { + "service": [], + "HIVE_METASTORE": [ + { + "name": "hive_metastore_process", + "label": "Hive Metastore Process", + "description": "This host-level alert is triggered if the Hive Metastore process cannot be determined to be up and listening on the network.", + "interval": 3, + "scope": "ANY", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "HIVE/0.12.0.2.0/package/alerts/alert_hive_metastore.py", + "parameters": [ + { + "name": "check.command.timeout", + "display_name": "Command Timeout", + "value": 60.0, + "type": "NUMERIC", + "description": "The maximum time before check command will be killed by timeout", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "default.smoke.user", + "display_name": "Default Smoke User", + "value": "ambari-qa", + "type": "STRING", + "description": "The user that will run the Hive commands if not specified in cluster-env/smokeuser", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.principal", + "display_name": "Default Smoke Principal", + "value": "[email protected]", + "type": "STRING", + "description": "The principal to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_principal_name", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.keytab", + "display_name": "Default Smoke Keytab", + "value": "/etc/security/keytabs/smokeuser.headless.keytab", + "type": "STRING", + "description": "The keytab to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_keytab", + "visibility": "HIDDEN" + } + ] + } + } + ], + "HIVE_SERVER": [ + { + "name": "hive_server_process", + "label": "HiveServer2 Process", + "description": "This host-level alert is triggered if the HiveServer cannot be determined to be up and responding to client requests.", + "interval": 3, + "scope": "ANY", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "HIVE/0.12.0.2.0/package/alerts/alert_hive_thrift_port.py", + "parameters": [ + { + "name": "check.command.timeout", + "display_name": "Command Timeout", + "value": 60.0, + "type": "NUMERIC", + "description": "The maximum time before check command will be killed by timeout", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "default.smoke.user", + "display_name": "Default Smoke User", + "value": "ambari-qa", + "type": "STRING", + "description": "The user that will run the Hive commands if not specified in cluster-env/smokeuser", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.principal", + "display_name": "Default Smoke Principal", + "value": "[email protected]", + "type": "STRING", + "description": "The principal to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_principal_name", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.keytab", + "display_name": "Default Smoke Keytab", + "value": "/etc/security/keytabs/smokeuser.headless.keytab", + "type": "STRING", + "description": "The keytab to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_keytab", + "visibility": "HIDDEN" + } + ] + } + } + ], + "HIVE_SERVER_INTERACTIVE": [ + { + "name": "hive_server_interactive_process", + "label": "HiveServer2 Interactive Process", + "description": "This host-level alert is triggered if the HiveServerInteractive cannot be determined to be up and responding to client requests.", + "interval": 3, + "scope": "ANY", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "HIVE/0.12.0.2.0/package/alerts/alert_hive_interactive_thrift_port.py", + "parameters": [ + { + "name": "check.command.timeout", + "display_name": "Command Timeout", + "value": 60.0, + "type": "NUMERIC", + "description": "The maximum time before check command will be killed by timeout", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "default.smoke.user", + "display_name": "Default Smoke User", + "value": "ambari-qa", + "type": "STRING", + "description": "The user that will run the Hive commands if not specified in cluster-env/smokeuser", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.principal", + "display_name": "Default Smoke Principal", + "value": "[email protected]", + "type": "STRING", + "description": "The principal to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_principal_name", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.keytab", + "display_name": "Default Smoke Keytab", + "value": "/etc/security/keytabs/smokeuser.headless.keytab", + "type": "STRING", + "description": "The keytab to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_keytab", + "visibility": "HIDDEN" + } + ] + } + }, + { + "name": "llap_application", + "label": "LLAP Application", + "description": "This alert is triggered if the LLAP Application cannot be determined to be up and responding to requests.", + "interval": 3, + "scope": "ANY", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "HIVE/0.12.0.2.0/package/alerts/alert_llap_app_status.py", + "parameters": [ + { + "name": "check.command.timeout", + "display_name": "Command Timeout", + "value": 120.0, + "type": "NUMERIC", + "description": "The maximum time before check command will be killed by timeout", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "default.hive.user", + "display_name": "Default HIVE User", + "value": "hive", + "type": "STRING", + "description": "The user that will run the Hive commands if not specified in cluster-env", + "visibility": "HIDDEN" + }, + { + "name": "default.hive.principal", + "display_name": "Default HIVE Principal", + "value": "hive/[email protected]", + "type": "STRING", + "description": "The principal to use when retrieving the kerberos ticket if not specified in cluster-env", + "visibility": "HIDDEN" + }, + { + "name": "default.hive.keytab", + "display_name": "Default HIVE Keytab", + "value": "/etc/security/keytabs/hive.llap.zk.sm.keytab", + "type": "STRING", + "description": "The keytab to use when retrieving the kerberos ticket if not specified in cluster-env.", + "visibility": "HIDDEN" + } + ] + } + } + ], + "WEBHCAT_SERVER": [ + { + "name": "hive_webhcat_server_status", + "label": "WebHCat Server Status", + "description": "This host-level alert is triggered if the templeton server status is not healthy.", + "interval": 1, + "scope": "ANY", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "HIVE/0.12.0.2.0/package/alerts/alert_webhcat_server.py", + "parameters": [ + { + "name": "default.smoke.user", + "display_name": "Default Smoke User", + "value": "ambari-qa", + "type": "STRING", + "description": "The user that will run the Hive commands if not specified in cluster-env/smokeuser", + "visibility": "HIDDEN" + }, + { + "name": "connection.timeout", + "display_name": "Connection Timeout", + "value": 5.0, + "type": "NUMERIC", + "description": "The maximum time before this alert is considered to be CRITICAL", + "units": "seconds", + "threshold": "CRITICAL" + } + ] + } + } + ] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/075cecbf/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/configuration/hcat-env.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/configuration/hcat-env.xml b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/configuration/hcat-env.xml new file mode 100755 index 0000000..3908d61 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/configuration/hcat-env.xml @@ -0,0 +1,41 @@ +<configuration><property require-input="false"> + <name>content</name> + <value> + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + JAVA_HOME={{java64_home}} + HCAT_PID_DIR={{hcat_pid_dir}}/ + HCAT_LOG_DIR={{hcat_log_dir}}/ + HCAT_CONF_DIR={{hcat_conf_dir}} + HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + #DBROOT is the path where the connector jars are downloaded + DBROOT={{hcat_dbroot}} + USER={{hcat_user}} + METASTORE_PORT={{hive_metastore_port}} + </value> + <description>This is the jinja template for hcat-env.sh file</description> + <display-name>hcat-env template</display-name> + <filename>hcat-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>content</type> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property></configuration> \ No newline at end of file
