http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/nodemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/nodemanager.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/nodemanager.py new file mode 100755 index 0000000..d185b7e --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/nodemanager.py @@ -0,0 +1,144 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import nodemanager_upgrade + +from resource_management import * +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions.version import compare_versions, format_stack_version +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.security_commons import build_expectations, \ + cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ + FILE_TYPE_XML + +from yarn import yarn +from service import service + +class Nodemanager(Script): + + def get_component_name(self): + return "hadoop-yarn-nodemanager" + + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + yarn(name="nodemanager") + + def pre_upgrade_restart(self, env, upgrade_type=None): + Logger.info("Executing NodeManager Stack Upgrade pre-restart") + import params + env.set_params(params) + + if params.version and compare_versions(format_stack_version(params.version), '4.0.0.0') >= 0: + conf_select.select(params.stack_name, "hadoop", params.version) + stack_select.select("hadoop-yarn-nodemanager", params.version) + #Execute(format("stack-select set hadoop-yarn-nodemanager {version}")) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) # FOR SECURITY + service('nodemanager',action='start') + + def post_upgrade_restart(self, env, upgrade_type=None): + Logger.info("Executing NodeManager Stack Upgrade post-restart") + import params + env.set_params(params) + + nodemanager_upgrade.post_upgrade_check() + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + + service('nodemanager',action='stop') + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.nodemanager_pid_file) + + def security_status(self, env): + import status_params + env.set_params(status_params) + if status_params.security_enabled: + props_value_check = {"yarn.timeline-service.http-authentication.type": "kerberos", + "yarn.acl.enable": "true"} + props_empty_check = ["yarn.nodemanager.principal", + "yarn.nodemanager.keytab", + "yarn.nodemanager.webapp.spnego-principal", + "yarn.nodemanager.webapp.spnego-keytab-file"] + + props_read_check = ["yarn.nodemanager.keytab", + "yarn.nodemanager.webapp.spnego-keytab-file"] + yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check, + props_read_check) + + yarn_expectations ={} + yarn_expectations.update(yarn_site_props) + + security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, + {'yarn-site.xml': FILE_TYPE_XML}) + result_issues = validate_security_config_properties(security_params, yarn_site_props) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ( 'yarn-site' not in security_params + or 'yarn.nodemanager.keytab' not in security_params['yarn-site'] + or 'yarn.nodemanager.principal' not in security_params['yarn-site']) \ + or 'yarn.nodemanager.webapp.spnego-keytab-file' not in security_params['yarn-site'] \ + or 'yarn.nodemanager.webapp.spnego-principal' not in security_params['yarn-site']: + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.yarn_user, + security_params['yarn-site']['yarn.nodemanager.keytab'], + security_params['yarn-site']['yarn.nodemanager.principal'], + status_params.hostname, + status_params.tmp_dir) + cached_kinit_executor(status_params.kinit_path_local, + status_params.yarn_user, + security_params['yarn-site']['yarn.nodemanager.webapp.spnego-keytab-file'], + security_params['yarn-site']['yarn.nodemanager.webapp.spnego-principal'], + status_params.hostname, + status_params.tmp_dir) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) + else: + self.put_structured_out({"securityState": "UNSECURED"}) + +if __name__ == "__main__": + Nodemanager().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/nodemanager_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/nodemanager_upgrade.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/nodemanager_upgrade.py new file mode 100755 index 0000000..54e0fae --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/nodemanager_upgrade.py @@ -0,0 +1,74 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import subprocess + +from resource_management.core.logger import Logger +from resource_management.core.exceptions import Fail +from resource_management.core.resources.system import Execute +from resource_management.core import shell +from resource_management.libraries.functions.decorator import retry + + +def post_upgrade_check(): + ''' + Checks that the NodeManager has rejoined the cluster. + This function will obtain the Kerberos ticket if security is enabled. + :return: + ''' + import params + + Logger.info('NodeManager executing "yarn node -list -states=RUNNING" to verify the node has rejoined the cluster...') + if params.security_enabled and params.nodemanager_kinit_cmd: + Execute(params.nodemanager_kinit_cmd, user = params.yarn_user) + + _check_nodemanager_startup() + + +@retry(times=12, sleep_time=10, err_class=Fail) +def _check_nodemanager_startup(): + ''' + Checks that a NodeManager is in a RUNNING state in the cluster via + "yarn node -list -states=RUNNING" command. Once the NodeManager is found to be + alive this method will return, otherwise it will raise a Fail(...) and retry + automatically. + :return: + ''' + import params + + command = 'yarn node -list -states=RUNNING' + + try: + # 'su - yarn -c "yarn node -status c6401.ambari.apache.org:45454"' + return_code, yarn_output = shell.call(command, user=params.hdfs_user) + except: + raise Fail('Unable to determine if the NodeManager has started after upgrade.') + + if return_code == 0: + hostname = params.hostname.lower() + nodemanager_address = params.nm_address.lower() + yarn_output = yarn_output.lower() + + if hostname in yarn_output or nodemanager_address in yarn_output: + Logger.info('NodeManager with ID {0} has rejoined the cluster.'.format(nodemanager_address)) + return + else: + raise Fail('NodeManager with ID {0} was not found in the list of running NodeManagers'.format(nodemanager_address)) + + raise Fail('Unable to determine if the NodeManager has started after upgrade (result code {0})'.format(str(return_code))) http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/params.py new file mode 100755 index 0000000..957e2dd --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/params.py @@ -0,0 +1,224 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +import os +from resource_management.libraries.functions.version import format_stack_version, compare_versions +from resource_management.libraries.functions.default import default +from resource_management import * +from resource_management.libraries.functions import conf_select, stack_select +import status_params + +# server configurations +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() + +stack_name = default("/hostLevelParams/stack_name", None) + +# This is expected to be of the form #.#.#.# +stack_version_unformatted = str(config['hostLevelParams']['stack_version']) +stack_version = format_stack_version(stack_version_unformatted) + +# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade +version = default("/commandParams/version", None) + +hostname = config['hostname'] + +host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) + +#hadoop params +yarn_role_root = "hadoop-yarn-client" +mapred_role_root = "hadoop-mapreduce-client" + +command_role = default("/role", "") +if command_role == "APP_TIMELINE_SERVER": + yarn_role_root = "hadoop-yarn-timelineserver" +elif command_role == "HISTORYSERVER": + mapred_role_root = "hadoop-mapreduce-historyserver" +elif command_role == "MAPREDUCE2_CLIENT": + mapred_role_root = "hadoop-mapreduce-client" +elif command_role == "NODEMANAGER": + yarn_role_root = "hadoop-yarn-nodemanager" +elif command_role == "RESOURCEMANAGER": + yarn_role_root = "hadoop-yarn-resourcemanager" +elif command_role == "YARN_CLIENT": + yarn_role_root = "hadoop-yarn-client" + +nm_hosts = default("/clusterHostInfo/nm_hosts", []) +number_of_nm = len(nm_hosts) + +hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec") +hadoop_bin = stack_select.get_hadoop_dir("sbin") +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() +hadoop_mapred2_jar_location = format("/usr/iop/current/{mapred_role_root}") +mapred_bin = format("/usr/iop/current/{mapred_role_root}/sbin") +hadoop_yarn_home = format("/usr/iop/current/{yarn_role_root}") +yarn_bin = format("/usr/iop/current/{yarn_role_root}/sbin") +yarn_container_bin = format("/usr/iop/current/{yarn_role_root}/bin") + +limits_conf_dir = "/etc/security/limits.d" +execute_path = os.environ['PATH'] + os.pathsep + hadoop_bin_dir + os.pathsep + yarn_container_bin + +ulimit_cmd = "ulimit -c unlimited;" + +mapred_user = status_params.mapred_user +yarn_user = status_params.yarn_user +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] + +smokeuser = config['configurations']['cluster-env']['smokeuser'] +smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +yarn_executor_container_group = config['configurations']['yarn-site']['yarn.nodemanager.linux-container-executor.group'] +kinit_path_local = functions.get_kinit_path() +rm_hosts = config['clusterHostInfo']['rm_host'] +rm_host = rm_hosts[0] +rm_port = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'].split(':')[-1] +rm_https_port = "8090" + +default_exclude_path=os.path.join(hadoop_conf_dir, 'yarn.exclude') +# TODO UPGRADE default, update site during upgrade +rm_nodes_exclude_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path",default_exclude_path) + +java64_home = config['hostLevelParams']['java_home'] +hadoop_ssl_enabled = default("/configurations/core-site/hadoop.ssl.enabled", False) + +yarn_heapsize = config['configurations']['yarn-env']['yarn_heapsize'] +resourcemanager_heapsize = config['configurations']['yarn-env']['resourcemanager_heapsize'] +nodemanager_heapsize = config['configurations']['yarn-env']['nodemanager_heapsize'] +apptimelineserver_heapsize = default("/configurations/yarn-env/apptimelineserver_heapsize", 1024) +ats_leveldb_dir = config['configurations']['yarn-site']['yarn.timeline-service.leveldb-timeline-store.path'] +yarn_log_dir_prefix = config['configurations']['yarn-env']['yarn_log_dir_prefix'] +yarn_pid_dir_prefix = status_params.yarn_pid_dir_prefix +mapred_pid_dir_prefix = status_params.mapred_pid_dir_prefix +mapred_log_dir_prefix = config['configurations']['mapred-env']['mapred_log_dir_prefix'] +mapred_env_sh_template = config['configurations']['mapred-env']['content'] +yarn_env_sh_template = config['configurations']['yarn-env']['content'] + +if len(rm_hosts) > 1: + additional_rm_host = rm_hosts[1] + rm_webui_address = format("{rm_host}:{rm_port},{additional_rm_host}:{rm_port}") + rm_webui_https_address = format("{rm_host}:{rm_https_port},{additional_rm_host}:{rm_https_port}") +else: + rm_webui_address = format("{rm_host}:{rm_port}") + rm_webui_https_address = format("{rm_host}:{rm_https_port}") + +nm_webui_address = config['configurations']['yarn-site']['yarn.nodemanager.webapp.address'] +hs_webui_address = config['configurations']['mapred-site']['mapreduce.jobhistory.webapp.address'] +nm_address = config['configurations']['yarn-site']['yarn.nodemanager.address'] # still contains 0.0.0.0 +if hostname and nm_address and nm_address.startswith("0.0.0.0:"): + nm_address = nm_address.replace("0.0.0.0", hostname) + +nm_local_dirs = config['configurations']['yarn-site']['yarn.nodemanager.local-dirs'] +nm_log_dirs = config['configurations']['yarn-site']['yarn.nodemanager.log-dirs'] + +distrAppJarName = "hadoop-yarn-applications-distributedshell-2.*.jar" +hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar" + +yarn_pid_dir = status_params.yarn_pid_dir +mapred_pid_dir = status_params.mapred_pid_dir + +mapred_log_dir = format("{mapred_log_dir_prefix}/{mapred_user}") +yarn_log_dir = format("{yarn_log_dir_prefix}/{yarn_user}") +mapred_job_summary_log = format("{mapred_log_dir_prefix}/{mapred_user}/hadoop-mapreduce.jobsummary.log") +yarn_job_summary_log = format("{yarn_log_dir_prefix}/{yarn_user}/hadoop-mapreduce.jobsummary.log") + +user_group = config['configurations']['cluster-env']['user_group'] + +#exclude file +default_exclude_path=os.path.join(hadoop_conf_dir, 'yarn.exclude') +exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) +exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path",default_exclude_path) + +ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", [])) +has_ats = not len(ats_host) == 0 + +# default kinit commands +rm_kinit_cmd = "" +yarn_timelineservice_kinit_cmd = "" +nodemanager_kinit_cmd = "" + +if security_enabled: + _rm_principal_name = config['configurations']['yarn-site']['yarn.resourcemanager.principal'] + _rm_principal_name = _rm_principal_name.replace('_HOST',hostname.lower()) + _rm_keytab = config['configurations']['yarn-site']['yarn.resourcemanager.keytab'] + rm_kinit_cmd = format("{kinit_path_local} -kt {_rm_keytab} {_rm_principal_name};") + + if has_ats: + _yarn_timelineservice_principal_name = config['configurations']['yarn-site']['yarn.timeline-service.principal'] + _yarn_timelineservice_principal_name = _yarn_timelineservice_principal_name.replace('_HOST', hostname.lower()) + _yarn_timelineservice_keytab = config['configurations']['yarn-site']['yarn.timeline-service.keytab'] + yarn_timelineservice_kinit_cmd = format("{kinit_path_local} -kt {_yarn_timelineservice_keytab} {_yarn_timelineservice_principal_name};") + + if 'yarn.nodemanager.principal' in config['configurations']['yarn-site']: + _nodemanager_principal_name = default('/configurations/yarn-site/yarn.nodemanager.principal', None) + if _nodemanager_principal_name: + _nodemanager_principal_name = _nodemanager_principal_name.replace('_HOST', hostname.lower()) + + _nodemanager_keytab = config['configurations']['yarn-site']['yarn.nodemanager.keytab'] + nodemanager_kinit_cmd = format("{kinit_path_local} -kt {_nodemanager_keytab} {_nodemanager_principal_name};") + +else: + rm_kinit_cmd = "" + yarn_timelineservice_kinit_cmd = "" + +yarn_log_aggregation_enabled = config['configurations']['yarn-site']['yarn.log-aggregation-enable'] +yarn_nm_app_log_dir = config['configurations']['yarn-site']['yarn.nodemanager.remote-app-log-dir'] +mapreduce_jobhistory_intermediate_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.intermediate-done-dir'] +mapreduce_jobhistory_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.done-dir'] +jobhistory_heapsize = default("/configurations/mapred-env/jobhistory_heapsize", "900") +jhs_leveldb_state_store_dir = default('/configurations/mapred-site/mapreduce.jobhistory.recovery.store.leveldb.path', "/hadoop/mapreduce/jhs") + +#for create_hdfs_directory +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] + + + +hdfs_site = config['configurations']['hdfs-site'] +default_fs = config['configurations']['core-site']['fs.defaultFS'] + +import functools +#create partial functions with common arguments for every HdfsResource call +#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_principal_name, + hdfs_site = hdfs_site, + default_fs = default_fs + ) +update_exclude_file_only = default("/commandParams/update_exclude_file_only",False) + +mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group) + +#taskcontroller.cfg + +mapred_local_dir = "/tmp/hadoop-mapred/mapred/local" +hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] +min_user_id = config['configurations']['yarn-env']['min_user_id'] + +# Node labels +node_labels_dir = default("/configurations/yarn-site/yarn.node-labels.fs-store.root-dir", None) http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/resourcemanager.py new file mode 100755 index 0000000..8e350b9 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/resourcemanager.py @@ -0,0 +1,179 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions.version import compare_versions, format_stack_version +from resource_management.libraries.functions.security_commons import build_expectations, \ + cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ + FILE_TYPE_XML + +from yarn import yarn +from service import service + + +class Resourcemanager(Script): + + def get_component_name(self): + return "hadoop-yarn-resourcemanager" + + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + + env.set_params(params) + yarn(name='resourcemanager') + + def pre_upgrade_restart(self, env, upgrade_type=None): + Logger.info("Executing Stack Upgrade post-restart") + import params + env.set_params(params) + + if params.version and compare_versions(format_stack_version(params.version), '4.0.0.0') >= 0: + conf_select.select(params.stack_name, "hadoop", params.version) + stack_select.select("hadoop-yarn-resourcemanager", params.version) + #Execute(format("stack-select set hadoop-yarn-resourcemanager {version}")) + + def start(self, env, upgrade_type=None): + import params + + env.set_params(params) + self.configure(env) # FOR SECURITY + service('resourcemanager', + action='start' + ) + + def stop(self, env, upgrade_type=None): + import params + + env.set_params(params) + + service('resourcemanager', + action='stop' + ) + + def status(self, env): + import status_params + + env.set_params(status_params) + check_process_status(status_params.resourcemanager_pid_file) + pass + + def security_status(self, env): + import status_params + env.set_params(status_params) + if status_params.security_enabled: + props_value_check = {"yarn.timeline-service.http-authentication.type": "kerberos", + "yarn.acl.enable": "true"} + props_empty_check = ["yarn.resourcemanager.principal", + "yarn.resourcemanager.keytab", + "yarn.resourcemanager.webapp.spnego-principal", + "yarn.resourcemanager.webapp.spnego-keytab-file"] + + props_read_check = ["yarn.resourcemanager.keytab", + "yarn.resourcemanager.webapp.spnego-keytab-file"] + yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check, + props_read_check) + + yarn_expectations ={} + yarn_expectations.update(yarn_site_props) + + security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, + {'yarn-site.xml': FILE_TYPE_XML}) + result_issues = validate_security_config_properties(security_params, yarn_site_props) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ( 'yarn-site' not in security_params + or 'yarn.resourcemanager.keytab' not in security_params['yarn-site'] + or 'yarn.resourcemanager.principal' not in security_params['yarn-site']) \ + or 'yarn.resourcemanager.webapp.spnego-keytab-file' not in security_params['yarn-site'] \ + or 'yarn.resourcemanager.webapp.spnego-principal' not in security_params['yarn-site']: + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.yarn_user, + security_params['yarn-site']['yarn.resourcemanager.keytab'], + security_params['yarn-site']['yarn.resourcemanager.principal'], + status_params.hostname, + status_params.tmp_dir) + cached_kinit_executor(status_params.kinit_path_local, + status_params.yarn_user, + security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-keytab-file'], + security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-principal'], + status_params.hostname, + status_params.tmp_dir) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) + else: + self.put_structured_out({"securityState": "UNSECURED"}) + + def refreshqueues(self, env): + import params + + self.configure(env) + env.set_params(params) + + service('resourcemanager', + action='refreshQueues' + ) + + def decommission(self, env): + import params + + env.set_params(params) + rm_kinit_cmd = params.rm_kinit_cmd + yarn_user = params.yarn_user + conf_dir = params.hadoop_conf_dir + user_group = params.user_group + + yarn_refresh_cmd = format("{rm_kinit_cmd} yarn --config {conf_dir} rmadmin -refreshNodes") + + File(params.exclude_file_path, + content=Template("exclude_hosts_list.j2"), + owner=yarn_user, + group=user_group + ) + + if params.update_exclude_file_only == False: + Execute(yarn_refresh_cmd, + environment= {'PATH' : params.execute_path }, + user=yarn_user) + pass + pass + + +if __name__ == "__main__": + Resourcemanager().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/service.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/service.py new file mode 100755 index 0000000..1002094 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/service.py @@ -0,0 +1,76 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * + + +def service(componentName, action='start', serviceName='yarn'): + + import params + + if serviceName == 'mapreduce' and componentName == 'historyserver': + daemon = format("{mapred_bin}/mr-jobhistory-daemon.sh") + pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-{componentName}.pid") + usr = params.mapred_user + else: + daemon = format("{yarn_bin}/yarn-daemon.sh") + pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-{componentName}.pid") + usr = params.yarn_user + + cmd = format("export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --config {hadoop_conf_dir}") + + if action == 'start': + daemon_cmd = format("{ulimit_cmd} {cmd} start {componentName}") + check_process = format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1") + + # Remove the pid file if its corresponding process is not running. + File(pid_file, + action="delete", + not_if=check_process) + + # Attempt to start the process. Internally, this is skipped if the process is already running. + Execute(daemon_cmd, + user=usr, + not_if=check_process + ) + + # Ensure that the process with the expected PID exists. + Execute(check_process, + user=usr, + not_if=check_process, + initial_wait=5 + ) + + elif action == 'stop': + daemon_cmd = format("{cmd} stop {componentName}") + Execute(daemon_cmd, + user=usr) + + File(pid_file, + action="delete") + + elif action == 'refreshQueues': + rm_kinit_cmd = params.rm_kinit_cmd + refresh_cmd = format("{rm_kinit_cmd} export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {yarn_container_bin}/yarn rmadmin -refreshQueues") + + Execute(refresh_cmd, + user=usr, + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/service_check.py new file mode 100755 index 0000000..7f42f9e --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/service_check.py @@ -0,0 +1,89 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import sys +import re +from resource_management.libraries.functions.version import compare_versions +from resource_management import * +import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. + +CURL_CONNECTION_TIMEOUT = '5' + + +class ServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + + if params.stack_version != "" and compare_versions(params.stack_version, '4.0') >= 0: + path_to_distributed_shell_jar = "/usr/iop/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar" + else: + path_to_distributed_shell_jar = "/usr/lib/hadoop-yarn/hadoop-yarn-applications-distributedshell*.jar" + + yarn_distrubuted_shell_check_cmd = format("yarn org.apache.hadoop.yarn.applications.distributedshell.Client " + "-shell_command ls -num_containers {number_of_nm} -jar {path_to_distributed_shell_jar}") + + if params.security_enabled: + kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};") + smoke_cmd = format("{kinit_cmd} {yarn_distrubuted_shell_check_cmd}") + else: + smoke_cmd = yarn_distrubuted_shell_check_cmd + + return_code, out = shell.checked_call(smoke_cmd, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + user=params.smokeuser, + ) + + m = re.search("appTrackingUrl=(.*),\s", out) + app_url = m.group(1) + + splitted_app_url = str(app_url).split('/') + + for item in splitted_app_url: + if "application" in item: + application_name = item + + json_response_received = False + for rm_host in params.rm_hosts: + info_app_url = "http://"; + rm_host + ":" + params.rm_port + "/ws/v1/cluster/apps/" + application_name + + get_app_info_cmd = "curl --negotiate -u : -sL --connect-timeout " + CURL_CONNECTION_TIMEOUT + " " + info_app_url + + return_code, stdout = shell.checked_call(get_app_info_cmd, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + user=params.smokeuser, + ) + + try: + json_response = json.loads(stdout) + + json_response_received = True + + if json_response['app']['state'] != "FINISHED" or json_response['app']['finalStatus'] != "SUCCEEDED": + raise Exception("Application " + app_url + " state/status is not valid. Should be FINISHED/SUCCEEDED.") + except Exception as e: + pass + + if not json_response_received: + raise Exception("Could not get json response from YARN API") + +if __name__ == "__main__": + ServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/status_params.py new file mode 100755 index 0000000..3b7f5ba --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/status_params.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from resource_management.libraries.functions import conf_select + +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() + +mapred_user = config['configurations']['mapred-env']['mapred_user'] +yarn_user = config['configurations']['yarn-env']['yarn_user'] +yarn_pid_dir_prefix = config['configurations']['yarn-env']['yarn_pid_dir_prefix'] +mapred_pid_dir_prefix = config['configurations']['mapred-env']['mapred_pid_dir_prefix'] +yarn_pid_dir = format("{yarn_pid_dir_prefix}/{yarn_user}") +mapred_pid_dir = format("{mapred_pid_dir_prefix}/{mapred_user}") + +resourcemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-resourcemanager.pid") +nodemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-nodemanager.pid") +yarn_historyserver_pid_file_old = format("{yarn_pid_dir}/yarn-{yarn_user}-historyserver.pid") +yarn_historyserver_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-timelineserver.pid") # *-historyserver.pid is deprecated +mapred_historyserver_pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-historyserver.pid") + +# Security related/required params +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() +hostname = config['hostname'] +kinit_path_local = functions.get_kinit_path() +security_enabled = config['configurations']['cluster-env']['security_enabled'] http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/yarn.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/yarn.py new file mode 100755 index 0000000..1264b87 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/yarn.py @@ -0,0 +1,277 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * +import sys +import os + + +def yarn(name = None): + import params + + + if name in ["nodemanager","historyserver"]: + if params.yarn_log_aggregation_enabled: + params.HdfsResource(params.yarn_nm_app_log_dir, + action="create_on_execute", + type="directory", + owner=params.yarn_user, + group=params.user_group, + mode=0777, + recursive_chmod=True + ) + params.HdfsResource("/mapred", + type="directory", + action="create_on_execute", + owner=params.mapred_user + ) + params.HdfsResource("/mapred/system", + type="directory", + action="create_on_execute", + owner=params.hdfs_user + ) + params.HdfsResource(params.mapreduce_jobhistory_intermediate_done_dir, + type="directory", + action="create_on_execute", + owner=params.mapred_user, + group=params.user_group, + mode=0777 + ) + + params.HdfsResource(params.mapreduce_jobhistory_done_dir, + type="directory", + action="create_on_execute", + owner=params.mapred_user, + group=params.user_group, + mode=0777 + ) + params.HdfsResource(None, action="execute") + Directory(params.jhs_leveldb_state_store_dir, + owner=params.mapred_user, + group=params.user_group, + create_parents=True, + cd_access="a", + ) + + if name == "nodemanager": + Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','), + owner=params.yarn_user, + group=params.user_group, + create_parents=True, + cd_access="a", + ignore_failures=True, + mode=0775 + ) + + Execute(('chown', '-R', params.yarn_user, params.nm_local_dirs), + only_if=format("test -d {nm_local_dirs}"), + sudo=True) + + + if params.security_enabled: + smokeuser_directories = [os.path.join(dir, 'usercache' ,params.smokeuser) + for dir in params.nm_local_dirs.split(',')] + for directory in smokeuser_directories: + Execute(('chown', '-R', params.smokeuser, directory), + only_if=format("test -d {directory}"), + sudo=True, + ) + Directory([params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir], + owner=params.yarn_user, + group=params.user_group, + create_parents=True, + cd_access = 'a', + ) + + Directory([params.mapred_pid_dir_prefix, params.mapred_pid_dir, params.mapred_log_dir_prefix, params.mapred_log_dir], + owner=params.mapred_user, + group=params.user_group, + create_parents=True, + cd_access = 'a', + ) + Directory([params.yarn_log_dir_prefix], + owner=params.yarn_user, + create_parents=True, + ignore_failures=True, + cd_access = 'a', + ) + + XmlConfig("core-site.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations']['core-site'], + configuration_attributes=params.config['configuration_attributes']['core-site'], + owner=params.hdfs_user, + group=params.user_group, + mode=0644 + ) + + XmlConfig("mapred-site.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations']['mapred-site'], + configuration_attributes=params.config['configuration_attributes']['mapred-site'], + owner=params.yarn_user, + group=params.user_group, + mode=0644 + ) + + XmlConfig("yarn-site.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations']['yarn-site'], + configuration_attributes=params.config['configuration_attributes']['yarn-site'], + owner=params.yarn_user, + group=params.user_group, + mode=0644 + ) + + XmlConfig("capacity-scheduler.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations']['capacity-scheduler'], + configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'], + owner=params.yarn_user, + group=params.user_group, + mode=0644 + ) + + if name == 'resourcemanager': + File(params.yarn_job_summary_log, + owner=params.yarn_user, + group=params.user_group + ) + if params.node_labels_dir: + params.HdfsResource(params.node_labels_dir, + type="directory", + action="create_on_execute", + owner=params.yarn_user, + group=params.user_group, + mode=0700 + ) + params.HdfsResource(None, action="execute") + elif name == 'apptimelineserver': + Directory(params.ats_leveldb_dir, + owner=params.yarn_user, + group=params.user_group, + create_parents=True, + cd_access="a", + ) + + File(params.rm_nodes_exclude_path, + owner=params.yarn_user, + group=params.user_group + ) + + File(format("{limits_conf_dir}/yarn.conf"), + mode=0644, + content=Template('yarn.conf.j2') + ) + + File(format("{limits_conf_dir}/mapreduce.conf"), + mode=0644, + content=Template('mapreduce.conf.j2') + ) + + File(format("{hadoop_conf_dir}/yarn-env.sh"), + owner=params.yarn_user, + group=params.user_group, + mode=0755, + content=InlineTemplate(params.yarn_env_sh_template) + ) + + container_executor = format("{yarn_container_bin}/container-executor") + File(container_executor, + group=params.yarn_executor_container_group, + mode=06050 + ) + + File(format("{hadoop_conf_dir}/container-executor.cfg"), + group=params.user_group, + mode=0644, + content=Template('container-executor.cfg.j2') + ) + + + if params.security_enabled: + tc_mode = 0644 + tc_owner = "root" + else: + tc_mode = None + tc_owner = params.hdfs_user + + File(format("{hadoop_conf_dir}/mapred-env.sh"), + owner=tc_owner, + content=InlineTemplate(params.mapred_env_sh_template) + ) + + if params.security_enabled: + File(os.path.join(params.hadoop_bin, "task-controller"), + owner="root", + group=params.mapred_tt_group, + mode=06050 + ) + File(os.path.join(params.hadoop_conf_dir, 'taskcontroller.cfg'), + owner = tc_owner, + mode = tc_mode, + group = params.mapred_tt_group, + content=Template("taskcontroller.cfg.j2") + ) + else: + File(os.path.join(params.hadoop_conf_dir, 'taskcontroller.cfg'), + owner=tc_owner, + content=Template("taskcontroller.cfg.j2") + ) + + if "mapred-site" in params.config['configurations']: + XmlConfig("mapred-site.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations']['mapred-site'], + configuration_attributes=params.config['configuration_attributes']['mapred-site'], + owner=params.mapred_user, + group=params.user_group + ) + + if "capacity-scheduler" in params.config['configurations']: + XmlConfig("capacity-scheduler.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations'][ + 'capacity-scheduler'], + configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'], + owner=params.hdfs_user, + group=params.user_group + ) + + if os.path.exists(os.path.join(params.hadoop_conf_dir, 'fair-scheduler.xml')): + File(os.path.join(params.hadoop_conf_dir, 'fair-scheduler.xml'), + owner=params.mapred_user, + group=params.user_group + ) + + if os.path.exists( + os.path.join(params.hadoop_conf_dir, 'ssl-client.xml.example')): + File(os.path.join(params.hadoop_conf_dir, 'ssl-client.xml.example'), + owner=params.mapred_user, + group=params.user_group + ) + + if os.path.exists( + os.path.join(params.hadoop_conf_dir, 'ssl-server.xml.example')): + File(os.path.join(params.hadoop_conf_dir, 'ssl-server.xml.example'), + owner=params.mapred_user, + group=params.user_group + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/yarn_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/yarn_client.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/yarn_client.py new file mode 100755 index 0000000..0af5ccb --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/scripts/yarn_client.py @@ -0,0 +1,56 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import sys +from resource_management import * +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select + +from yarn import yarn + +class YarnClient(Script): + + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + yarn() + + def status(self, env): + raise ClientComponentHasNoStatus() + + def get_component_name(self): + return "hadoop-client" + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + if params.version and compare_versions(format_stack_version(params.version), '4.0.0.0') >= 0: + conf_select.select(params.stack_name, "hadoop", params.version) + stack_select.select("hadoop-client", params.version) + #Execute(format("stack-select set hadoop-client {version}")) + +if __name__ == "__main__": + YarnClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/container-executor.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/container-executor.cfg.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/container-executor.cfg.j2 new file mode 100755 index 0000000..9f38f183 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/container-executor.cfg.j2 @@ -0,0 +1,40 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +#/* +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +yarn.nodemanager.local-dirs={{nm_local_dirs}} +yarn.nodemanager.log-dirs={{nm_log_dirs}} +yarn.nodemanager.linux-container-executor.group={{yarn_executor_container_group}} +banned.users=hdfs,yarn,mapred,bin +min.user.id=200 http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/exclude_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/exclude_hosts_list.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/exclude_hosts_list.j2 new file mode 100755 index 0000000..c7ce416 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/exclude_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in exclude_hosts %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/mapreduce.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/mapreduce.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/mapreduce.conf.j2 new file mode 100755 index 0000000..b996645 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/mapreduce.conf.j2 @@ -0,0 +1,35 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{mapred_user}} - nofile 32768 +{{mapred_user}} - nproc 65536 http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/taskcontroller.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/taskcontroller.cfg.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/taskcontroller.cfg.j2 new file mode 100755 index 0000000..3d5f4f2 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/taskcontroller.cfg.j2 @@ -0,0 +1,38 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +#/* +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +mapred.local.dir={{mapred_local_dir}} +mapreduce.tasktracker.group={{mapred_tt_group}} +hadoop.log.dir={{hdfs_log_dir_prefix}}/{{mapred_user}} http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/yarn.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/yarn.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/yarn.conf.j2 new file mode 100755 index 0000000..3bd7a45 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/YARN/package/templates/yarn.conf.j2 @@ -0,0 +1,35 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{yarn_user}} - nofile 32768 +{{yarn_user}} - nproc 65536 http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/alerts.json b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/alerts.json new file mode 100755 index 0000000..027d7fa --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/alerts.json @@ -0,0 +1,58 @@ +{ + "ZOOKEEPER": { + "service": [ + { + "name": "zookeeper_server_process_percent", + "label": "Percent ZooKeeper Servers Available", + "description": "This alert is triggered if the number of down ZooKeeper servers in the cluster is greater than the configured critical threshold. It aggregates the results of ZooKeeper process checks.", + "interval": 1, + "scope": "SERVICE", + "enabled": true, + "source": { + "type": "AGGREGATE", + "alert_name": "zookeeper_server_process", + "reporting": { + "ok": { + "text": "affected: [{1}], total: [{0}]" + }, + "warning": { + "text": "affected: [{1}], total: [{0}]", + "value": 0.35 + }, + "critical": { + "text": "affected: [{1}], total: [{0}]", + "value": 0.70 + } + } + } + } + ], + "ZOOKEEPER_SERVER": [ + { + "name": "zookeeper_server_process", + "label": "ZooKeeper Server Process", + "description": "This host-level alert is triggered if the ZooKeeper server process cannot be determined to be up and listening on the network.", + "interval": 1, + "scope": "ANY", + "source": { + "type": "PORT", + "uri": "{{zoo.cfg/clientPort}}", + "default_port": 2181, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + } + ] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/configuration/zoo.cfg.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/configuration/zoo.cfg.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/configuration/zoo.cfg.xml new file mode 100755 index 0000000..275cc6a --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/configuration/zoo.cfg.xml @@ -0,0 +1,84 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>tickTime</name> + <display-name>Length of single Tick</display-name> + <value>2000</value> + <description>The length of a single tick in milliseconds, which is the basic time unit used by ZooKeeper</description> + <value-attributes> + <type>int</type> + <overridable>false</overridable> + <unit>ms</unit> + </value-attributes> + </property> + <property> + <name>initLimit</name> + <display-name>Ticks to allow for sync at Init</display-name> + <value>10</value> + <description>Ticks to allow for sync at Init.</description> + <value-attributes> + <type>int</type> + <overridable>false</overridable> + </value-attributes> + </property> + <property> + <name>syncLimit</name> + <display-name>Ticks to allow for sync at Runtime</display-name> + <value>5</value> + <description>Ticks to allow for sync at Runtime.</description> + <value-attributes> + <type>int</type> + <overridable>false</overridable> + </value-attributes> + </property> + <property> + <name>clientPort</name> + <display-name>Port for running ZK Server</display-name> + <value>2181</value> + <description>Port for running ZK Server.</description> + <value-attributes> + <type>int</type> + <overridable>false</overridable> + </value-attributes> + </property> + <property> + <name>dataDir</name> + <display-name>ZooKeeper directory</display-name> + <value>/hadoop/zookeeper</value> + <description>Data directory for ZooKeeper.</description> + <value-attributes> + <type>directory</type> + </value-attributes> + </property> + <property> + <name>autopurge.snapRetainCount</name> + <value>30</value> + <description>ZooKeeper purge feature retains the autopurge.snapRetainCount most recent snapshots and the corresponding transaction logs in the dataDir and dataLogDir respectively and deletes the rest. </description> + </property> + <property> + <name>autopurge.purgeInterval</name> + <value>24</value> + <description>The time interval in hours for which the purge task has to be triggered. Set to a positive integer (1 and above) to enable the auto purging.</description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/configuration/zookeeper-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/configuration/zookeeper-env.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/configuration/zookeeper-env.xml new file mode 100755 index 0000000..b57992e --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/configuration/zookeeper-env.xml @@ -0,0 +1,73 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>zk_user</name> + <value>zookeeper</value> + <property-type>USER</property-type> + <description>ZooKeeper User.</description> + </property> + <property> + <name>zk_log_dir</name> + <display-name>ZooKeeper Log Dir</display-name> + <value>/var/log/zookeeper</value> + <description>ZooKeeper Log Dir</description> + <value-attributes> + <type>directory</type> + <editable-only-at-install>true</editable-only-at-install> + <overridable>false</overridable> + </value-attributes> + </property> + <property> + <name>zk_pid_dir</name> + <display-name>ZooKeeper PID Dir</display-name> + <value>/var/run/zookeeper</value> + <description>ZooKeeper Pid Dir</description> + <value-attributes> + <type>directory</type> + <editable-only-at-install>true</editable-only-at-install> + <overridable>false</overridable> + </value-attributes> + </property> + + + <!-- zookeeper-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for zookeeper-env.sh file</description> + <value> +export JAVA_HOME={{java64_home}} +export ZOOKEEPER_HOME={{zk_home}} +export ZOO_LOG_DIR={{zk_log_dir}} +export ZOOPIDFILE={{zk_pid_file}} +export SERVER_JVMFLAGS={{zk_server_heapsize}} +export JAVA=$JAVA_HOME/bin/java +export CLASSPATH=$CLASSPATH:/usr/share/zookeeper/* + +{% if security_enabled %} +export SERVER_JVMFLAGS="$SERVER_JVMFLAGS -Djava.security.auth.login.config={{zk_server_jaas_file}}" +export CLIENT_JVMFLAGS="$CLIENT_JVMFLAGS -Djava.security.auth.login.config={{zk_client_jaas_file}}" +{% endif %} + </value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/configuration/zookeeper-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/configuration/zookeeper-log4j.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/configuration/zookeeper-log4j.xml new file mode 100755 index 0000000..6fcf5bc --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/configuration/zookeeper-log4j.xml @@ -0,0 +1,101 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_final="false"> + + <property> + <name>content</name> + <description>Custom log4j.properties</description> + <value> +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +# + +# +# ZooKeeper Logging Configuration +# + +# DEFAULT: console appender only +log4j.rootLogger=INFO, CONSOLE + +# Example with rolling log file +#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE + +# Example with rolling log file and tracing +#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE + +# +# Log INFO level and above messages to the console +# +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.Threshold=INFO +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + +# +# Add ROLLINGFILE to rootLogger to get log file output +# Log DEBUG level and above messages to a log file +log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender +log4j.appender.ROLLINGFILE.Threshold=DEBUG +log4j.appender.ROLLINGFILE.File=zookeeper.log + +# Max log file size of 10MB +log4j.appender.ROLLINGFILE.MaxFileSize=10MB +# uncomment the next line to limit number of backup files +#log4j.appender.ROLLINGFILE.MaxBackupIndex=10 + +log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout +log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + + +# +# Add TRACEFILE to rootLogger to get log file output +# Log DEBUG level and above messages to a log file +log4j.appender.TRACEFILE=org.apache.log4j.FileAppender +log4j.appender.TRACEFILE.Threshold=TRACE +log4j.appender.TRACEFILE.File=zookeeper_trace.log + +log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout +### Notice we are including log4j's NDC here (%x) +log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/kerberos.json b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/kerberos.json new file mode 100755 index 0000000..0a64ea5 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/kerberos.json @@ -0,0 +1,39 @@ +{ + "services": [ + { + "name": "ZOOKEEPER", + "identities": [ + { + "name": "/smokeuser" + } + ], + "components": [ + { + "name": "ZOOKEEPER_SERVER", + "identities": [ + { + "name": "zookeeper_zk", + "principal": { + "value": "zookeeper/_HOST@${realm}", + "type" : "service", + "configuration": "zookeeper-env/zookeeper_principal_name" + }, + "keytab": { + "file": "${keytab_dir}/zk.service.keytab", + "owner": { + "name": "${zookeeper-env/zk_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "zookeeper-env/zookeeper_keytab_path" + } + } + ] + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/metainfo.xml new file mode 100755 index 0000000..2bd2532 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/metainfo.xml @@ -0,0 +1,91 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>ZOOKEEPER</name> + <displayName>ZooKeeper</displayName> + <comment>Centralized service which provides highly reliable distributed coordination</comment> + <version>3.4.6</version> + <components> + + <component> + <name>ZOOKEEPER_SERVER</name> + <displayName>ZooKeeper Server</displayName> + <category>MASTER</category> + <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> + <commandScript> + <script>scripts/zookeeper_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>ZOOKEEPER_CLIENT</name> + <displayName>ZooKeeper Client</displayName> + <category>CLIENT</category> + <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> + <commandScript> + <script>scripts/zookeeper_client.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + <configFiles> + <configFile> + <type>env</type> + <fileName>zookeeper-env.sh</fileName> + <dictionaryName>zookeeper-env</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>log4j.properties</fileName> + <dictionaryName>zookeeper-log4j</dictionaryName> + </configFile> + </configFiles> + </component> + </components> + + <osSpecifics> + <osSpecific> + <osFamily>any</osFamily> + <packages> + <package> + <name>zookeeper</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <configuration-dependencies> + <config-type>zookeeper-log4j</config-type> + <config-type>zookeeper-env</config-type> + <config-type>zoo.cfg</config-type> + </configuration-dependencies> + <restartRequiredAfterChange>true</restartRequiredAfterChange> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/package/files/zkEnv.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/package/files/zkEnv.sh b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/package/files/zkEnv.sh new file mode 100755 index 0000000..fa1b832 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/ZOOKEEPER/package/files/zkEnv.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script should be sourced into other zookeeper +# scripts to setup the env variables + +# We use ZOOCFGDIR if defined, +# otherwise we use /etc/zookeeper +# or the conf directory that is +# a sibling of this script's directory +if [ "x$ZOOCFGDIR" = "x" ] +then + if [ -d "/etc/zookeeper" ] + then + ZOOCFGDIR="/etc/zookeeper" + else + ZOOCFGDIR="$ZOOBINDIR/../conf" + fi +fi + +if [ "x$ZOOCFG" = "x" ] +then + ZOOCFG="zoo.cfg" +fi + +ZOOCFG="$ZOOCFGDIR/$ZOOCFG" + +if [ -e "$ZOOCFGDIR/zookeeper-env.sh" ] +then + . "$ZOOCFGDIR/zookeeper-env.sh" +fi + +if [ "x${ZOO_LOG_DIR}" = "x" ] +then + ZOO_LOG_DIR="." +fi + +if [ "x${ZOO_LOG4J_PROP}" = "x" ] +then + ZOO_LOG4J_PROP="INFO,CONSOLE" +fi + +#add the zoocfg dir to classpath +CLASSPATH="$ZOOCFGDIR:$CLASSPATH" + +for i in "$ZOOBINDIR"/../src/java/lib/*.jar +do + CLASSPATH="$i:$CLASSPATH" +done + +#make it work in the release +for i in "$ZOOBINDIR"/../lib/*.jar +do + CLASSPATH="$i:$CLASSPATH" +done + +#make it work in the release +for i in "$ZOOBINDIR"/../zookeeper-*.jar +do + CLASSPATH="$i:$CLASSPATH" +done + +#make it work for developers +for d in "$ZOOBINDIR"/../build/lib/*.jar +do + CLASSPATH="$d:$CLASSPATH" +done + +#make it work for developers +CLASSPATH="$ZOOBINDIR/../build/classes:$CLASSPATH" + +case "`uname`" in + CYGWIN*) cygwin=true ;; + *) cygwin=false ;; +esac + +if $cygwin +then + CLASSPATH=`cygpath -wp "$CLASSPATH"` +fi + +#echo "CLASSPATH=$CLASSPATH"
