http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/alerts/alert_nodemanager_health.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/alerts/alert_nodemanager_health.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/alerts/alert_nodemanager_health.py deleted file mode 100644 index b1de951..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/alerts/alert_nodemanager_health.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python - -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import json -import socket -import urllib2 - -RESULT_CODE_OK = 'OK' -RESULT_CODE_CRITICAL = 'CRITICAL' -RESULT_CODE_UNKNOWN = 'UNKNOWN' - -NODEMANAGER_HTTP_ADDRESS_KEY = '{{yarn-site/yarn.nodemanager.webapp.address}}' -NODEMANAGER_HTTPS_ADDRESS_KEY = '{{yarn-site/yarn.nodemanager.webapp.https.address}}' -YARN_HTTP_POLICY_KEY = '{{yarn-site/yarn.http.policy}}' - -OK_MESSAGE = 'NodeManager Healthy' -CRITICAL_CONNECTION_MESSAGE = 'Connection failed to {0}' -CRITICAL_NODEMANAGER_STATUS_MESSAGE = 'NodeManager returned an unexpected status of "{0}"' -CRITICAL_NODEMANAGER_UNKNOWN_JSON_MESSAGE = 'Unable to determine NodeManager health from unexpected JSON response' - -NODEMANAGER_DEFAULT_PORT = 8042 - -def get_tokens(): - """ - Returns a tuple of tokens in the format {{site/property}} that will be used - to build the dictionary passed into execute - """ - return (NODEMANAGER_HTTP_ADDRESS_KEY,NODEMANAGER_HTTPS_ADDRESS_KEY, - YARN_HTTP_POLICY_KEY) - - -def execute(parameters=None, host_name=None): - """ - Returns a tuple containing the result code and a pre-formatted result label - - Keyword arguments: - parameters (dictionary): a mapping of parameter key to value - host_name (string): the name of this host where the alert is running - """ - result_code = RESULT_CODE_UNKNOWN - - if parameters is None: - return (result_code, ['There were no parameters supplied to the script.']) - - scheme = 'http' - http_uri = None - https_uri = None - http_policy = 'HTTP_ONLY' - - if NODEMANAGER_HTTP_ADDRESS_KEY in parameters: - http_uri = parameters[NODEMANAGER_HTTP_ADDRESS_KEY] - - if NODEMANAGER_HTTPS_ADDRESS_KEY in parameters: - https_uri = parameters[NODEMANAGER_HTTPS_ADDRESS_KEY] - - if YARN_HTTP_POLICY_KEY in parameters: - http_policy = parameters[YARN_HTTP_POLICY_KEY] - - # determine the right URI and whether to use SSL - uri = http_uri - if http_policy == 'HTTPS_ONLY': - scheme = 'https' - - if https_uri is not None: - uri = https_uri - - label = '' - url_response = None - node_healthy = 'false' - total_time = 0 - - # some yarn-site structures don't have the web ui address - if uri is None: - if host_name is None: - host_name = socket.getfqdn() - - uri = '{0}:{1}'.format(host_name, NODEMANAGER_DEFAULT_PORT) - - try: - query = "{0}://{1}/ws/v1/node/info".format(scheme,uri) - - # execute the query for the JSON that includes templeton status - url_response = urllib2.urlopen(query) - except: - label = CRITICAL_CONNECTION_MESSAGE.format(uri) - return (RESULT_CODE_CRITICAL, [label]) - - # URL response received, parse it - try: - json_response = json.loads(url_response.read()) - node_healthy = json_response['nodeInfo']['nodeHealthy'] - - # convert boolean to string - node_healthy = str(node_healthy) - except: - return (RESULT_CODE_CRITICAL, [query]) - - # proper JSON received, compare against known value - if node_healthy.lower() == 'true': - result_code = RESULT_CODE_OK - label = OK_MESSAGE - else: - result_code = RESULT_CODE_CRITICAL - label = CRITICAL_NODEMANAGER_STATUS_MESSAGE.format(node_healthy) - - return (result_code, [label])
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/files/validateYarnComponentStatus.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/files/validateYarnComponentStatus.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/files/validateYarnComponentStatus.py deleted file mode 100644 index 33ed8b1..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/files/validateYarnComponentStatus.py +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env python - -''' -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -''' - -import optparse -import subprocess -import json - -RESOURCEMANAGER = 'rm' -NODEMANAGER = 'nm' -HISTORYSERVER = 'hs' - -STARTED_STATE = 'STARTED' -RUNNING_STATE = 'RUNNING' - -#Return reponse for given path and address -def getResponse(path, address, ssl_enabled): - - command = "curl" - httpGssnegotiate = "--negotiate" - userpswd = "-u:" - insecure = "-k"# This is smoke test, no need to check CA of server - if ssl_enabled: - url = 'https://' + address + path - else: - url = 'http://' + address + path - - command_with_flags = [command,httpGssnegotiate,userpswd,insecure,url] - - proc = subprocess.Popen(command_with_flags, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = proc.communicate() - response = json.loads(stdout) - if response == None: - print 'There is no response for url: ' + str(url) - raise Exception('There is no response for url: ' + str(url)) - return response - -#Verify that REST api is available for given component -def validateAvailability(component, path, addresses, ssl_enabled): - responses = {} - for address in addresses.split(','): - try: - responses[address] = getResponse(path, address, ssl_enabled) - except Exception as e: - print 'Error checking availability status of component.', e - - if not responses: - exit(1) - - is_valid = validateAvailabilityResponse(component, responses.values()[0]) - if not is_valid: - exit(1) - -#Validate component-specific response -def validateAvailabilityResponse(component, response): - try: - if component == RESOURCEMANAGER: - rm_state = response['clusterInfo']['state'] - if rm_state == STARTED_STATE: - return True - else: - print 'Resourcemanager is not started' - return False - - elif component == NODEMANAGER: - node_healthy = bool(response['nodeInfo']['nodeHealthy']) - if node_healthy: - return True - else: - return False - elif component == HISTORYSERVER: - hs_start_time = response['historyInfo']['startedOn'] - if hs_start_time > 0: - return True - else: - return False - else: - return False - except Exception as e: - print 'Error validation of availability response for ' + str(component), e - return False - -#Verify that component has required resources to work -def validateAbility(component, path, addresses, ssl_enabled): - responses = {} - for address in addresses.split(','): - try: - responses[address] = getResponse(path, address, ssl_enabled) - except Exception as e: - print 'Error checking ability of component.', e - - if not responses: - exit(1) - - is_valid = validateAbilityResponse(component, responses.values()[0]) - if not is_valid: - exit(1) - -#Validate component-specific response that it has required resources to work -def validateAbilityResponse(component, response): - try: - if component == RESOURCEMANAGER: - nodes = [] - if response.has_key('nodes') and not response['nodes'] == None and response['nodes'].has_key('node'): - nodes = response['nodes']['node'] - connected_nodes_count = len(nodes) - if connected_nodes_count == 0: - print 'There is no connected nodemanagers to resourcemanager' - return False - active_nodes = filter(lambda x: x['state'] == RUNNING_STATE, nodes) - active_nodes_count = len(active_nodes) - - if connected_nodes_count == 0: - print 'There is no connected active nodemanagers to resourcemanager' - return False - else: - return True - else: - return False - except Exception as e: - print 'Error validation of ability response', e - return False - -# -# Main. -# -def main(): - parser = optparse.OptionParser(usage="usage: %prog [options] component ") - parser.add_option("-p", "--port", dest="address", help="Host:Port for REST API of a desired component") - parser.add_option("-s", "--ssl", dest="ssl_enabled", help="Is SSL enabled for UI of component") - - (options, args) = parser.parse_args() - - component = args[0] - - address = options.address - ssl_enabled = (options.ssl_enabled) in 'true' - if component == RESOURCEMANAGER: - path = '/ws/v1/cluster/info' - elif component == NODEMANAGER: - path = '/ws/v1/node/info' - elif component == HISTORYSERVER: - path = '/ws/v1/history/info' - else: - parser.error("Invalid component") - - validateAvailability(component, path, address, ssl_enabled) - - if component == RESOURCEMANAGER: - path = '/ws/v1/cluster/nodes' - validateAbility(component, path, address, ssl_enabled) - -if __name__ == "__main__": - main() http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/__init__.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/__init__.py deleted file mode 100644 index 35de4bb..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/application_timeline_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/application_timeline_server.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/application_timeline_server.py deleted file mode 100644 index e9c0fa1..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/application_timeline_server.py +++ /dev/null @@ -1,132 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management import * -from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version -from resource_management.libraries.functions.security_commons import build_expectations, \ - cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties,\ - FILE_TYPE_XML -from resource_management.libraries.functions.format import format - -from yarn import yarn -from service import service - -class ApplicationTimelineServer(Script): - - def install(self, env): - self.install_packages(env) - #self.configure(env) - - def configure(self, env): - import params - env.set_params(params) - yarn(name='apptimelineserver') - - def pre_rolling_restart(self, env): - Logger.info("Executing Rolling Upgrade pre-restart") - import params - env.set_params(params) - - if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: - Execute(format("hdp-select set hadoop-yarn-timelineserver {version}")) - - def start(self, env, rolling_restart=False): - import params - env.set_params(params) - self.configure(env) # FOR SECURITY - service('timelineserver', action='start') - - def stop(self, env, rolling_restart=False): - import params - env.set_params(params) - service('timelineserver', action='stop') - - def status(self, env): - import status_params - env.set_params(status_params) - Execute(format("mv {yarn_historyserver_pid_file_old} {yarn_historyserver_pid_file}"), - only_if = format("test -e {yarn_historyserver_pid_file_old}", user=status_params.yarn_user)) - functions.check_process_status(status_params.yarn_historyserver_pid_file) - - def security_status(self, env): - import status_params - env.set_params(status_params) - - props_value_check = {"yarn.timeline-service.enabled": "true", - "yarn.timeline-service.http-authentication.type": "kerberos", - "yarn.acl.enable": "true"} - props_empty_check = ["yarn.timeline-service.principal", - "yarn.timeline-service.keytab", - "yarn.timeline-service.http-authentication.kerberos.principal", - "yarn.timeline-service.http-authentication.kerberos.keytab"] - - props_read_check = ["yarn.timeline-service.keytab", - "yarn.timeline-service.http-authentication.kerberos.keytab"] - yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check, - props_read_check) - - yarn_expectations ={} - yarn_expectations.update(yarn_site_props) - - security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, - {'yarn-site.xml': FILE_TYPE_XML}) - result_issues = validate_security_config_properties(security_params, yarn_expectations) - if not result_issues: # If all validations passed successfully - try: - # Double check the dict before calling execute - if ( 'yarn-site' not in security_params - or 'yarn.timeline-service.keytab' not in security_params['yarn-site'] - or 'yarn.timeline-service.principal' not in security_params['yarn-site']) \ - or 'yarn.timeline-service.http-authentication.kerberos.keytab' not in security_params['yarn-site'] \ - or 'yarn.timeline-service.http-authentication.kerberos.principal' not in security_params['yarn-site']: - self.put_structured_out({"securityState": "UNSECURED"}) - self.put_structured_out( - {"securityIssuesFound": "Keytab file or principal are not set property."}) - return - - cached_kinit_executor(status_params.kinit_path_local, - status_params.yarn_user, - security_params['yarn-site']['yarn.timeline-service.keytab'], - security_params['yarn-site']['yarn.timeline-service.principal'], - status_params.hostname, - status_params.tmp_dir, - 30) - cached_kinit_executor(status_params.kinit_path_local, - status_params.yarn_user, - security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.keytab'], - security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.principal'], - status_params.hostname, - status_params.tmp_dir, - 30) - self.put_structured_out({"securityState": "SECURED_KERBEROS"}) - except Exception as e: - self.put_structured_out({"securityState": "ERROR"}) - self.put_structured_out({"securityStateErrorInfo": str(e)}) - else: - issues = [] - for cf in result_issues: - issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) - self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) - self.put_structured_out({"securityState": "UNSECURED"}) - - -if __name__ == "__main__": - ApplicationTimelineServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/historyserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/historyserver.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/historyserver.py deleted file mode 100644 index 66c2143..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/historyserver.py +++ /dev/null @@ -1,123 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management import * -from resource_management.libraries.functions.dynamic_variable_interpretation import copy_tarballs_to_hdfs -from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version -from resource_management.libraries.functions.format import format -from resource_management.libraries.functions.security_commons import build_expectations, \ - cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ - FILE_TYPE_XML - -from yarn import yarn -from service import service - -class HistoryServer(Script): - def install(self, env): - self.install_packages(env) - - def configure(self, env): - import params - env.set_params(params) - yarn(name="historyserver") - - def pre_rolling_restart(self, env): - Logger.info("Executing Rolling Upgrade pre-restart") - import params - env.set_params(params) - - if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: - Execute(format("hdp-select set hadoop-mapreduce-historyserver {version}")) - copy_tarballs_to_hdfs('mapreduce', params.mapred_user, params.hdfs_user, params.user_group) - - def start(self, env, rolling_restart=False): - import params - env.set_params(params) - self.configure(env) # FOR SECURITY - copy_tarballs_to_hdfs('mapreduce', params.mapred_user, params.hdfs_user, params.user_group) - service('historyserver', action='start', serviceName='mapreduce') - - def stop(self, env, rolling_restart=False): - import params - env.set_params(params) - service('historyserver', action='stop', serviceName='mapreduce') - - def status(self, env): - import status_params - env.set_params(status_params) - check_process_status(status_params.mapred_historyserver_pid_file) - - def security_status(self, env): - import status_params - env.set_params(status_params) - - expectations = {} - expectations.update(build_expectations('mapred-site', - None, - [ - 'mapreduce.jobhistory.keytab', - 'mapreduce.jobhistory.principal', - 'mapreduce.jobhistory.webapp.spnego-keytab-file', - 'mapreduce.jobhistory.webapp.spnego-principal' - ], - None)) - - security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, - {'mapred-site.xml': FILE_TYPE_XML}) - result_issues = validate_security_config_properties(security_params, expectations) - if not result_issues: # If all validations passed successfully - try: - # Double check the dict before calling execute - if ( 'mapred-site' not in security_params or - 'mapreduce.jobhistory.keytab' not in security_params['mapred-site'] or - 'mapreduce.jobhistory.principal' not in security_params['mapred-site'] or - 'mapreduce.jobhistory.webapp.spnego-keytab-file' not in security_params['mapred-site'] or - 'mapreduce.jobhistory.webapp.spnego-principal' not in security_params['mapred-site']): - self.put_structured_out({"securityState": "UNSECURED"}) - self.put_structured_out( - {"securityIssuesFound": "Keytab file or principal not set."}) - return - - cached_kinit_executor(status_params.kinit_path_local, - status_params.mapred_user, - security_params['mapred-site']['mapreduce.jobhistory.keytab'], - security_params['mapred-site']['mapreduce.jobhistory.principal'], - status_params.hostname, - status_params.tmp_dir) - cached_kinit_executor(status_params.kinit_path_local, - status_params.mapred_user, - security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-keytab-file'], - security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-principal'], - status_params.hostname, - status_params.tmp_dir) - self.put_structured_out({"securityState": "SECURED_KERBEROS"}) - except Exception as e: - self.put_structured_out({"securityState": "ERROR"}) - self.put_structured_out({"securityStateErrorInfo": str(e)}) - else: - issues = [] - for cf in result_issues: - issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) - self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) - self.put_structured_out({"securityState": "UNSECURED"}) - -if __name__ == "__main__": - HistoryServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/mapred_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/mapred_service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/mapred_service_check.py deleted file mode 100644 index a9f4367..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/mapred_service_check.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management import * - -class MapReduce2ServiceCheck(Script): - def service_check(self, env): - import params - env.set_params(params) - - jar_path = format("{hadoop_mapred2_jar_location}/{hadoopMapredExamplesJarName}") - input_file = format("/user/{smokeuser}/mapredsmokeinput") - output_file = format("/user/{smokeuser}/mapredsmokeoutput") - - cleanup_cmd = format("fs -rm -r -f {output_file} {input_file}") - create_file_cmd = format("fs -put /etc/passwd {input_file}") - test_cmd = format("fs -test -e {output_file}") - run_wordcount_job = format("jar {jar_path} wordcount {input_file} {output_file}") - - if params.security_enabled: - kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser};") - - Execute(kinit_cmd, - user=params.smokeuser - ) - - ExecuteHadoop(cleanup_cmd, - tries=1, - try_sleep=5, - user=params.smokeuser, - bin_dir=params.execute_path, - conf_dir=params.hadoop_conf_dir - ) - - ExecuteHadoop(create_file_cmd, - tries=1, - try_sleep=5, - user=params.smokeuser, - bin_dir=params.execute_path, - conf_dir=params.hadoop_conf_dir - ) - - ExecuteHadoop(run_wordcount_job, - tries=1, - try_sleep=5, - user=params.smokeuser, - bin_dir=params.execute_path, - conf_dir=params.hadoop_conf_dir, - logoutput=True - ) - - ExecuteHadoop(test_cmd, - user=params.smokeuser, - bin_dir=params.execute_path, - conf_dir=params.hadoop_conf_dir - ) - -if __name__ == "__main__": - MapReduce2ServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/mapreduce2_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/mapreduce2_client.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/mapreduce2_client.py deleted file mode 100644 index c4e2800..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/mapreduce2_client.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -import sys -from resource_management import * - -from yarn import yarn - -class MapReduce2Client(Script): - - def pre_rolling_restart(self, env): - import params - env.set_params(params) - - if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: - Execute(format("hdp-select set hadoop-client {version}")) - - def install(self, env): - self.install_packages(env) - self.configure(env) - - def configure(self, env): - import params - env.set_params(params) - yarn() - - def status(self, env): - raise ClientComponentHasNoStatus() - -if __name__ == "__main__": - MapReduce2Client().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager.py deleted file mode 100644 index 87be214..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager.py +++ /dev/null @@ -1,137 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -import nodemanager_upgrade - -from resource_management import * -from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version -from resource_management.libraries.functions.format import format -from resource_management.libraries.functions.security_commons import build_expectations, \ - cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ - FILE_TYPE_XML - -from yarn import yarn -from service import service - -class Nodemanager(Script): - def install(self, env): - self.install_packages(env) - - def configure(self, env): - import params - env.set_params(params) - yarn(name="nodemanager") - - def pre_rolling_restart(self, env): - Logger.info("Executing NodeManager Rolling Upgrade pre-restart") - import params - env.set_params(params) - - if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: - Execute(format("hdp-select set hadoop-yarn-nodemanager {version}")) - - def start(self, env, rolling_restart=False): - import params - env.set_params(params) - self.configure(env) # FOR SECURITY - service('nodemanager',action='start') - - def post_rolling_restart(self, env): - Logger.info("Executing NodeManager Rolling Upgrade post-restart") - import params - env.set_params(params) - - nodemanager_upgrade.post_upgrade_check() - - def stop(self, env, rolling_restart=False): - import params - env.set_params(params) - - service('nodemanager',action='stop') - - def status(self, env): - import status_params - env.set_params(status_params) - check_process_status(status_params.nodemanager_pid_file) - - def security_status(self, env): - import status_params - env.set_params(status_params) - - props_value_check = {"yarn.timeline-service.enabled": "true", - "yarn.timeline-service.http-authentication.type": "kerberos", - "yarn.acl.enable": "true"} - props_empty_check = ["yarn.nodemanager.principal", - "yarn.nodemanager.keytab", - "yarn.nodemanager.webapp.spnego-principal", - "yarn.nodemanager.webapp.spnego-keytab-file"] - - props_read_check = ["yarn.nodemanager.keytab", - "yarn.nodemanager.webapp.spnego-keytab-file"] - yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check, - props_read_check) - - yarn_expectations ={} - yarn_expectations.update(yarn_site_props) - - security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, - {'yarn-site.xml': FILE_TYPE_XML}) - result_issues = validate_security_config_properties(security_params, yarn_site_props) - if not result_issues: # If all validations passed successfully - try: - # Double check the dict before calling execute - if ( 'yarn-site' not in security_params - or 'yarn.nodemanager.keytab' not in security_params['yarn-site'] - or 'yarn.nodemanager.principal' not in security_params['yarn-site']) \ - or 'yarn.nodemanager.webapp.spnego-keytab-file' not in security_params['yarn-site'] \ - or 'yarn.nodemanager.webapp.spnego-principal' not in security_params['yarn-site']: - self.put_structured_out({"securityState": "UNSECURED"}) - self.put_structured_out( - {"securityIssuesFound": "Keytab file or principal are not set property."}) - return - - cached_kinit_executor(status_params.kinit_path_local, - status_params.yarn_user, - security_params['yarn-site']['yarn.nodemanager.keytab'], - security_params['yarn-site']['yarn.nodemanager.principal'], - status_params.hostname, - status_params.tmp_dir, - 30) - cached_kinit_executor(status_params.kinit_path_local, - status_params.yarn_user, - security_params['yarn-site']['yarn.nodemanager.webapp.spnego-keytab-file'], - security_params['yarn-site']['yarn.nodemanager.webapp.spnego-principal'], - status_params.hostname, - status_params.tmp_dir, - 30) - self.put_structured_out({"securityState": "SECURED_KERBEROS"}) - except Exception as e: - self.put_structured_out({"securityState": "ERROR"}) - self.put_structured_out({"securityStateErrorInfo": str(e)}) - else: - issues = [] - for cf in result_issues: - issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) - self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) - self.put_structured_out({"securityState": "UNSECURED"}) - -if __name__ == "__main__": - Nodemanager().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager_upgrade.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager_upgrade.py deleted file mode 100644 index 54e0fae..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager_upgrade.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import subprocess - -from resource_management.core.logger import Logger -from resource_management.core.exceptions import Fail -from resource_management.core.resources.system import Execute -from resource_management.core import shell -from resource_management.libraries.functions.decorator import retry - - -def post_upgrade_check(): - ''' - Checks that the NodeManager has rejoined the cluster. - This function will obtain the Kerberos ticket if security is enabled. - :return: - ''' - import params - - Logger.info('NodeManager executing "yarn node -list -states=RUNNING" to verify the node has rejoined the cluster...') - if params.security_enabled and params.nodemanager_kinit_cmd: - Execute(params.nodemanager_kinit_cmd, user = params.yarn_user) - - _check_nodemanager_startup() - - -@retry(times=12, sleep_time=10, err_class=Fail) -def _check_nodemanager_startup(): - ''' - Checks that a NodeManager is in a RUNNING state in the cluster via - "yarn node -list -states=RUNNING" command. Once the NodeManager is found to be - alive this method will return, otherwise it will raise a Fail(...) and retry - automatically. - :return: - ''' - import params - - command = 'yarn node -list -states=RUNNING' - - try: - # 'su - yarn -c "yarn node -status c6401.ambari.apache.org:45454"' - return_code, yarn_output = shell.call(command, user=params.hdfs_user) - except: - raise Fail('Unable to determine if the NodeManager has started after upgrade.') - - if return_code == 0: - hostname = params.hostname.lower() - nodemanager_address = params.nm_address.lower() - yarn_output = yarn_output.lower() - - if hostname in yarn_output or nodemanager_address in yarn_output: - Logger.info('NodeManager with ID {0} has rejoined the cluster.'.format(nodemanager_address)) - return - else: - raise Fail('NodeManager with ID {0} was not found in the list of running NodeManagers'.format(nodemanager_address)) - - raise Fail('Unable to determine if the NodeManager has started after upgrade (result code {0})'.format(str(return_code))) http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py deleted file mode 100644 index d9c73bf..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py +++ /dev/null @@ -1,209 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" -import os -from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions -from resource_management.libraries.functions.default import default -from resource_management import * -import status_params - -# server configurations -config = Script.get_config() -tmp_dir = Script.get_tmp_dir() - -# This is expected to be of the form #.#.#.# -stack_version_unformatted = str(config['hostLevelParams']['stack_version']) -hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) - -# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade -version = default("/commandParams/version", None) - -hostname = config['hostname'] - -#hadoop params -if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0: - yarn_role_root = "hadoop-yarn-client" - mapred_role_root = "hadoop-mapreduce-client" - - command_role = default("/role", "") - if command_role == "APP_TIMELINE_SERVER": - yarn_role_root = "hadoop-yarn-timelineserver" - elif command_role == "HISTORYSERVER": - mapred_role_root = "hadoop-mapreduce-historyserver" - elif command_role == "MAPREDUCE2_CLIENT": - mapred_role_root = "hadoop-mapreduce-client" - elif command_role == "NODEMANAGER": - yarn_role_root = "hadoop-yarn-nodemanager" - elif command_role == "RESOURCEMANAGER": - yarn_role_root = "hadoop-yarn-resourcemanager" - elif command_role == "YARN_CLIENT": - yarn_role_root = "hadoop-yarn-client" - - hadoop_libexec_dir = "/usr/hdp/current/hadoop-client/libexec" - hadoop_bin = "/usr/hdp/current/hadoop-client/sbin" - hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin" - - hadoop_mapred2_jar_location = format("/usr/hdp/current/{mapred_role_root}") - mapred_bin = format("/usr/hdp/current/{mapred_role_root}/sbin") - - hadoop_yarn_home = format("/usr/hdp/current/{yarn_role_root}") - yarn_bin = format("/usr/hdp/current/{yarn_role_root}/sbin") - yarn_container_bin = format("/usr/hdp/current/{yarn_role_root}/bin") -else: - hadoop_libexec_dir = "/usr/lib/hadoop/libexec" - hadoop_bin = "/usr/lib/hadoop/sbin" - hadoop_bin_dir = "/usr/bin" - hadoop_yarn_home = '/usr/lib/hadoop-yarn' - hadoop_mapred2_jar_location = "/usr/lib/hadoop-mapreduce" - mapred_bin = "/usr/lib/hadoop-mapreduce/sbin" - yarn_bin = "/usr/lib/hadoop-yarn/sbin" - yarn_container_bin = "/usr/lib/hadoop-yarn/bin" - -hadoop_conf_dir = "/etc/hadoop/conf" -limits_conf_dir = "/etc/security/limits.d" -execute_path = os.environ['PATH'] + os.pathsep + hadoop_bin_dir + os.pathsep + yarn_container_bin - -ulimit_cmd = "ulimit -c unlimited;" - -mapred_user = status_params.mapred_user -yarn_user = status_params.yarn_user -hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] - -smokeuser = config['configurations']['cluster-env']['smokeuser'] -security_enabled = config['configurations']['cluster-env']['security_enabled'] -smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] -yarn_executor_container_group = config['configurations']['yarn-site']['yarn.nodemanager.linux-container-executor.group'] -kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) -rm_hosts = config['clusterHostInfo']['rm_host'] -rm_host = rm_hosts[0] -rm_port = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'].split(':')[-1] -rm_https_port = "8090" -# TODO UPGRADE default, update site during upgrade -rm_nodes_exclude_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") - -java64_home = config['hostLevelParams']['java_home'] -hadoop_ssl_enabled = default("/configurations/core-site/hadoop.ssl.enabled", False) - -yarn_heapsize = config['configurations']['yarn-env']['yarn_heapsize'] -resourcemanager_heapsize = config['configurations']['yarn-env']['resourcemanager_heapsize'] -nodemanager_heapsize = config['configurations']['yarn-env']['nodemanager_heapsize'] -apptimelineserver_heapsize = default("/configurations/yarn-env/apptimelineserver_heapsize", 1024) -ats_leveldb_dir = config['configurations']['yarn-site']['yarn.timeline-service.leveldb-timeline-store.path'] -yarn_log_dir_prefix = config['configurations']['yarn-env']['yarn_log_dir_prefix'] -yarn_pid_dir_prefix = status_params.yarn_pid_dir_prefix -mapred_pid_dir_prefix = status_params.mapred_pid_dir_prefix -mapred_log_dir_prefix = config['configurations']['mapred-env']['mapred_log_dir_prefix'] -mapred_env_sh_template = config['configurations']['mapred-env']['content'] -yarn_env_sh_template = config['configurations']['yarn-env']['content'] - -if len(rm_hosts) > 1: - additional_rm_host = rm_hosts[1] - rm_webui_address = format("{rm_host}:{rm_port},{additional_rm_host}:{rm_port}") - rm_webui_https_address = format("{rm_host}:{rm_https_port},{additional_rm_host}:{rm_https_port}") -else: - rm_webui_address = format("{rm_host}:{rm_port}") - rm_webui_https_address = format("{rm_host}:{rm_https_port}") - -nm_webui_address = config['configurations']['yarn-site']['yarn.nodemanager.webapp.address'] -hs_webui_address = config['configurations']['mapred-site']['mapreduce.jobhistory.webapp.address'] -nm_address = config['configurations']['yarn-site']['yarn.nodemanager.address'] # still contains 0.0.0.0 -if hostname and nm_address and nm_address.startswith("0.0.0.0:"): - nm_address = nm_address.replace("0.0.0.0", hostname) - -nm_local_dirs = config['configurations']['yarn-site']['yarn.nodemanager.local-dirs'] -nm_log_dirs = config['configurations']['yarn-site']['yarn.nodemanager.log-dirs'] - -distrAppJarName = "hadoop-yarn-applications-distributedshell-2.*.jar" -hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar" - -yarn_pid_dir = status_params.yarn_pid_dir -mapred_pid_dir = status_params.mapred_pid_dir - -mapred_log_dir = format("{mapred_log_dir_prefix}/{mapred_user}") -yarn_log_dir = format("{yarn_log_dir_prefix}/{yarn_user}") -mapred_job_summary_log = format("{mapred_log_dir_prefix}/{mapred_user}/hadoop-mapreduce.jobsummary.log") -yarn_job_summary_log = format("{yarn_log_dir_prefix}/{yarn_user}/hadoop-mapreduce.jobsummary.log") - -user_group = config['configurations']['cluster-env']['user_group'] - -#exclude file -exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) -exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") - -ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", [])) -has_ats = not len(ats_host) == 0 - -# default kinit commands -rm_kinit_cmd = "" -yarn_timelineservice_kinit_cmd = "" -nodemanager_kinit_cmd = "" - -if security_enabled: - _rm_principal_name = config['configurations']['yarn-site']['yarn.resourcemanager.principal'] - _rm_principal_name = _rm_principal_name.replace('_HOST',hostname.lower()) - _rm_keytab = config['configurations']['yarn-site']['yarn.resourcemanager.keytab'] - rm_kinit_cmd = format("{kinit_path_local} -kt {_rm_keytab} {_rm_principal_name};") - - # YARN timeline security options are only available in HDP Champlain - if has_ats: - _yarn_timelineservice_principal_name = config['configurations']['yarn-site']['yarn.timeline-service.principal'] - _yarn_timelineservice_principal_name = _yarn_timelineservice_principal_name.replace('_HOST', hostname.lower()) - _yarn_timelineservice_keytab = config['configurations']['yarn-site']['yarn.timeline-service.keytab'] - yarn_timelineservice_kinit_cmd = format("{kinit_path_local} -kt {_yarn_timelineservice_keytab} {_yarn_timelineservice_principal_name};") - - if 'yarn.nodemanager.principal' in config['configurations']['yarn-site']: - _nodemanager_principal_name = config['configurations']['yarn-site']['yarn.nodemanager.principal'] - _nodemanager_keytab = config['configurations']['yarn-site']['yarn.nodemanager.keytab'] - nodemanager_kinit_cmd = format("{kinit_path_local} -kt {_nodemanager_keytab} {_nodemanager_principal_name};") - - -yarn_log_aggregation_enabled = config['configurations']['yarn-site']['yarn.log-aggregation-enable'] -yarn_nm_app_log_dir = config['configurations']['yarn-site']['yarn.nodemanager.remote-app-log-dir'] -mapreduce_jobhistory_intermediate_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.intermediate-done-dir'] -mapreduce_jobhistory_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.done-dir'] -jobhistory_heapsize = default("/configurations/mapred-env/jobhistory_heapsize", "900") - -#for create_hdfs_directory -hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] -hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] -import functools -#create partial functions with common arguments for every HdfsDirectory call -#to create hdfs directory we need to call params.HdfsDirectory in code -HdfsDirectory = functools.partial( - HdfsDirectory, - conf_dir=hadoop_conf_dir, - hdfs_user=hdfs_user, - security_enabled = security_enabled, - keytab = hdfs_user_keytab, - kinit_path_local = kinit_path_local, - bin_dir = hadoop_bin_dir -) -update_exclude_file_only = default("/commandParams/update_exclude_file_only",False) - -mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group) - -#taskcontroller.cfg - -mapred_local_dir = "/tmp/hadoop-mapred/mapred/local" -hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] -min_user_id = config['configurations']['yarn-env']['min_user_id'] - -# Node labels -node_labels_dir = default("/configurations/yarn-site/yarn.node-labels.fs-store.root-dir", None) http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py deleted file mode 100644 index 3d6a00e..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py +++ /dev/null @@ -1,173 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management import * -from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version -from resource_management.libraries.functions.security_commons import build_expectations, \ - cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ - FILE_TYPE_XML - -from yarn import yarn -from service import service - - -class Resourcemanager(Script): - def install(self, env): - self.install_packages(env) - - def configure(self, env): - import params - - env.set_params(params) - yarn(name='resourcemanager') - - def pre_rolling_restart(self, env): - Logger.info("Executing Rolling Upgrade post-restart") - import params - env.set_params(params) - - if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: - Execute(format("hdp-select set hadoop-yarn-resourcemanager {version}")) - - def start(self, env, rolling_restart=False): - import params - - env.set_params(params) - self.configure(env) # FOR SECURITY - service('resourcemanager', - action='start' - ) - - def stop(self, env, rolling_restart=False): - import params - - env.set_params(params) - - service('resourcemanager', - action='stop' - ) - - def status(self, env): - import status_params - - env.set_params(status_params) - check_process_status(status_params.resourcemanager_pid_file) - pass - - def security_status(self, env): - import status_params - env.set_params(status_params) - - props_value_check = {"yarn.timeline-service.enabled": "true", - "yarn.timeline-service.http-authentication.type": "kerberos", - "yarn.acl.enable": "true"} - props_empty_check = ["yarn.resourcemanager.principal", - "yarn.resourcemanager.keytab", - "yarn.resourcemanager.webapp.spnego-principal", - "yarn.resourcemanager.webapp.spnego-keytab-file"] - - props_read_check = ["yarn.resourcemanager.keytab", - "yarn.resourcemanager.webapp.spnego-keytab-file"] - yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check, - props_read_check) - - yarn_expectations ={} - yarn_expectations.update(yarn_site_props) - - security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, - {'yarn-site.xml': FILE_TYPE_XML}) - result_issues = validate_security_config_properties(security_params, yarn_site_props) - if not result_issues: # If all validations passed successfully - try: - # Double check the dict before calling execute - if ( 'yarn-site' not in security_params - or 'yarn.resourcemanager.keytab' not in security_params['yarn-site'] - or 'yarn.resourcemanager.principal' not in security_params['yarn-site']) \ - or 'yarn.resourcemanager.webapp.spnego-keytab-file' not in security_params['yarn-site'] \ - or 'yarn.resourcemanager.webapp.spnego-principal' not in security_params['yarn-site']: - self.put_structured_out({"securityState": "UNSECURED"}) - self.put_structured_out( - {"securityIssuesFound": "Keytab file or principal are not set property."}) - return - - cached_kinit_executor(status_params.kinit_path_local, - status_params.yarn_user, - security_params['yarn-site']['yarn.resourcemanager.keytab'], - security_params['yarn-site']['yarn.resourcemanager.principal'], - status_params.hostname, - status_params.tmp_dir, - 30) - cached_kinit_executor(status_params.kinit_path_local, - status_params.yarn_user, - security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-keytab-file'], - security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-principal'], - status_params.hostname, - status_params.tmp_dir, - 30) - self.put_structured_out({"securityState": "SECURED_KERBEROS"}) - except Exception as e: - self.put_structured_out({"securityState": "ERROR"}) - self.put_structured_out({"securityStateErrorInfo": str(e)}) - else: - issues = [] - for cf in result_issues: - issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) - self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) - self.put_structured_out({"securityState": "UNSECURED"}) - - - def refreshqueues(self, env): - import params - - self.configure(env) - env.set_params(params) - - service('resourcemanager', - action='refreshQueues' - ) - - def decommission(self, env): - import params - - env.set_params(params) - rm_kinit_cmd = params.rm_kinit_cmd - yarn_user = params.yarn_user - conf_dir = params.hadoop_conf_dir - user_group = params.user_group - - yarn_refresh_cmd = format("{rm_kinit_cmd} yarn --config {conf_dir} rmadmin -refreshNodes") - - File(params.exclude_file_path, - content=Template("exclude_hosts_list.j2"), - owner=yarn_user, - group=user_group - ) - - if params.update_exclude_file_only == False: - Execute(yarn_refresh_cmd, - environment= {'PATH' : params.execute_path }, - user=yarn_user) - pass - pass - - -if __name__ == "__main__": - Resourcemanager().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/service.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/service.py deleted file mode 100644 index 1002094..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/service.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management import * - - -def service(componentName, action='start', serviceName='yarn'): - - import params - - if serviceName == 'mapreduce' and componentName == 'historyserver': - daemon = format("{mapred_bin}/mr-jobhistory-daemon.sh") - pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-{componentName}.pid") - usr = params.mapred_user - else: - daemon = format("{yarn_bin}/yarn-daemon.sh") - pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-{componentName}.pid") - usr = params.yarn_user - - cmd = format("export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --config {hadoop_conf_dir}") - - if action == 'start': - daemon_cmd = format("{ulimit_cmd} {cmd} start {componentName}") - check_process = format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1") - - # Remove the pid file if its corresponding process is not running. - File(pid_file, - action="delete", - not_if=check_process) - - # Attempt to start the process. Internally, this is skipped if the process is already running. - Execute(daemon_cmd, - user=usr, - not_if=check_process - ) - - # Ensure that the process with the expected PID exists. - Execute(check_process, - user=usr, - not_if=check_process, - initial_wait=5 - ) - - elif action == 'stop': - daemon_cmd = format("{cmd} stop {componentName}") - Execute(daemon_cmd, - user=usr) - - File(pid_file, - action="delete") - - elif action == 'refreshQueues': - rm_kinit_cmd = params.rm_kinit_cmd - refresh_cmd = format("{rm_kinit_cmd} export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {yarn_container_bin}/yarn rmadmin -refreshQueues") - - Execute(refresh_cmd, - user=usr, - ) http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/service_check.py deleted file mode 100644 index 7189664..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/service_check.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management import * -import sys - -class ServiceCheck(Script): - def service_check(self, env): - import params - env.set_params(params) - - run_yarn_check_cmd = format("yarn --config {hadoop_conf_dir} node -list") - - component_type = 'rm' - if params.hadoop_ssl_enabled: - component_address = params.rm_webui_https_address - else: - component_address = params.rm_webui_address - - validateStatusFileName = "validateYarnComponentStatus.py" - validateStatusFilePath = format("{tmp_dir}/{validateStatusFileName}") - python_executable = sys.executable - validateStatusCmd = format("{python_executable} {validateStatusFilePath} {component_type} -p {component_address} -s {hadoop_ssl_enabled}") - - if params.security_enabled: - kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser};") - smoke_cmd = format("{kinit_cmd} {validateStatusCmd}") - else: - smoke_cmd = validateStatusCmd - - File(validateStatusFilePath, - content=StaticFile(validateStatusFileName), - mode=0755 - ) - - Execute(smoke_cmd, - tries=3, - try_sleep=5, - path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', - user=params.smokeuser, - logoutput=True - ) - - Execute(run_yarn_check_cmd, - path=params.execute_path, - user=params.smokeuser - ) - -if __name__ == "__main__": - ServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/status_params.py deleted file mode 100644 index 3cfff05..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/status_params.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * - -config = Script.get_config() -tmp_dir = Script.get_tmp_dir() - -mapred_user = config['configurations']['mapred-env']['mapred_user'] -yarn_user = config['configurations']['yarn-env']['yarn_user'] -yarn_pid_dir_prefix = config['configurations']['yarn-env']['yarn_pid_dir_prefix'] -mapred_pid_dir_prefix = config['configurations']['mapred-env']['mapred_pid_dir_prefix'] -yarn_pid_dir = format("{yarn_pid_dir_prefix}/{yarn_user}") -mapred_pid_dir = format("{mapred_pid_dir_prefix}/{mapred_user}") - -resourcemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-resourcemanager.pid") -nodemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-nodemanager.pid") -yarn_historyserver_pid_file_old = format("{yarn_pid_dir}/yarn-{yarn_user}-historyserver.pid") -yarn_historyserver_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-timelineserver.pid") # *-historyserver.pid is deprecated -mapred_historyserver_pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-historyserver.pid") - -# Security related/required params -hadoop_conf_dir = "/etc/hadoop/conf" -hostname = config['hostname'] -kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) -security_enabled = config['configurations']['cluster-env']['security_enabled'] http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py deleted file mode 100644 index cf0d211..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py +++ /dev/null @@ -1,249 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management import * -import sys -import os - - -def yarn(name = None): - import params - - - if name in ["nodemanager","historyserver"]: - if params.yarn_log_aggregation_enabled: - params.HdfsDirectory(params.yarn_nm_app_log_dir, - action="create_delayed", - owner=params.yarn_user, - group=params.user_group, - mode=0777, - recursive_chmod=True - ) - params.HdfsDirectory("/mapred", - action="create_delayed", - owner=params.mapred_user - ) - params.HdfsDirectory("/mapred/system", - action="create_delayed", - owner=params.hdfs_user - ) - params.HdfsDirectory(params.mapreduce_jobhistory_intermediate_done_dir, - action="create_delayed", - owner=params.mapred_user, - group=params.user_group, - mode=0777 - ) - - params.HdfsDirectory(params.mapreduce_jobhistory_done_dir, - action="create_delayed", - owner=params.mapred_user, - group=params.user_group, - mode=01777 - ) - params.HdfsDirectory(None, action="create") - - if name == "nodemanager": - Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','), - owner=params.yarn_user, - group=params.user_group, - recursive=True, - recursive_permission=True, - ignore_failures=True, - mode=0775 - ) - - Directory([params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir], - owner=params.yarn_user, - group=params.user_group, - recursive=True - ) - - Directory([params.mapred_pid_dir_prefix, params.mapred_pid_dir, params.mapred_log_dir_prefix, params.mapred_log_dir], - owner=params.mapred_user, - group=params.user_group, - recursive=True - ) - Directory([params.yarn_log_dir_prefix], - owner=params.yarn_user, - recursive=True, - ignore_failures=True, - ) - - XmlConfig("core-site.xml", - conf_dir=params.hadoop_conf_dir, - configurations=params.config['configurations']['core-site'], - configuration_attributes=params.config['configuration_attributes']['core-site'], - owner=params.hdfs_user, - group=params.user_group, - mode=0644 - ) - - XmlConfig("mapred-site.xml", - conf_dir=params.hadoop_conf_dir, - configurations=params.config['configurations']['mapred-site'], - configuration_attributes=params.config['configuration_attributes']['mapred-site'], - owner=params.yarn_user, - group=params.user_group, - mode=0644 - ) - - XmlConfig("yarn-site.xml", - conf_dir=params.hadoop_conf_dir, - configurations=params.config['configurations']['yarn-site'], - configuration_attributes=params.config['configuration_attributes']['yarn-site'], - owner=params.yarn_user, - group=params.user_group, - mode=0644 - ) - - XmlConfig("capacity-scheduler.xml", - conf_dir=params.hadoop_conf_dir, - configurations=params.config['configurations']['capacity-scheduler'], - configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'], - owner=params.yarn_user, - group=params.user_group, - mode=0644 - ) - - if name == 'resourcemanager': - File(params.yarn_job_summary_log, - owner=params.yarn_user, - group=params.user_group - ) - if params.node_labels_dir: - params.HdfsDirectory(params.node_labels_dir, - action="create", - owner=params.yarn_user, - group=params.user_group, - mode=0700 - ) - elif name == 'apptimelineserver': - Directory(params.ats_leveldb_dir, - owner=params.yarn_user, - group=params.user_group, - recursive=True, - recursive_permission=True - ) - - File(params.rm_nodes_exclude_path, - owner=params.yarn_user, - group=params.user_group - ) - - File(format("{limits_conf_dir}/yarn.conf"), - mode=0644, - content=Template('yarn.conf.j2') - ) - - File(format("{limits_conf_dir}/mapreduce.conf"), - mode=0644, - content=Template('mapreduce.conf.j2') - ) - - File(format("{hadoop_conf_dir}/yarn-env.sh"), - owner=params.yarn_user, - group=params.user_group, - mode=0755, - content=InlineTemplate(params.yarn_env_sh_template) - ) - - if params.security_enabled: - container_executor = format("{yarn_container_bin}/container-executor") - File(container_executor, - group=params.yarn_executor_container_group, - mode=06050 - ) - - File(format("{hadoop_conf_dir}/container-executor.cfg"), - group=params.user_group, - mode=0644, - content=Template('container-executor.cfg.j2') - ) - - - if params.security_enabled: - tc_mode = 0644 - tc_owner = "root" - else: - tc_mode = None - tc_owner = params.hdfs_user - - File(format("{hadoop_conf_dir}/mapred-env.sh"), - owner=tc_owner, - content=InlineTemplate(params.mapred_env_sh_template) - ) - - if params.security_enabled: - File(os.path.join(params.hadoop_bin, "task-controller"), - owner="root", - group=params.mapred_tt_group, - mode=06050 - ) - File(os.path.join(params.hadoop_conf_dir, 'taskcontroller.cfg'), - owner = tc_owner, - mode = tc_mode, - group = params.mapred_tt_group, - content=Template("taskcontroller.cfg.j2") - ) - else: - File(os.path.join(params.hadoop_conf_dir, 'taskcontroller.cfg'), - owner=tc_owner, - content=Template("taskcontroller.cfg.j2") - ) - - if "mapred-site" in params.config['configurations']: - XmlConfig("mapred-site.xml", - conf_dir=params.hadoop_conf_dir, - configurations=params.config['configurations']['mapred-site'], - configuration_attributes=params.config['configuration_attributes']['mapred-site'], - owner=params.mapred_user, - group=params.user_group - ) - - if "capacity-scheduler" in params.config['configurations']: - XmlConfig("capacity-scheduler.xml", - conf_dir=params.hadoop_conf_dir, - configurations=params.config['configurations'][ - 'capacity-scheduler'], - configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'], - owner=params.hdfs_user, - group=params.user_group - ) - - if os.path.exists(os.path.join(params.hadoop_conf_dir, 'fair-scheduler.xml')): - File(os.path.join(params.hadoop_conf_dir, 'fair-scheduler.xml'), - owner=params.mapred_user, - group=params.user_group - ) - - if os.path.exists( - os.path.join(params.hadoop_conf_dir, 'ssl-client.xml.example')): - File(os.path.join(params.hadoop_conf_dir, 'ssl-client.xml.example'), - owner=params.mapred_user, - group=params.user_group - ) - - if os.path.exists( - os.path.join(params.hadoop_conf_dir, 'ssl-server.xml.example')): - File(os.path.join(params.hadoop_conf_dir, 'ssl-server.xml.example'), - owner=params.mapred_user, - group=params.user_group - ) http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn_client.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn_client.py deleted file mode 100644 index 60b9bce..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn_client.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -import sys -from resource_management import * - -from yarn import yarn - -class YarnClient(Script): - - def pre_rolling_restart(self, env): - import params - env.set_params(params) - - if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: - Execute(format("hdp-select set hadoop-client {version}")) - - def install(self, env): - self.install_packages(env) - self.configure(env) - - def configure(self, env): - import params - env.set_params(params) - yarn() - - def status(self, env): - raise ClientComponentHasNoStatus() - -if __name__ == "__main__": - YarnClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/container-executor.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/container-executor.cfg.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/container-executor.cfg.j2 deleted file mode 100644 index c6f1ff6..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/container-executor.cfg.j2 +++ /dev/null @@ -1,40 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -#/* -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -yarn.nodemanager.local-dirs={{nm_local_dirs}} -yarn.nodemanager.log-dirs={{nm_log_dirs}} -yarn.nodemanager.linux-container-executor.group={{yarn_executor_container_group}} -banned.users=hdfs,yarn,mapred,bin -min.user.id={{min_user_id}} http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/exclude_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/exclude_hosts_list.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/exclude_hosts_list.j2 deleted file mode 100644 index c7ce416..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/exclude_hosts_list.j2 +++ /dev/null @@ -1,21 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -{% for host in exclude_hosts %} -{{host}} -{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/mapreduce.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/mapreduce.conf.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/mapreduce.conf.j2 deleted file mode 100644 index b996645..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/mapreduce.conf.j2 +++ /dev/null @@ -1,35 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{mapred_user}} - nofile 32768 -{{mapred_user}} - nproc 65536 http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/taskcontroller.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/taskcontroller.cfg.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/taskcontroller.cfg.j2 deleted file mode 100644 index 3d5f4f2..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/taskcontroller.cfg.j2 +++ /dev/null @@ -1,38 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -#/* -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -mapred.local.dir={{mapred_local_dir}} -mapreduce.tasktracker.group={{mapred_tt_group}} -hadoop.log.dir={{hdfs_log_dir_prefix}}/{{mapred_user}}
