Repository: ambari Updated Branches: refs/heads/trunk 1daffd8d8 -> 47626bd65
http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/files/validateYarnComponentStatus.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/files/validateYarnComponentStatus.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/files/validateYarnComponentStatus.py new file mode 100644 index 0000000..dac198a --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/files/validateYarnComponentStatus.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import optparse +import subprocess +import json + +RESOURCEMANAGER = 'rm' +NODEMANAGER = 'nm' +HISTORYSERVER = 'hs' + +STARTED_STATE = 'STARTED' +RUNNING_STATE = 'RUNNING' + +#Return reponse for given path and address +def getResponse(path, address, ssl_enabled): + + command = "curl" + httpGssnegotiate = "--negotiate" + userpswd = "-u:" + insecure = "-k"# This is smoke test, no need to check CA of server + if ssl_enabled: + url = 'https://' + address + path + else: + url = 'http://' + address + path + + command_with_flags = [command,httpGssnegotiate,userpswd,insecure,url] + try: + proc = subprocess.Popen(command_with_flags, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = proc.communicate() + response = json.loads(stdout) + if response == None: + print 'There is no response for url: ' + str(url) + exit(1) + return response + except Exception as e: + print 'Error getting response for url:' + str(url), e + exit(1) + +#Verify that REST api is available for given component +def validateAvailability(component, path, address, ssl_enabled): + + try: + response = getResponse(path, address, ssl_enabled) + is_valid = validateAvailabilityResponse(component, response) + if not is_valid: + exit(1) + except Exception as e: + print 'Error checking availability status of component', e + exit(1) + +#Validate component-specific response +def validateAvailabilityResponse(component, response): + try: + if component == RESOURCEMANAGER: + rm_state = response['clusterInfo']['state'] + if rm_state == STARTED_STATE: + return True + else: + print 'Resourcemanager is not started' + return False + + elif component == NODEMANAGER: + node_healthy = bool(response['nodeInfo']['nodeHealthy']) + if node_healthy: + return True + else: + return False + elif component == HISTORYSERVER: + hs_start_time = response['historyInfo']['startedOn'] + if hs_start_time > 0: + return True + else: + return False + else: + return False + except Exception as e: + print 'Error validation of availability response for ' + str(component), e + return False + +#Verify that component has required resources to work +def validateAbility(component, path, address, ssl_enabled): + + try: + response = getResponse(path, address, ssl_enabled) + is_valid = validateAbilityResponse(component, response) + if not is_valid: + exit(1) + except Exception as e: + print 'Error checking ability of component', e + exit(1) + +#Validate component-specific response that it has required resources to work +def validateAbilityResponse(component, response): + try: + if component == RESOURCEMANAGER: + nodes = [] + if response.has_key('nodes') and not response['nodes'] == None and response['nodes'].has_key('node'): + nodes = response['nodes']['node'] + connected_nodes_count = len(nodes) + if connected_nodes_count == 0: + print 'There is no connected nodemanagers to resourcemanager' + return False + active_nodes = filter(lambda x: x['state'] == RUNNING_STATE, nodes) + active_nodes_count = len(active_nodes) + + if connected_nodes_count == 0: + print 'There is no connected active nodemanagers to resourcemanager' + return False + else: + return True + else: + return False + except Exception as e: + print 'Error validation of ability response', e + return False + +# +# Main. +# +def main(): + parser = optparse.OptionParser(usage="usage: %prog [options] component ") + parser.add_option("-p", "--port", dest="address", help="Host:Port for REST API of a desired component") + parser.add_option("-s", "--ssl", dest="ssl_enabled", help="Is SSL enabled for UI of component") + + (options, args) = parser.parse_args() + + component = args[0] + + address = options.address + ssl_enabled = (options.ssl_enabled) in 'true' + if component == RESOURCEMANAGER: + path = '/ws/v1/cluster/info' + elif component == NODEMANAGER: + path = '/ws/v1/node/info' + elif component == HISTORYSERVER: + path = '/ws/v1/history/info' + else: + parser.error("Invalid component") + + validateAvailability(component, path, address, ssl_enabled) + + if component == RESOURCEMANAGER: + path = '/ws/v1/cluster/nodes' + validateAbility(component, path, address, ssl_enabled) + +if __name__ == "__main__": + main() http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/__init__.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/__init__.py new file mode 100644 index 0000000..a582077 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/__init__.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/application_timeline_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/application_timeline_server.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/application_timeline_server.py new file mode 100644 index 0000000..77c3777 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/application_timeline_server.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import sys +from resource_management import * +from yarn import yarn +from service import service + +class ApplicationTimelineServer(Script): + + def install(self, env): + self.install_packages(env) + #self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + yarn() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # FOR SECURITY + service('historyserver', action='start') + + def stop(self, env): + import params + env.set_params(params) + service('historyserver', action='stop') + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.yarn_historyserver_pid_file) + +if __name__ == "__main__": + ApplicationTimelineServer().execute() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/historyserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/historyserver.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/historyserver.py new file mode 100644 index 0000000..6736d40 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/historyserver.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +import sys +from resource_management import * + +from yarn import yarn +from service import service + +class Histroryserver(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + yarn(name="historyserver") + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # FOR SECURITY + service('historyserver', action='start', serviceName='mapreduce') + + def stop(self, env): + import params + env.set_params(params) + service('historyserver', action='stop', serviceName='mapreduce') + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.mapred_historyserver_pid_file) + +if __name__ == "__main__": + Histroryserver().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/mapred_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/mapred_service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/mapred_service_check.py new file mode 100644 index 0000000..3b789f8 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/mapred_service_check.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * + +class MapReduce2ServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + + jar_path = format("{hadoop_mapred2_jar_location}/{hadoopMapredExamplesJarName}") + input_file = format("/user/{smokeuser}/mapredsmokeinput") + output_file = format("/user/{smokeuser}/mapredsmokeoutput") + + cleanup_cmd = format("fs -rm -r -f {output_file} {input_file}") + create_file_cmd = format("fs -put /etc/passwd {input_file}") + test_cmd = format("fs -test -e {output_file}") + run_wordcount_job = format("jar {jar_path} wordcount {input_file} {output_file}") + + if params.security_enabled: + kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser};") + + Execute(kinit_cmd, + user=params.smokeuser + ) + + ExecuteHadoop(cleanup_cmd, + tries=1, + try_sleep=5, + user=params.smokeuser, + conf_dir=params.hadoop_conf_dir + ) + + ExecuteHadoop(create_file_cmd, + tries=1, + try_sleep=5, + user=params.smokeuser, + conf_dir=params.hadoop_conf_dir + ) + + ExecuteHadoop(run_wordcount_job, + tries=1, + try_sleep=5, + user=params.smokeuser, + conf_dir=params.hadoop_conf_dir, + logoutput=True + ) + + ExecuteHadoop(test_cmd, + user=params.smokeuser, + conf_dir=params.hadoop_conf_dir + ) + +if __name__ == "__main__": + MapReduce2ServiceCheck().execute() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/mapreduce2_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/mapreduce2_client.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/mapreduce2_client.py new file mode 100644 index 0000000..54119a7 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/mapreduce2_client.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import sys +from resource_management import * + +from yarn import yarn + +class MapReduce2Client(Script): + + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + yarn() + + def status(self, env): + raise ClientComponentHasNoStatus() + +if __name__ == "__main__": + MapReduce2Client().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/nodemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/nodemanager.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/nodemanager.py new file mode 100644 index 0000000..473f50c --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/nodemanager.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import sys +from resource_management import * + +from yarn import yarn +from service import service + +class Nodemanager(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + yarn(name="nodemanager") + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # FOR SECURITY + service('nodemanager', + action='start' + ) + + def stop(self, env): + import params + env.set_params(params) + + service('nodemanager', + action='stop' + ) + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.nodemanager_pid_file) + +if __name__ == "__main__": + Nodemanager().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py new file mode 100644 index 0000000..a523a4f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * +import status_params + +# server configurations +config = Script.get_config() + +config_dir = "/etc/hadoop/conf" + +mapred_user = status_params.mapred_user +yarn_user = status_params.yarn_user +hdfs_user = config['configurations']['global']['hdfs_user'] + +smokeuser = config['configurations']['global']['smokeuser'] +_authentication = config['configurations']['core-site']['hadoop.security.authentication'] +security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +smoke_user_keytab = config['configurations']['global']['smokeuser_keytab'] +yarn_executor_container_group = config['configurations']['yarn-site']['yarn.nodemanager.linux-container-executor.group'] +kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) +rm_host = config['clusterHostInfo']['rm_host'][0] +rm_port = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'].split(':')[-1] +rm_https_port = "8090" +rm_nodes_exclude_path = config['configurations']['yarn-site']['yarn.resourcemanager.nodes.exclude-path'] + +java64_home = config['hostLevelParams']['java_home'] +hadoop_ssl_enabled = default("/configurations/core-site/hadoop.ssl.enabled", False) + +hadoop_libexec_dir = '/usr/lib/hadoop/libexec' +hadoop_yarn_home = '/usr/lib/hadoop-yarn' +yarn_heapsize = config['configurations']['global']['yarn_heapsize'] +resourcemanager_heapsize = config['configurations']['global']['resourcemanager_heapsize'] +nodemanager_heapsize = config['configurations']['global']['nodemanager_heapsize'] +apptimelineserver_heapsize = default("/configurations/global/apptimelineserver_heapsize", 1024) +yarn_log_dir_prefix = config['configurations']['global']['yarn_log_dir_prefix'] +yarn_pid_dir_prefix = status_params.yarn_pid_dir_prefix +mapred_pid_dir_prefix = status_params.mapred_pid_dir_prefix +mapred_log_dir_prefix = config['configurations']['global']['mapred_log_dir_prefix'] + +rm_webui_address = format("{rm_host}:{rm_port}") +rm_webui_https_address = format("{rm_host}:{rm_https_port}") +nm_webui_address = config['configurations']['yarn-site']['yarn.nodemanager.webapp.address'] +hs_webui_address = config['configurations']['mapred-site']['mapreduce.jobhistory.webapp.address'] + +nm_local_dirs = config['configurations']['yarn-site']['yarn.nodemanager.local-dirs'] +nm_log_dirs = config['configurations']['yarn-site']['yarn.nodemanager.log-dirs'] + + +hadoop_mapred2_jar_location = "/usr/lib/hadoop-mapreduce" +distrAppJarName = "hadoop-yarn-applications-distributedshell-2.*.jar" +hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar" + +yarn_pid_dir = status_params.yarn_pid_dir +mapred_pid_dir = status_params.mapred_pid_dir + +mapred_log_dir = format("{mapred_log_dir_prefix}/{mapred_user}") +yarn_log_dir = format("{yarn_log_dir_prefix}/{yarn_user}") +mapred_job_summary_log = format("{mapred_log_dir_prefix}/{mapred_user}/hadoop-mapreduce.jobsummary.log") +yarn_job_summary_log = format("{yarn_log_dir_prefix}/{yarn_user}/hadoop-mapreduce.jobsummary.log") + +mapred_bin = "/usr/lib/hadoop-mapreduce/sbin" +yarn_bin = "/usr/lib/hadoop-yarn/sbin" + +user_group = config['configurations']['global']['user_group'] +limits_conf_dir = "/etc/security/limits.d" +hadoop_conf_dir = "/etc/hadoop/conf" +yarn_container_bin = "/usr/lib/hadoop-yarn/bin" + +#exclude file +exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) +exclude_file_path = config['configurations']['yarn-site']['yarn.resourcemanager.nodes.exclude-path'] + +hostname = config['hostname'] + +if security_enabled: + nm_principal_name = config['configurations']['global']['nodemanager_principal_name'] + nodemanager_keytab = config['configurations']['global']['nodemanager_keytab'] + nodemanager_principal_name = nm_principal_name.replace('_HOST',hostname.lower()) + nm_kinit_cmd = format("{kinit_path_local} -kt {nodemanager_keytab} {nodemanager_principal_name};") +else: + nm_kinit_cmd = "" + +yarn_log_aggregation_enabled = config['configurations']['yarn-site']['yarn.log-aggregation-enable'] +yarn_nm_app_log_dir = config['configurations']['yarn-site']['yarn.nodemanager.remote-app-log-dir'] +mapreduce_jobhistory_intermediate_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.intermediate-done-dir'] +mapreduce_jobhistory_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.done-dir'] + +#for create_hdfs_directory +hostname = config["hostname"] +hadoop_conf_dir = "/etc/hadoop/conf" +hdfs_user_keytab = config['configurations']['global']['hdfs_user_keytab'] +hdfs_user = config['configurations']['global']['hdfs_user'] +kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) +import functools +#create partial functions with common arguments for every HdfsDirectory call +#to create hdfs directory we need to call params.HdfsDirectory in code +HdfsDirectory = functools.partial( + HdfsDirectory, + conf_dir=hadoop_conf_dir, + hdfs_user=hdfs_user, + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local +) +update_exclude_file_only = config['commandParams']['update_exclude_file_only'] http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py new file mode 100644 index 0000000..6e6b34f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import sys +from resource_management import * + +from yarn import yarn +from service import service + + +class Resourcemanager(Script): + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + + env.set_params(params) + yarn(name='resourcemanager') + + def start(self, env): + import params + + env.set_params(params) + self.configure(env) # FOR SECURITY + service('resourcemanager', + action='start' + ) + + def stop(self, env): + import params + + env.set_params(params) + + service('resourcemanager', + action='stop' + ) + + def status(self, env): + import status_params + + env.set_params(status_params) + check_process_status(status_params.resourcemanager_pid_file) + pass + + def decommission(self, env): + import params + + env.set_params(params) + nm_kinit_cmd = params.nm_kinit_cmd + yarn_user = params.yarn_user + conf_dir = params.config_dir + user_group = params.user_group + + yarn_refresh_cmd = format("{nm_kinit_cmd} /usr/bin/yarn --config {conf_dir} rmadmin -refreshNodes") + + File(params.exclude_file_path, + content=Template("exclude_hosts_list.j2"), + owner=yarn_user, + group=user_group + ) + + if params.update_exclude_file_only == False: + Execute(yarn_refresh_cmd, + user=yarn_user) + pass + pass + + +if __name__ == "__main__": + Resourcemanager().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/service.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/service.py new file mode 100644 index 0000000..0692c1b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/service.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * + + +def service(componentName, action='start', serviceName='yarn'): + + import params + + if (serviceName == 'mapreduce' and componentName == 'historyserver'): + daemon = format("{mapred_bin}/mr-jobhistory-daemon.sh") + pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-{componentName}.pid") + usr = params.mapred_user + else: + daemon = format("{yarn_bin}/yarn-daemon.sh") + pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-{componentName}.pid") + usr = params.yarn_user + + cmd = format("export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --config {config_dir}") + + if action == 'start': + daemon_cmd = format("{cmd} start {componentName}") + no_op = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1") + Execute(daemon_cmd, + user=usr, + not_if=no_op + ) + + Execute(no_op, + user=usr, + not_if=no_op, + initial_wait=5 + ) + + elif action == 'stop': + daemon_cmd = format("{cmd} stop {componentName}") + Execute(daemon_cmd, + user=usr, + ) + rm_pid = format("rm -f {pid_file}") + Execute(rm_pid, + user=usr + ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/service_check.py new file mode 100644 index 0000000..135b30b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/service_check.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * +import sys + +class ServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + + run_yarn_check_cmd = "/usr/bin/yarn node -list" + + component_type = 'rm' + if params.hadoop_ssl_enabled: + component_address = params.rm_webui_https_address + else: + component_address = params.rm_webui_address + + validateStatusFileName = "validateYarnComponentStatus.py" + validateStatusFilePath = format("/tmp/{validateStatusFileName}") + python_executable = sys.executable + validateStatusCmd = format("{python_executable} {validateStatusFilePath} {component_type} -p {component_address} -s {hadoop_ssl_enabled}") + + if params.security_enabled: + kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser};") + smoke_cmd = format("{kinit_cmd} {validateStatusCmd}") + else: + smoke_cmd = validateStatusCmd + + File(validateStatusFilePath, + content=StaticFile(validateStatusFileName), + mode=0755 + ) + + Execute(smoke_cmd, + tries=3, + try_sleep=5, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + user=params.smokeuser, + logoutput=True + ) + + Execute(run_yarn_check_cmd, + user=params.smokeuser + ) + +if __name__ == "__main__": + ServiceCheck().execute() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/status_params.py new file mode 100644 index 0000000..1a67de7 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/status_params.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +config = Script.get_config() + +mapred_user = config['configurations']['global']['mapred_user'] +yarn_user = config['configurations']['global']['yarn_user'] +yarn_pid_dir_prefix = config['configurations']['global']['yarn_pid_dir_prefix'] +mapred_pid_dir_prefix = config['configurations']['global']['mapred_pid_dir_prefix'] +yarn_pid_dir = format("{yarn_pid_dir_prefix}/{yarn_user}") +mapred_pid_dir = format("{mapred_pid_dir_prefix}/{mapred_user}") + +resourcemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-resourcemanager.pid") +nodemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-nodemanager.pid") +yarn_historyserver_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-historyserver.pid") +mapred_historyserver_pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-historyserver.pid") \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/yarn.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/yarn.py new file mode 100644 index 0000000..9387022 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/yarn.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * +import sys + + +def yarn(name = None): + import params + + + if name in ["nodemanager","historyserver"]: + if params.yarn_log_aggregation_enabled: + params.HdfsDirectory(params.yarn_nm_app_log_dir, + action="create_delayed", + owner=params.yarn_user, + group=params.user_group, + mode=0777, + recursive_chmod=True + ) + params.HdfsDirectory("/mapred", + action="create_delayed", + owner=params.mapred_user + ) + params.HdfsDirectory("/mapred/system", + action="create_delayed", + owner=params.hdfs_user + ) + params.HdfsDirectory(params.mapreduce_jobhistory_intermediate_done_dir, + action="create_delayed", + owner=params.mapred_user, + group=params.user_group, + mode=0777 + ) + + params.HdfsDirectory(params.mapreduce_jobhistory_done_dir, + action="create_delayed", + owner=params.mapred_user, + group=params.user_group, + mode=01777 + ) + params.HdfsDirectory(None, action="create") + + Directory([params.yarn_pid_dir, params.yarn_log_dir], + owner=params.yarn_user, + group=params.user_group, + recursive=True + ) + + Directory([params.mapred_pid_dir, params.mapred_log_dir], + owner=params.mapred_user, + group=params.user_group, + recursive=True + ) + Directory(params.nm_local_dirs.split(','), + owner=params.yarn_user, + recursive=True + ) + Directory(params.nm_log_dirs.split(','), + owner=params.yarn_user, + recursive=True + ) + Directory(params.yarn_log_dir_prefix, + owner=params.yarn_user, + recursive=True + ) + + XmlConfig("core-site.xml", + conf_dir=params.config_dir, + configurations=params.config['configurations']['core-site'], + owner=params.hdfs_user, + group=params.user_group, + mode=0644 + ) + + XmlConfig("mapred-site.xml", + conf_dir=params.config_dir, + configurations=params.config['configurations']['mapred-site'], + owner=params.yarn_user, + group=params.user_group, + mode=0644 + ) + + XmlConfig("yarn-site.xml", + conf_dir=params.config_dir, + configurations=params.config['configurations']['yarn-site'], + owner=params.yarn_user, + group=params.user_group, + mode=0644 + ) + + XmlConfig("capacity-scheduler.xml", + conf_dir=params.config_dir, + configurations=params.config['configurations']['capacity-scheduler'], + owner=params.yarn_user, + group=params.user_group, + mode=0644 + ) + + if name == 'resourcemanager': + File(params.yarn_job_summary_log, + owner=params.yarn_user, + group=params.user_group + ) + + File(params.rm_nodes_exclude_path, + owner=params.yarn_user, + group=params.user_group + ) + + File(format("{limits_conf_dir}/yarn.conf"), + mode=0644, + content=Template('yarn.conf.j2') + ) + + File(format("{limits_conf_dir}/mapreduce.conf"), + mode=0644, + content=Template('mapreduce.conf.j2') + ) + + File(format("{config_dir}/yarn-env.sh"), + owner=params.yarn_user, + group=params.user_group, + mode=0755, + content=Template('yarn-env.sh.j2') + ) + + if params.security_enabled: + container_executor = format("{yarn_container_bin}/container-executor") + File(container_executor, + group=params.yarn_executor_container_group, + mode=06050 + ) + + File(format("{config_dir}/container-executor.cfg"), + group=params.user_group, + mode=0644, + content=Template('container-executor.cfg.j2') + ) + + http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/yarn_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/yarn_client.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/yarn_client.py new file mode 100644 index 0000000..7e9c564 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/yarn_client.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import sys +from resource_management import * + +from yarn import yarn + +class YarnClient(Script): + + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + yarn() + + def status(self, env): + raise ClientComponentHasNoStatus() + +if __name__ == "__main__": + YarnClient().execute() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/container-executor.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/container-executor.cfg.j2 b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/container-executor.cfg.j2 new file mode 100644 index 0000000..29ad949 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/container-executor.cfg.j2 @@ -0,0 +1,22 @@ +#/* +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +yarn.nodemanager.local-dirs={{nm_local_dirs}} +yarn.nodemanager.log-dirs={{nm_log_dirs}} +yarn.nodemanager.linux-container-executor.group={{yarn_executor_container_group}} +banned.users = hfds,yarn,mapred,bin +min.user.id=1000 http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2 b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2 new file mode 100644 index 0000000..4a4c698 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2 @@ -0,0 +1,3 @@ +{% for host in exclude_hosts %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/mapreduce.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/mapreduce.conf.j2 b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/mapreduce.conf.j2 new file mode 100644 index 0000000..76caea4 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/mapreduce.conf.j2 @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{mapred_user}} - nofile 32768 +{{mapred_user}} - nproc 65536 http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/yarn-env.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/yarn-env.sh.j2 b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/yarn-env.sh.j2 new file mode 100644 index 0000000..abdb003 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/yarn-env.sh.j2 @@ -0,0 +1,128 @@ +#/* +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +export HADOOP_YARN_HOME={{hadoop_yarn_home}} +export YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER +export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER +export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}} +export JAVA_HOME={{java64_home}} + +# User for YARN daemons +export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn} + +# resolve links - $0 may be a softlink +export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}" + +# some Java parameters +# export JAVA_HOME=/home/y/libexec/jdk1.6.0/ +if [ "$JAVA_HOME" != "" ]; then + #echo "run java in $JAVA_HOME" + JAVA_HOME=$JAVA_HOME +fi + +if [ "$JAVA_HOME" = "" ]; then + echo "Error: JAVA_HOME is not set." + exit 1 +fi + +JAVA=$JAVA_HOME/bin/java +JAVA_HEAP_MAX=-Xmx1000m + +# For setting YARN specific HEAP sizes please use this +# Parameter and set appropriately +YARN_HEAPSIZE={{yarn_heapsize}} + +# check envvars which might override default args +if [ "$YARN_HEAPSIZE" != "" ]; then + JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m" +fi + +# Resource Manager specific parameters + +# Specify the max Heapsize for the ResourceManager using a numerical value +# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set +# the value to 1000. +# This value will be overridden by an Xmx setting specified in either YARN_OPTS +# and/or YARN_RESOURCEMANAGER_OPTS. +# If not specified, the default value will be picked from either YARN_HEAPMAX +# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. +export YARN_RESOURCEMANAGER_HEAPSIZE={{resourcemanager_heapsize}} + +# Specify the JVM options to be used when starting the ResourceManager. +# These options will be appended to the options specified as YARN_OPTS +# and therefore may override any similar flags set in YARN_OPTS +#export YARN_RESOURCEMANAGER_OPTS= + +# Node Manager specific parameters + +# Specify the max Heapsize for the NodeManager using a numerical value +# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set +# the value to 1000. +# This value will be overridden by an Xmx setting specified in either YARN_OPTS +# and/or YARN_NODEMANAGER_OPTS. +# If not specified, the default value will be picked from either YARN_HEAPMAX +# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. +export YARN_NODEMANAGER_HEAPSIZE={{nodemanager_heapsize}} + +# Specify the max Heapsize for the HistoryManager using a numerical value +# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set +# the value to 1024. +# This value will be overridden by an Xmx setting specified in either YARN_OPTS +# and/or YARN_HISTORYSERVER_OPTS. +# If not specified, the default value will be picked from either YARN_HEAPMAX +# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. +export YARN_HISTORYSERVER_HEAPSIZE={{apptimelineserver_heapsize}} + +# Specify the JVM options to be used when starting the NodeManager. +# These options will be appended to the options specified as YARN_OPTS +# and therefore may override any similar flags set in YARN_OPTS +#export YARN_NODEMANAGER_OPTS= + +# so that filenames w/ spaces are handled correctly in loops below +IFS= + + +# default log directory & file +if [ "$YARN_LOG_DIR" = "" ]; then + YARN_LOG_DIR="$HADOOP_YARN_HOME/logs" +fi +if [ "$YARN_LOGFILE" = "" ]; then + YARN_LOGFILE='yarn.log' +fi + +# default policy file for service-level authorization +if [ "$YARN_POLICYFILE" = "" ]; then + YARN_POLICYFILE="hadoop-policy.xml" +fi + +# restore ordinary behaviour +unset IFS + + +YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR" +YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR" +YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE" +YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE" +YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME" +YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING" +YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" +YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" +if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then + YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" +fi +YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE" http://git-wip-us.apache.org/repos/asf/ambari/blob/47626bd6/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/yarn.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/yarn.conf.j2 b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/yarn.conf.j2 new file mode 100644 index 0000000..be89b07 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/yarn.conf.j2 @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{yarn_user}} - nofile 32768 +{{yarn_user}} - nproc 65536
