http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/default_update_exclude_file_only.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_update_exclude_file_only.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_update_exclude_file_only.json index 2ea07e4..f50a207 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_update_exclude_file_only.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_update_exclude_file_only.json @@ -33,7 +33,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "true", + "update_files_only" : "true", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}],
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/default_with_bucket.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_with_bucket.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_with_bucket.json index 5080d30..c1eb868 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_with_bucket.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_with_bucket.json @@ -38,7 +38,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/flume_22.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/flume_22.json b/ambari-server/src/test/python/stacks/2.0.6/configs/flume_22.json index 6ec9ec9..c99d10b 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/flume_22.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/flume_22.json @@ -32,7 +32,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "output_file":"HDFS_CLIENT-configs.tar.gz" http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/flume_only.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/flume_only.json b/ambari-server/src/test/python/stacks/2.0.6/configs/flume_only.json index 1550715..1a4d676 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/flume_only.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/flume_only.json @@ -31,7 +31,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json index f572413..52a1fde 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json @@ -35,7 +35,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json index 9979e9d..7283bf5 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json @@ -47,7 +47,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/oozie_existing_sqla.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/oozie_existing_sqla.json b/ambari-server/src/test/python/stacks/2.0.6/configs/oozie_existing_sqla.json index 8d12b98..4ffa29f 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/oozie_existing_sqla.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/oozie_existing_sqla.json @@ -34,7 +34,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/repository_file.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/repository_file.json b/ambari-server/src/test/python/stacks/2.0.6/configs/repository_file.json index 1554f1b..7efb7d9 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/repository_file.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/repository_file.json @@ -77,7 +77,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json index 5327865..76a110e 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json @@ -38,7 +38,7 @@ "script": "scripts/yarn_client.py", "excluded_hosts": "host1", "mark_draining_only" : "false", - "update_exclude_file_only" : "false" + "update_files_only" : "false" }, "taskId": 186, "public_hostname": "c6401.ambari.apache.org", http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json b/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json index b4342ad..475a6f9 100644 --- a/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json +++ b/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json @@ -31,7 +31,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json index 9dcb451..7622212 100644 --- a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json +++ b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json @@ -38,7 +38,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.5/configs/hsi_default_for_restart.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default_for_restart.json b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default_for_restart.json index f6de1c4..c2320ba 100644 --- a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default_for_restart.json +++ b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default_for_restart.json @@ -39,7 +39,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.5/configs/hsi_ha.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/configs/hsi_ha.json b/ambari-server/src/test/python/stacks/2.5/configs/hsi_ha.json index 3fd9f72..7b79d84 100644 --- a/ambari-server/src/test/python/stacks/2.5/configs/hsi_ha.json +++ b/ambari-server/src/test/python/stacks/2.5/configs/hsi_ha.json @@ -38,7 +38,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/configuration/yarn-site.xml b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/configuration/yarn-site.xml index 59ff82b..f3ea462 100755 --- a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/configuration/yarn-site.xml +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/configuration/yarn-site.xml @@ -388,6 +388,12 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>manage.include.files</name> + <value>false</value> + <description>If true Ambari will manage include file if yarn.resourcemanager.nodes.include-path is configured.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>yarn.http.policy</name> <value>HTTP_ONLY</value> <description> http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/params_linux.py b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/params_linux.py index 4d42861..da54b7c 100755 --- a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/params_linux.py +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/params_linux.py @@ -233,6 +233,13 @@ has_ats = not len(ats_host) == 0 nm_hosts = default("/clusterHostInfo/nm_hosts", []) +#incude file +include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None) +include_hosts = None +manage_include_files = default("/configurations/yarn-site/manage.include.files", False) +if include_file_path and manage_include_files: + include_hosts = list(set(nm_hosts) - set(exclude_hosts)) + # don't using len(nm_hosts) here, because check can take too much time on large clusters number_of_nm = 1 @@ -315,7 +322,7 @@ HdfsResource = functools.partial( immutable_paths = get_not_managed_resources(), dfs_type = dfs_type ) -update_exclude_file_only = default("/commandParams/update_exclude_file_only",False) +update_files_only = default("/commandParams/update_files_only",False) mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/params_windows.py b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/params_windows.py index 0f8ce73..cac93ee 100755 --- a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/params_windows.py +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/params_windows.py @@ -56,4 +56,12 @@ hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar" exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") -update_exclude_file_only = config['commandParams']['update_exclude_file_only'] +update_files_only = default("/commandParams/update_files_only", False) + +nm_hosts = default("/clusterHostInfo/nm_hosts", []) +#incude file +include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None) +include_hosts = None +manage_include_files = default("/configurations/yarn-site/manage.include.files", False) +if include_file_path and manage_include_files: + include_hosts = list(set(nm_hosts) - set(exclude_hosts)) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/resourcemanager.py b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/resourcemanager.py index 6a7eea7..71c7bc1 100755 --- a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/resourcemanager.py +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/resourcemanager.py @@ -91,7 +91,14 @@ class ResourcemanagerWindows(Resourcemanager): mode="f" ) - if params.update_exclude_file_only == False: + if params.include_hosts: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=yarn_user, + mode="f" + ) + + if params.update_files_only == False: Execute(yarn_refresh_cmd, user=yarn_user) @@ -219,7 +226,14 @@ class ResourcemanagerDefault(Resourcemanager): group=user_group ) - if params.update_exclude_file_only == False: + if params.include_hosts: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=yarn_user, + group=user_group + ) + + if params.update_files_only == False: Execute(yarn_refresh_cmd, environment= {'PATH' : params.execute_path }, user=yarn_user) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/templates/include_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/templates/include_hosts_list.j2 b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/templates/include_hosts_list.j2 new file mode 100644 index 0000000..42e33c0 --- /dev/null +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/templates/include_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in include_hosts %} +{{host}} +{% endfor %} \ No newline at end of file
