AMBARI-21345 Add host doesn't fully add a node when include/exclude files are used (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7764e387 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7764e387 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7764e387 Branch: refs/heads/branch-feature-AMBARI-21348 Commit: 7764e387782962030de93f22f27d037040589595 Parents: 805dbe4 Author: Dmytro Sen <[email protected]> Authored: Mon Jul 17 13:55:53 2017 +0300 Committer: Dmytro Sen <[email protected]> Committed: Mon Jul 17 13:55:53 2017 +0300 ---------------------------------------------------------------------- .../AmbariCustomCommandExecutionHelper.java | 14 ++- .../AmbariManagementControllerImpl.java | 122 ++++++++++++++----- .../internal/HostResourceProvider.java | 1 + .../HDFS/2.1.0.2.0/configuration/hdfs-site.xml | 6 + .../HDFS/2.1.0.2.0/package/scripts/hdfs.py | 8 ++ .../2.1.0.2.0/package/scripts/hdfs_namenode.py | 26 +++- .../2.1.0.2.0/package/scripts/params_linux.py | 11 +- .../2.1.0.2.0/package/scripts/params_windows.py | 7 ++ .../package/templates/include_hosts_list.j2 | 21 ++++ .../YARN/2.1.0.2.0/configuration/yarn-site.xml | 6 + .../2.1.0.2.0/package/scripts/params_linux.py | 12 +- .../2.1.0.2.0/package/scripts/params_windows.py | 10 +- .../package/scripts/resourcemanager.py | 18 ++- .../package/templates/include_hosts_list.j2 | 21 ++++ .../services/HDFS/configuration/hdfs-site.xml | 6 + .../HDFS/package/scripts/hdfs_namenode.py | 12 +- .../0.8/services/HDFS/package/scripts/params.py | 11 +- .../package/templates/include_hosts_list.j2 | 21 ++++ .../services/YARN/configuration/yarn-site.xml | 6 + .../0.8/services/YARN/package/scripts/params.py | 10 +- .../YARN/package/scripts/resourcemanager.py | 9 +- .../package/templates/include_hosts_list.j2 | 21 ++++ .../services/YARN/configuration/yarn-site.xml | 6 + .../services/YARN/package/scripts/params.py | 9 +- .../YARN/package/scripts/resourcemanager.py | 9 +- .../package/templates/exclude_hosts_list.j2 | 21 ++++ .../package/templates/include_hosts_list.j2 | 21 ++++ .../services/YARN/configuration/yarn-site.xml | 6 + .../services/YARN/package/scripts/params.py | 10 +- .../YARN/package/scripts/resourcemanager.py | 9 +- .../package/templates/include_hosts_list.j2 | 21 ++++ .../AmbariManagementControllerTest.java | 8 +- .../python/stacks/2.0.6/HDFS/test_namenode.py | 2 +- .../stacks/2.0.6/configs/altfs_plus_hdfs.json | 2 +- .../python/stacks/2.0.6/configs/default.json | 2 +- .../2.0.6/configs/default_ams_embedded.json | 2 +- .../2.0.6/configs/default_hive_nn_ha.json | 2 +- .../2.0.6/configs/default_hive_nn_ha_2.json | 2 +- .../2.0.6/configs/default_hive_non_hdfs.json | 2 +- .../2.0.6/configs/default_no_install.json | 2 +- .../2.0.6/configs/default_oozie_mysql.json | 2 +- .../default_update_exclude_file_only.json | 2 +- .../2.0.6/configs/default_with_bucket.json | 2 +- .../python/stacks/2.0.6/configs/flume_22.json | 2 +- .../python/stacks/2.0.6/configs/flume_only.json | 2 +- .../stacks/2.0.6/configs/hbase_no_phx.json | 2 +- .../stacks/2.0.6/configs/hbase_with_phx.json | 2 +- .../2.0.6/configs/oozie_existing_sqla.json | 2 +- .../python/stacks/2.0.6/configs/secured.json | 2 +- .../test/python/stacks/2.3/configs/ats_1_5.json | 2 +- .../python/stacks/2.5/configs/hsi_default.json | 2 +- .../2.5/configs/hsi_default_for_restart.json | 2 +- .../test/python/stacks/2.5/configs/hsi_ha.json | 2 +- .../services/YARN/configuration/yarn-site.xml | 6 + .../YARN/package/scripts/params_linux.py | 9 +- .../YARN/package/scripts/params_windows.py | 10 +- .../YARN/package/scripts/resourcemanager.py | 18 ++- .../package/templates/include_hosts_list.j2 | 21 ++++ 58 files changed, 520 insertions(+), 85 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index 9b19ade..e321559 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -139,7 +139,7 @@ public class AmbariCustomCommandExecutionHelper { public final static String DECOM_EXCLUDED_HOSTS = "excluded_hosts"; public final static String DECOM_SLAVE_COMPONENT = "slave_type"; public final static String HBASE_MARK_DRAINING_ONLY = "mark_draining_only"; - public final static String UPDATE_EXCLUDE_FILE_ONLY = "update_exclude_file_only"; + public final static String UPDATE_FILES_ONLY = "update_files_only"; private final static String ALIGN_MAINTENANCE_STATE = "align_maintenance_state"; @@ -870,9 +870,9 @@ public class AmbariCustomCommandExecutionHelper { @Override public boolean shouldHostBeRemoved(final String hostname) throws AmbariException { - //Get UPDATE_EXCLUDE_FILE_ONLY parameter as string + //Get UPDATE_FILES_ONLY parameter as string String upd_excl_file_only_str = actionExecutionContext.getParameters() - .get(UPDATE_EXCLUDE_FILE_ONLY); + .get(UPDATE_FILES_ONLY); String decom_incl_hosts_str = actionExecutionContext.getParameters() .get(DECOM_INCLUDED_HOSTS); @@ -946,15 +946,17 @@ public class AmbariCustomCommandExecutionHelper { listOfExcludedHosts.add(sch.getHostName()); if (alignMtnState) { sch.setMaintenanceState(MaintenanceState.ON); + LOG.info("marking Maintenance=ON on " + sch.getHostName()); } - LOG.info("Decommissioning " + slaveCompType + " and marking Maintenance=ON on " + sch.getHostName()); + LOG.info("Decommissioning " + slaveCompType + " on " + sch.getHostName()); } if (filteredIncludedHosts.contains(sch.getHostName())) { sch.setComponentAdminState(HostComponentAdminState.INSERVICE); if (alignMtnState) { sch.setMaintenanceState(MaintenanceState.OFF); + LOG.info("marking Maintenance=OFF on " + sch.getHostName()); } - LOG.info("Recommissioning " + slaveCompType + " and marking Maintenance=OFF on " + sch.getHostName()); + LOG.info("Recommissioning " + slaveCompType + " on " + sch.getHostName()); } } @@ -1008,7 +1010,7 @@ public class AmbariCustomCommandExecutionHelper { } if (!serviceName.equals(Service.Type.HBASE.name()) || hostName.equals(primaryCandidate)) { - commandParams.put(UPDATE_EXCLUDE_FILE_ONLY, "false"); + commandParams.put(UPDATE_FILES_ONLY, "false"); addCustomCommandAction(commandContext, commandFilter, stage, commandParams, commandDetail.toString()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 82171f6..a34422d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -42,6 +42,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.UNLIMITED import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_GROUPS; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION; +import static org.apache.ambari.server.controller.AmbariCustomCommandExecutionHelper.masterToSlaveMappingForDecom; import java.io.File; import java.io.FileNotFoundException; @@ -3337,17 +3338,49 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } } + Map<String, String> serviceMasterForDecommissionMap = new HashMap<>(); for (Map<State, List<ServiceComponentHost>> stateScHostMap : changedScHosts.values()) { for (Entry<State, List<ServiceComponentHost>> entry : stateScHostMap.entrySet()) { State newState = entry.getKey(); for (ServiceComponentHost sch : entry.getValue()) { + String componentName = sch.getServiceComponentName(); + //Create map for include/exclude files refresh + if (masterToSlaveMappingForDecom.containsValue(componentName) && + sch.getState() == State.INIT && newState == State.INSTALLED) { + String serviceName = sch.getServiceName(); + String masterComponentName = null; + for (Entry<String, String> entrySet : masterToSlaveMappingForDecom.entrySet()) { + if (entrySet.getValue().equals(componentName)) { + masterComponentName = entrySet.getKey(); + } + } + try { + Service s = cluster.getService(serviceName); + //Filter services whose masters are not started + if (s.getServiceComponent(masterComponentName).getDesiredState() == State.STARTED) { + serviceMasterForDecommissionMap.put(serviceName, masterComponentName); + } else { + LOG.info(String.format("Not adding %s service from include/exclude files refresh map because it's master is not started", serviceName)); + } + } catch (AmbariException e) { + LOG.error("Exception during INIT masters cleanup : ", e); + } + } + + //actually set the new state sch.setDesiredState(newState); } } } + try { + createAndExecuteRefreshIncludeExcludeFilesActionForMasters(serviceMasterForDecommissionMap, cluster.getClusterName()); + } catch (AmbariException e) { + LOG.error("Exception during refresh include exclude files action : ", e); + } + if (ignoredScHosts != null) { for (ServiceComponentHost scHost : ignoredScHosts) { scHost.setDesiredState(scHost.getState()); @@ -3602,18 +3635,40 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } } + Map<String, Map<String, String>> clusterServiceMasterForDecommissionMap = new HashMap<>(); + for (Entry<ServiceComponent, Set<ServiceComponentHost>> entry : safeToRemoveSCHs.entrySet()) { for (ServiceComponentHost componentHost : entry.getValue()) { try { deleteHostComponent(entry.getKey(), componentHost); deleteStatusMetaData.addDeletedKey(componentHost.getHostName() + "/" + componentHost.getServiceComponentName()); - + //create cluster-master-service map to update all include/exclude files in one action + String componentName = componentHost.getServiceComponentName(); + if (masterToSlaveMappingForDecom.containsValue(componentName)) { + String masterComponentName = null; + for (Entry<String, String> entrySet : masterToSlaveMappingForDecom.entrySet()) { + if (entrySet.getValue().equals(componentName)) { + masterComponentName = entrySet.getKey(); + } + } + if (clusterServiceMasterForDecommissionMap.containsKey(componentHost.getClusterName())) { + clusterServiceMasterForDecommissionMap.get(componentHost.getClusterName()).put(componentHost.getServiceName(), masterComponentName); + } else { + Map<String, String> tempMap = new HashMap<>(); + tempMap.put(componentHost.getServiceName(), masterComponentName); + clusterServiceMasterForDecommissionMap.put(componentHost.getClusterName(), tempMap); + } + } } catch (Exception ex) { deleteStatusMetaData.addException(componentHost.getHostName() + "/" + componentHost.getServiceComponentName(), ex); } } } + for (String cluster : clusterServiceMasterForDecommissionMap.keySet()) { + createAndExecuteRefreshIncludeExcludeFilesActionForMasters(clusterServiceMasterForDecommissionMap.get(cluster), cluster); + } + //Do not break behavior for existing clients where delete request contains only 1 host component. //Response for these requests will have empty body with appropriate error code. if (deleteStatusMetaData.getDeletedKeys().size() + deleteStatusMetaData.getExceptionForKeys().size() == 1) { @@ -3636,7 +3691,6 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } private void deleteHostComponent(ServiceComponent serviceComponent, ServiceComponentHost componentHost) throws AmbariException { - String included_hostname = componentHost.getHostName(); String serviceName = serviceComponent.getServiceName(); String master_component_name = null; String slave_component_name = componentHost.getServiceComponentName(); @@ -3644,37 +3698,15 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle State slaveState = componentHost.getState(); //Delete hostcomponents serviceComponent.deleteServiceComponentHosts(componentHost.getHostName()); - // If deleted hostcomponents support decomission and were decommited and stopped - if (AmbariCustomCommandExecutionHelper.masterToSlaveMappingForDecom.containsValue(slave_component_name) + // If deleted hostcomponents support decomission and were decommited and stopped or in unknown state + if (masterToSlaveMappingForDecom.containsValue(slave_component_name) && desiredAdminState.equals(HostComponentAdminState.DECOMMISSIONED) - && slaveState.equals(State.INSTALLED)) { - - for (Entry<String, String> entrySet : AmbariCustomCommandExecutionHelper.masterToSlaveMappingForDecom.entrySet()) { + && (slaveState.equals(State.INSTALLED) || slaveState.equals(State.UNKNOWN))) { + for (Entry<String, String> entrySet : masterToSlaveMappingForDecom.entrySet()) { if (entrySet.getValue().equals(slave_component_name)) { master_component_name = entrySet.getKey(); } } - //Clear exclud file or draining list except HBASE - if (!serviceName.equals(Service.Type.HBASE.toString())) { - HashMap<String, String> requestProperties = new HashMap<>(); - requestProperties.put("context", "Remove host " + - included_hostname + " from exclude file"); - requestProperties.put("exclusive", "true"); - HashMap<String, String> params = new HashMap<>(); - params.put("included_hosts", included_hostname); - params.put("slave_type", slave_component_name); - params.put(AmbariCustomCommandExecutionHelper.UPDATE_EXCLUDE_FILE_ONLY, "true"); - - //Create filter for RECOMISSION command - RequestResourceFilter resourceFilter - = new RequestResourceFilter(serviceName, master_component_name, null); - //Create request for RECOMISSION command - ExecuteActionRequest actionRequest = new ExecuteActionRequest( - serviceComponent.getClusterName(), AmbariCustomCommandExecutionHelper.DECOMMISSION_COMMAND_NAME, null, - Collections.singletonList(resourceFilter), null, params, true); - //Send request - createAction(actionRequest, requestProperties); - } //Mark master component as needed to restart for remove host info from components UI Cluster cluster = clusters.getCluster(serviceComponent.getClusterName()); @@ -3689,6 +3721,40 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } } + /** + * Creates and triggers an action to update include and exclude files for the master components depending on current cluster topology and components state + * @param serviceMasterMap + * @param clusterName + * @throws AmbariException + */ + private void createAndExecuteRefreshIncludeExcludeFilesActionForMasters(Map<String, String> serviceMasterMap, String clusterName) throws AmbariException { + //Clear include/exclude files or draining list except HBASE + serviceMasterMap.remove(Service.Type.HBASE.toString()); + //exit if empty + if (serviceMasterMap.isEmpty()) { + return; + } + LOG.debug("Refresh include/exclude files action will be executed for " + serviceMasterMap); + HashMap<String, String> requestProperties = new HashMap<>(); + requestProperties.put("context", "Update Include and Exclude Files for " + serviceMasterMap.keySet().toString()); + requestProperties.put("exclusive", "true"); + HashMap<String, String> params = new HashMap<>(); + params.put(AmbariCustomCommandExecutionHelper.UPDATE_FILES_ONLY, "false"); + + //Create filter for command + List<RequestResourceFilter> resourceFilters = new ArrayList<>(serviceMasterMap.size()); + for (String serviceName : serviceMasterMap.keySet()) { + resourceFilters.add(new RequestResourceFilter(serviceName, serviceMasterMap.get(serviceName), null)); + } + + //Create request for command + ExecuteActionRequest actionRequest = new ExecuteActionRequest( + clusterName, AmbariCustomCommandExecutionHelper.DECOMMISSION_COMMAND_NAME, null, + resourceFilters, null, params, true); + //Send action + createAction(actionRequest, requestProperties); + } + @Override public void deleteUsers(Set<UserRequest> requests) throws AmbariException { http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java index 71e105e..3fb6f07 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java @@ -980,6 +980,7 @@ public class HostResourceProvider extends AbstractControllerResourceProvider { throw new AmbariException(reason.toString()); } } else { +// TODO why host with all components stopped can't be deleted? This functional is implemented and only this validation stops the request. if (!componentsToRemove.isEmpty()) { StringBuilder reason = new StringBuilder("Cannot remove host ") .append(hostName) http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml index aad2db0..4eab367 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml @@ -99,6 +99,12 @@ excluded.</description> <on-ambari-upgrade add="true"/> </property> + <property> + <name>manage.include.files</name> + <value>false</value> + <description>If true Ambari will manage include file if dfs.hosts is configured.</description> + <on-ambari-upgrade add="false"/> + </property> <!-- <property> <name>dfs.hosts</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py index f15c5d6..6f702d3 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py @@ -165,6 +165,14 @@ def hdfs(component=None): owner=params.hdfs_user, mode="f", ) + + if params.hdfs_include_file: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=params.hdfs_user, + mode="f", + ) + pass if params.service_map.has_key(component): service_name = params.service_map[component] ServiceConfig(service_name, http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py index 28036cf..e8a591c 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py @@ -110,6 +110,14 @@ def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None, group=params.user_group ) + if params.hdfs_include_file: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=params.hdfs_user, + group=params.user_group + ) + pass + if do_format and not params.hdfs_namenode_format_disabled: format_namenode() pass @@ -435,7 +443,15 @@ def decommission(): group=user_group ) - if not params.update_exclude_file_only: + if params.hdfs_include_file: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=params.hdfs_user, + group=params.user_group + ) + pass + + if not params.update_files_only: Execute(nn_kinit_cmd, user=hdfs_user ) @@ -462,6 +478,14 @@ def decommission(): owner=hdfs_user ) + if params.hdfs_include_file: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=params.hdfs_user, + group=params.user_group + ) + pass + if params.dfs_ha_enabled: # due to a bug in hdfs, refreshNodes will not run on both namenodes so we # need to execute each command scoped to a particular namenode http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index f98aafa..1d19175 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -161,7 +161,13 @@ falcon_user = config['configurations']['falcon-env']['falcon_user'] #exclude file hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", []) exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude'] -update_exclude_file_only = default("/commandParams/update_exclude_file_only",False) +slave_hosts = default("/clusterHostInfo/slave_hosts", []) +include_file_path = default("/configurations/hdfs-site/dfs.hosts", None) +hdfs_include_file = None +manage_include_files = default("/configurations/hdfs-site/manage.include.files", False) +if include_file_path and manage_include_files: + hdfs_include_file = list(set(slave_hosts) - set(hdfs_exclude_file)) +update_files_only = default("/commandParams/update_files_only",False) command_phase = default("/commandParams/phase","") klist_path_local = get_klist_path(default('/configurations/kerberos-env/executable_search_paths', None)) @@ -170,7 +176,6 @@ kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executab hostname = config["hostname"] public_hostname = config["public_hostname"] rm_host = default("/clusterHostInfo/rm_host", []) -slave_hosts = default("/clusterHostInfo/slave_hosts", []) oozie_servers = default("/clusterHostInfo/oozie_server", []) hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", []) hive_server_host = default("/clusterHostInfo/hive_server_host", []) @@ -557,4 +562,4 @@ if enable_ranger_hdfs: # need this to capture cluster name from where ranger hdfs plugin is enabled cluster_name = config['clusterName'] -# ranger hdfs plugin section end \ No newline at end of file +# ranger hdfs plugin section end http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_windows.py index 4a43626..3e712b3 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_windows.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_windows.py @@ -40,6 +40,13 @@ dfs_data_dir = config['configurations']['hdfs-site']['dfs.datanode.data.dir'] #decomission hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", []) exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude'] +include_file_path = default("/configurations/hdfs-site/dfs.hosts", None) +hdfs_include_file = None +manage_include_files = default("/configurations/hdfs-site/manage.include.files", False) +if include_file_path and manage_include_files: + slave_hosts = default("/clusterHostInfo/slave_hosts", []) + hdfs_include_file = list(set(slave_hosts) - set(hdfs_exclude_file)) +update_files_only = default("/commandParams/update_files_only",False) # HDFS High Availability properties dfs_ha_enabled = False dfs_ha_nameservices = default("/configurations/hdfs-site/dfs.internal.nameservices", None) http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/include_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/include_hosts_list.j2 new file mode 100644 index 0000000..115a8a4 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/include_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in hdfs_include_file %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml index d0d0ede..a65b801 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml @@ -408,6 +408,12 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>manage.include.files</name> + <value>false</value> + <description>If true Ambari will manage include file if yarn.resourcemanager.nodes.include-path is configured.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>yarn.http.policy</name> <value>HTTP_ONLY</value> <description> http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index f28274b..5f400a3 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -234,11 +234,17 @@ user_group = config['configurations']['cluster-env']['user_group'] exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") +nm_hosts = default("/clusterHostInfo/nm_hosts", []) +#incude file +include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None) +include_hosts = None +manage_include_files = default("/configurations/yarn-site/manage.include.files", False) +if include_file_path and manage_include_files: + include_hosts = list(set(nm_hosts) - set(exclude_hosts)) + ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", [])) has_ats = not len(ats_host) == 0 -nm_hosts = default("/clusterHostInfo/nm_hosts", []) - # don't using len(nm_hosts) here, because check can take too much time on large clusters number_of_nm = 1 @@ -338,7 +344,7 @@ HdfsResource = functools.partial( immutable_paths = get_not_managed_resources(), dfs_type = dfs_type ) -update_exclude_file_only = default("/commandParams/update_exclude_file_only",False) +update_files_only = default("/commandParams/update_files_only",False) mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_windows.py index 0f8ce73..652ffd9 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_windows.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_windows.py @@ -56,4 +56,12 @@ hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar" exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") -update_exclude_file_only = config['commandParams']['update_exclude_file_only'] + +nm_hosts = default("/clusterHostInfo/nm_hosts", []) +#incude file +include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None) +include_hosts = None +manage_include_files = default("/configurations/yarn-site/manage.include.files", False) +if include_file_path and manage_include_files: + include_hosts = list(set(nm_hosts) - set(exclude_hosts)) +update_files_only = default("/commandParams/update_files_only",False) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py index cf43f77..ecd8147 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py @@ -91,7 +91,14 @@ class ResourcemanagerWindows(Resourcemanager): mode="f" ) - if params.update_exclude_file_only == False: + if params.include_hosts: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=yarn_user, + mode="f" + ) + + if params.update_files_only == False: Execute(yarn_refresh_cmd, user=yarn_user) @@ -159,7 +166,14 @@ class ResourcemanagerDefault(Resourcemanager): group=user_group ) - if params.update_exclude_file_only == False: + if params.include_hosts: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=yarn_user, + group=user_group + ) + + if params.update_files_only == False: Execute(yarn_refresh_cmd, environment= {'PATH' : params.execute_path }, user=yarn_user) http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/include_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/include_hosts_list.j2 new file mode 100644 index 0000000..42e33c0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/include_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in include_hosts %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/configuration/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/configuration/hdfs-site.xml b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/configuration/hdfs-site.xml index 87684df..66d25cf 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/configuration/hdfs-site.xml +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/configuration/hdfs-site.xml @@ -72,6 +72,12 @@ excluded.</description> <on-ambari-upgrade add="true"/> </property> + <property> + <name>manage.include.files</name> + <value>false</value> + <description>If true Ambari will manage include file if dfs.hosts is configured.</description> + <on-ambari-upgrade add="false"/> + </property> <!-- <property> <name>dfs.hosts</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/hdfs_namenode.py index 6de7735..19751f6 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/hdfs_namenode.py @@ -143,8 +143,16 @@ def decommission(): owner=hdfs_user, group=user_group ) - - if not params.update_exclude_file_only: + + if params.hdfs_include_file: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=params.hdfs_user, + group=params.user_group + ) + pass + + if not params.update_files_only: Execute(nn_kinit_cmd, user=hdfs_user ) http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py index 19e223c..9cf163a 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py @@ -55,13 +55,18 @@ falcon_user = config['configurations']['falcon-env']['falcon_user'] #exclude file hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", []) exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude'] -update_exclude_file_only = default("/commandParams/update_exclude_file_only",False) +slave_hosts = default("/clusterHostInfo/slave_hosts", []) +include_file_path = default("/configurations/hdfs-site/dfs.hosts", None) +hdfs_include_file = None +manage_include_files = default("/configurations/hdfs-site/manage.include.files", False) +if include_file_path and manage_include_files: + hdfs_include_file = list(set(slave_hosts) - set(hdfs_exclude_file)) +update_files_only = default("/commandParams/update_files_only",False) kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) #hosts hostname = config["hostname"] rm_host = default("/clusterHostInfo/rm_host", []) -slave_hosts = default("/clusterHostInfo/slave_hosts", []) oozie_servers = default("/clusterHostInfo/oozie_server", []) hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", []) hive_server_host = default("/clusterHostInfo/hive_server_host", []) @@ -241,4 +246,4 @@ ttnode_heapsize = "1024m" dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize'] mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce") mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce") -script_https_protocol = Script.get_force_https_protocol_name() \ No newline at end of file +script_https_protocol = Script.get_force_https_protocol_name() http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/templates/include_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/templates/include_hosts_list.j2 new file mode 100644 index 0000000..115a8a4 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/templates/include_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in hdfs_include_file %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/configuration/yarn-site.xml index 8e9b8b1..3cb5add 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/configuration/yarn-site.xml @@ -334,6 +334,12 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>manage.include.files</name> + <value>false</value> + <description>If true Ambari will manage include file if yarn.resourcemanager.nodes.include-path is configured.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>yarn.timeline-service.enabled</name> <value>true</value> <description>Indicate to clients whether timeline service is enabled or not. http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/params.py index 33496cfe..87b5992 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/params.py @@ -118,6 +118,14 @@ user_group = config['configurations']['cluster-env']['user_group'] exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") +nm_hosts = default("/clusterHostInfo/nm_hosts", []) +#incude file +include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None) +include_hosts = None +manage_include_files = default("/configurations/yarn-site/manage.include.files", False) +if include_file_path and manage_include_files: + include_hosts = list(set(nm_hosts) - set(exclude_hosts)) + hostname = config['hostname'] ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", [])) @@ -162,7 +170,7 @@ HdfsDirectory = functools.partial( kinit_path_local = kinit_path_local, bin_dir = hadoop_bin_dir ) -update_exclude_file_only = default("/commandParams/update_exclude_file_only",False) +update_files_only = default("/commandParams/update_files_only",False) mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/resourcemanager.py index 4d40d68..8bd76bf 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/resourcemanager.py @@ -89,7 +89,14 @@ class Resourcemanager(Script): group=user_group ) - if params.update_exclude_file_only == False: + if params.include_hosts: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=yarn_user, + group=user_group + ) + + if params.update_files_only == False: Execute(yarn_refresh_cmd, environment= {'PATH' : params.execute_path }, user=yarn_user) http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/templates/include_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/templates/include_hosts_list.j2 new file mode 100644 index 0000000..42e33c0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/templates/include_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in include_hosts %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/configuration/yarn-site.xml index 93a35cd..f2da835 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/configuration/yarn-site.xml @@ -334,6 +334,12 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>manage.include.files</name> + <value>false</value> + <description>If true Ambari will manage include file if yarn.resourcemanager.nodes.include-path is configured.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>yarn.timeline-service.enabled</name> <value>true</value> <description>Indicate to clients whether timeline service is enabled or not. http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/params.py index 5a7e508..71f30f0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/params.py @@ -98,6 +98,13 @@ yarn_container_bin = "/usr/lib/hadoop-yarn/bin" #exclude file exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) exclude_file_path = config['configurations']['yarn-site']['yarn.resourcemanager.nodes.exclude-path'] +nm_hosts = default("/clusterHostInfo/nm_hosts", []) +#incude file +include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None) +include_hosts = None +manage_include_files = default("/configurations/yarn-site/manage.include.files", False) +if include_file_path and manage_include_files: + include_hosts = list(set(nm_hosts) - set(exclude_hosts)) hostname = config['hostname'] @@ -128,7 +135,7 @@ HdfsDirectory = functools.partial( keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local ) -update_exclude_file_only = config['commandParams']['update_exclude_file_only'] +update_files_only = default("/commandParams/update_files_only",False) hadoop_bin = "/usr/lib/hadoop/sbin" mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/resourcemanager.py index a286ae3..f92938b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/resourcemanager.py @@ -79,7 +79,14 @@ class Resourcemanager(Script): group=user_group ) - if params.update_exclude_file_only == False: + if params.include_hosts: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=yarn_user, + group=user_group + ) + + if params.update_files_only == False: Execute(yarn_refresh_cmd, user=yarn_user) pass http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2 new file mode 100644 index 0000000..c7ce416 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in exclude_hosts %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 new file mode 100644 index 0000000..42e33c0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in include_hosts %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/configuration/yarn-site.xml index 93a35cd..f2da835 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/configuration/yarn-site.xml @@ -334,6 +334,12 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>manage.include.files</name> + <value>false</value> + <description>If true Ambari will manage include file if yarn.resourcemanager.nodes.include-path is configured.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>yarn.timeline-service.enabled</name> <value>true</value> <description>Indicate to clients whether timeline service is enabled or not. http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py index 5a7e508..e39c4d1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py @@ -98,7 +98,13 @@ yarn_container_bin = "/usr/lib/hadoop-yarn/bin" #exclude file exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) exclude_file_path = config['configurations']['yarn-site']['yarn.resourcemanager.nodes.exclude-path'] - +nm_hosts = default("/clusterHostInfo/nm_hosts", []) +#incude file +include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None) +include_hosts = None +manage_include_files = default("/configurations/yarn-site/manage.include.files", False) +if include_file_path and manage_include_files: + include_hosts = list(set(nm_hosts) - set(exclude_hosts)) hostname = config['hostname'] if security_enabled: @@ -128,7 +134,7 @@ HdfsDirectory = functools.partial( keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local ) -update_exclude_file_only = config['commandParams']['update_exclude_file_only'] +update_files_only = default("/commandParams/update_files_only",False) hadoop_bin = "/usr/lib/hadoop/sbin" mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py index a286ae3..f92938b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py @@ -79,7 +79,14 @@ class Resourcemanager(Script): group=user_group ) - if params.update_exclude_file_only == False: + if params.include_hosts: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=yarn_user, + group=user_group + ) + + if params.update_files_only == False: Execute(yarn_refresh_cmd, user=yarn_user) pass http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 new file mode 100644 index 0000000..42e33c0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in include_hosts %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index 43e2ed4..81b3580 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -6345,8 +6345,8 @@ public class AmbariManagementControllerTest { execCmd = storedTasks.get(0).getExecutionCommandWrapper ().getExecutionCommand(); Map<String, String> cmdParams = execCmd.getCommandParams(); - Assert.assertTrue(cmdParams.containsKey("update_exclude_file_only")); - Assert.assertTrue(cmdParams.get("update_exclude_file_only").equals("false")); + Assert.assertTrue(cmdParams.containsKey("update_files_only")); + Assert.assertTrue(cmdParams.get("update_files_only").equals("false")); Assert.assertNotNull(storedTasks); Assert.assertEquals(1, storedTasks.size()); Assert.assertEquals(HostComponentAdminState.DECOMMISSIONED, scHost.getComponentAdminState()); @@ -6401,8 +6401,8 @@ public class AmbariManagementControllerTest { Assert.assertTrue(hrc.getCommandDetail().contains(host1)); Assert.assertTrue(hrc.getCommandDetail().contains(host2)); cmdParams = hrc.getExecutionCommandWrapper().getExecutionCommand().getCommandParams(); - if(!cmdParams.containsKey("update_exclude_file_only") - || !cmdParams.get("update_exclude_file_only").equals("true")) { + if(!cmdParams.containsKey("update_files_only") + || !cmdParams.get("update_files_only").equals("true")) { countRefresh++; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py index acd40b0..862a17e 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py @@ -1041,7 +1041,7 @@ class TestNamenode(RMFTestCase): bin_dir = '/usr/bin') self.assertNoMoreResources() - def test_decommission_update_exclude_file_only(self): + def test_decommission_update_files_only(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py", classname = "NameNode", command = "decommission", http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.0.6/configs/altfs_plus_hdfs.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/altfs_plus_hdfs.json b/ambari-server/src/test/python/stacks/2.0.6/configs/altfs_plus_hdfs.json index ea00a37..f928073 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/altfs_plus_hdfs.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/altfs_plus_hdfs.json @@ -33,7 +33,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.0.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json index 87a9034..f337f41 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json @@ -39,7 +39,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json index 71423c8..f3e8dc3 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json @@ -32,7 +32,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha.json index 009ff6d..7b0f78d 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha.json @@ -35,7 +35,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha_2.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha_2.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha_2.json index 2b078c3..01f0efc 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha_2.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha_2.json @@ -35,7 +35,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_non_hdfs.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_non_hdfs.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_non_hdfs.json index 571b737..0cbd322 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_non_hdfs.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_non_hdfs.json @@ -33,7 +33,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.0.6/configs/default_no_install.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_no_install.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_no_install.json index 7fdb449..cfcf5e1 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_no_install.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_no_install.json @@ -35,7 +35,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.0.6/configs/default_oozie_mysql.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_oozie_mysql.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_oozie_mysql.json index 7378b68..7db73ab 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_oozie_mysql.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_oozie_mysql.json @@ -36,7 +36,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.0.6/configs/default_update_exclude_file_only.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_update_exclude_file_only.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_update_exclude_file_only.json index 2ea07e4..f50a207 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_update_exclude_file_only.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_update_exclude_file_only.json @@ -33,7 +33,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "true", + "update_files_only" : "true", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.0.6/configs/default_with_bucket.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_with_bucket.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_with_bucket.json index 5080d30..c1eb868 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_with_bucket.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_with_bucket.json @@ -38,7 +38,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.0.6/configs/flume_22.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/flume_22.json b/ambari-server/src/test/python/stacks/2.0.6/configs/flume_22.json index 6ec9ec9..c99d10b 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/flume_22.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/flume_22.json @@ -32,7 +32,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "output_file":"HDFS_CLIENT-configs.tar.gz" http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.0.6/configs/flume_only.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/flume_only.json b/ambari-server/src/test/python/stacks/2.0.6/configs/flume_only.json index 1550715..1a4d676 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/flume_only.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/flume_only.json @@ -31,7 +31,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json index f572413..52a1fde 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json @@ -35,7 +35,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json index 5147603..06682bc 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json @@ -35,7 +35,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.0.6/configs/oozie_existing_sqla.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/oozie_existing_sqla.json b/ambari-server/src/test/python/stacks/2.0.6/configs/oozie_existing_sqla.json index 8d12b98..4ffa29f 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/oozie_existing_sqla.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/oozie_existing_sqla.json @@ -34,7 +34,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json index 5327865..76a110e 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json @@ -38,7 +38,7 @@ "script": "scripts/yarn_client.py", "excluded_hosts": "host1", "mark_draining_only" : "false", - "update_exclude_file_only" : "false" + "update_files_only" : "false" }, "taskId": 186, "public_hostname": "c6401.ambari.apache.org", http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json b/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json index b4342ad..475a6f9 100644 --- a/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json +++ b/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json @@ -31,7 +31,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json index 9dcb451..7622212 100644 --- a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json +++ b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json @@ -38,7 +38,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.5/configs/hsi_default_for_restart.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default_for_restart.json b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default_for_restart.json index f6de1c4..c2320ba 100644 --- a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default_for_restart.json +++ b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default_for_restart.json @@ -39,7 +39,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/ambari-server/src/test/python/stacks/2.5/configs/hsi_ha.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/configs/hsi_ha.json b/ambari-server/src/test/python/stacks/2.5/configs/hsi_ha.json index 3fd9f72..7b79d84 100644 --- a/ambari-server/src/test/python/stacks/2.5/configs/hsi_ha.json +++ b/ambari-server/src/test/python/stacks/2.5/configs/hsi_ha.json @@ -38,7 +38,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/configuration/yarn-site.xml b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/configuration/yarn-site.xml index 59ff82b..f3ea462 100755 --- a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/configuration/yarn-site.xml +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/configuration/yarn-site.xml @@ -388,6 +388,12 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>manage.include.files</name> + <value>false</value> + <description>If true Ambari will manage include file if yarn.resourcemanager.nodes.include-path is configured.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>yarn.http.policy</name> <value>HTTP_ONLY</value> <description> http://git-wip-us.apache.org/repos/asf/ambari/blob/7764e387/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/params_linux.py b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/params_linux.py index 4d42861..da54b7c 100755 --- a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/params_linux.py +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/YARN/package/scripts/params_linux.py @@ -233,6 +233,13 @@ has_ats = not len(ats_host) == 0 nm_hosts = default("/clusterHostInfo/nm_hosts", []) +#incude file +include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None) +include_hosts = None +manage_include_files = default("/configurations/yarn-site/manage.include.files", False) +if include_file_path and manage_include_files: + include_hosts = list(set(nm_hosts) - set(exclude_hosts)) + # don't using len(nm_hosts) here, because check can take too much time on large clusters number_of_nm = 1 @@ -315,7 +322,7 @@ HdfsResource = functools.partial( immutable_paths = get_not_managed_resources(), dfs_type = dfs_type ) -update_exclude_file_only = default("/commandParams/update_exclude_file_only",False) +update_files_only = default("/commandParams/update_files_only",False) mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group)
