AMBARI-21345 Add host doesn't fully add a node when include/exclude files are used (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cc412e66 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cc412e66 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cc412e66 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: cc412e66156d5a887a725015537dcb75b0caf986 Parents: 93fe848 Author: Dmytro Sen <[email protected]> Authored: Mon Jul 17 13:36:58 2017 +0300 Committer: Dmytro Sen <[email protected]> Committed: Mon Jul 17 13:36:58 2017 +0300 ---------------------------------------------------------------------- .../AmbariCustomCommandExecutionHelper.java | 14 ++- .../AmbariManagementControllerImpl.java | 121 ++++++++++++++----- .../internal/HostResourceProvider.java | 1 + .../HDFS/2.1.0.2.0/configuration/hdfs-site.xml | 6 + .../HDFS/2.1.0.2.0/package/scripts/hdfs.py | 8 ++ .../2.1.0.2.0/package/scripts/hdfs_namenode.py | 25 +++- .../2.1.0.2.0/package/scripts/params_linux.py | 9 +- .../2.1.0.2.0/package/scripts/params_windows.py | 7 ++ .../package/templates/include_hosts_list.j2 | 21 ++++ .../HDFS/3.0.0.3.0/package/scripts/hdfs.py | 8 ++ .../3.0.0.3.0/package/scripts/hdfs_namenode.py | 17 ++- .../3.0.0.3.0/package/scripts/params_linux.py | 8 +- .../3.0.0.3.0/package/scripts/params_windows.py | 7 ++ .../package/templates/include_hosts_list.j2 | 21 ++++ .../YARN/2.1.0.2.0/configuration/yarn-site.xml | 6 + .../2.1.0.2.0/package/scripts/params_linux.py | 12 +- .../2.1.0.2.0/package/scripts/params_windows.py | 10 +- .../package/scripts/resourcemanager.py | 18 ++- .../package/templates/include_hosts_list.j2 | 21 ++++ .../YARN/3.0.0.3.0/configuration/yarn-site.xml | 6 + .../3.0.0.3.0/package/scripts/params_linux.py | 11 +- .../3.0.0.3.0/package/scripts/params_windows.py | 10 +- .../package/scripts/resourcemanager.py | 18 ++- .../package/templates/include_hosts_list.j2 | 21 ++++ .../services/HDFS/configuration/hdfs-site.xml | 6 + .../HDFS/package/scripts/hdfs_namenode.py | 12 +- .../0.8/services/HDFS/package/scripts/params.py | 11 +- .../package/templates/include_hosts_list.j2 | 21 ++++ .../services/YARN/configuration/yarn-site.xml | 6 + .../0.8/services/YARN/package/scripts/params.py | 10 +- .../YARN/package/scripts/resourcemanager.py | 9 +- .../package/templates/include_hosts_list.j2 | 21 ++++ .../services/YARN/configuration/yarn-site.xml | 6 + .../services/YARN/package/scripts/params.py | 9 +- .../YARN/package/scripts/resourcemanager.py | 9 +- .../package/templates/exclude_hosts_list.j2 | 21 ++++ .../package/templates/include_hosts_list.j2 | 21 ++++ .../services/YARN/configuration/yarn-site.xml | 6 + .../services/YARN/package/scripts/params.py | 10 +- .../YARN/package/scripts/resourcemanager.py | 9 +- .../package/templates/include_hosts_list.j2 | 21 ++++ .../AmbariManagementControllerTest.java | 8 +- .../python/stacks/2.0.6/HDFS/test_namenode.py | 2 +- .../stacks/2.0.6/configs/altfs_plus_hdfs.json | 2 +- .../python/stacks/2.0.6/configs/default.json | 2 +- .../2.0.6/configs/default_ams_embedded.json | 2 +- .../2.0.6/configs/default_hive_nn_ha.json | 2 +- .../2.0.6/configs/default_hive_nn_ha_2.json | 2 +- .../2.0.6/configs/default_hive_non_hdfs.json | 2 +- .../2.0.6/configs/default_no_install.json | 2 +- .../2.0.6/configs/default_oozie_mysql.json | 2 +- .../default_update_exclude_file_only.json | 2 +- .../2.0.6/configs/default_with_bucket.json | 2 +- .../python/stacks/2.0.6/configs/flume_22.json | 2 +- .../python/stacks/2.0.6/configs/flume_only.json | 2 +- .../stacks/2.0.6/configs/hbase_no_phx.json | 2 +- .../stacks/2.0.6/configs/hbase_with_phx.json | 2 +- .../2.0.6/configs/oozie_existing_sqla.json | 2 +- .../stacks/2.0.6/configs/repository_file.json | 2 +- .../python/stacks/2.0.6/configs/secured.json | 2 +- .../test/python/stacks/2.3/configs/ats_1_5.json | 2 +- .../python/stacks/2.5/configs/hsi_default.json | 2 +- .../2.5/configs/hsi_default_for_restart.json | 2 +- .../test/python/stacks/2.5/configs/hsi_ha.json | 2 +- .../services/YARN/configuration/yarn-site.xml | 6 + .../YARN/package/scripts/params_linux.py | 9 +- .../YARN/package/scripts/params_windows.py | 10 +- .../YARN/package/scripts/resourcemanager.py | 18 ++- .../package/templates/include_hosts_list.j2 | 21 ++++ 69 files changed, 638 insertions(+), 92 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index 28aa4e4..aeb5a9c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -137,7 +137,7 @@ public class AmbariCustomCommandExecutionHelper { public final static String DECOM_EXCLUDED_HOSTS = "excluded_hosts"; public final static String DECOM_SLAVE_COMPONENT = "slave_type"; public final static String HBASE_MARK_DRAINING_ONLY = "mark_draining_only"; - public final static String UPDATE_EXCLUDE_FILE_ONLY = "update_exclude_file_only"; + public final static String UPDATE_FILES_ONLY = "update_files_only"; private final static String ALIGN_MAINTENANCE_STATE = "align_maintenance_state"; @@ -910,9 +910,9 @@ public class AmbariCustomCommandExecutionHelper { @Override public boolean shouldHostBeRemoved(final String hostname) throws AmbariException { - //Get UPDATE_EXCLUDE_FILE_ONLY parameter as string + //Get UPDATE_FILES_ONLY parameter as string String upd_excl_file_only_str = actionExecutionContext.getParameters() - .get(UPDATE_EXCLUDE_FILE_ONLY); + .get(UPDATE_FILES_ONLY); String decom_incl_hosts_str = actionExecutionContext.getParameters() .get(DECOM_INCLUDED_HOSTS); @@ -986,15 +986,17 @@ public class AmbariCustomCommandExecutionHelper { listOfExcludedHosts.add(sch.getHostName()); if (alignMtnState) { sch.setMaintenanceState(MaintenanceState.ON); + LOG.info("marking Maintenance=ON on " + sch.getHostName()); } - LOG.info("Decommissioning " + slaveCompType + " and marking Maintenance=ON on " + sch.getHostName()); + LOG.info("Decommissioning " + slaveCompType + " on " + sch.getHostName()); } if (filteredIncludedHosts.contains(sch.getHostName())) { sch.setComponentAdminState(HostComponentAdminState.INSERVICE); if (alignMtnState) { sch.setMaintenanceState(MaintenanceState.OFF); + LOG.info("marking Maintenance=OFF on " + sch.getHostName()); } - LOG.info("Recommissioning " + slaveCompType + " and marking Maintenance=OFF on " + sch.getHostName()); + LOG.info("Recommissioning " + slaveCompType + " on " + sch.getHostName()); } } @@ -1048,7 +1050,7 @@ public class AmbariCustomCommandExecutionHelper { } if (!serviceName.equals(Service.Type.HBASE.name()) || hostName.equals(primaryCandidate)) { - commandParams.put(UPDATE_EXCLUDE_FILE_ONLY, "false"); + commandParams.put(UPDATE_FILES_ONLY, "false"); addCustomCommandAction(commandContext, commandFilter, stage, commandParams, commandDetail.toString(), null); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 5639dc1..433ed56 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -42,6 +42,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.UNLIMITED import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_GROUPS; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION; +import static org.apache.ambari.server.controller.AmbariCustomCommandExecutionHelper.masterToSlaveMappingForDecom; import java.io.File; import java.io.FileNotFoundException; @@ -3334,17 +3335,49 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } } + Map<String, String> serviceMasterForDecommissionMap = new HashMap<>(); for (Map<State, List<ServiceComponentHost>> stateScHostMap : changedScHosts.values()) { for (Entry<State, List<ServiceComponentHost>> entry : stateScHostMap.entrySet()) { State newState = entry.getKey(); for (ServiceComponentHost sch : entry.getValue()) { + String componentName = sch.getServiceComponentName(); + //Create map for include/exclude files refresh + if (masterToSlaveMappingForDecom.containsValue(componentName) && + sch.getState() == State.INIT && newState == State.INSTALLED) { + String serviceName = sch.getServiceName(); + String masterComponentName = null; + for (Entry<String, String> entrySet : masterToSlaveMappingForDecom.entrySet()) { + if (entrySet.getValue().equals(componentName)) { + masterComponentName = entrySet.getKey(); + } + } + try { + Service s = cluster.getService(serviceName); + //Filter services whose masters are not started + if (s.getServiceComponent(masterComponentName).getDesiredState() == State.STARTED) { + serviceMasterForDecommissionMap.put(serviceName, masterComponentName); + } else { + LOG.info(String.format("Not adding %s service from include/exclude files refresh map because it's master is not started", serviceName)); + } + } catch (AmbariException e) { + LOG.error("Exception during INIT masters cleanup : ", e); + } + } + + //actually set the new state sch.setDesiredState(newState); } } } + try { + createAndExecuteRefreshIncludeExcludeFilesActionForMasters(serviceMasterForDecommissionMap, cluster.getClusterName()); + } catch (AmbariException e) { + LOG.error("Exception during refresh include exclude files action : ", e); + } + if (ignoredScHosts != null) { for (ServiceComponentHost scHost : ignoredScHosts) { scHost.setDesiredState(scHost.getState()); @@ -3582,18 +3615,39 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } } + Map<String, Map<String, String>> clusterServiceMasterForDecommissionMap = new HashMap<>(); for (Entry<ServiceComponent, Set<ServiceComponentHost>> entry : safeToRemoveSCHs.entrySet()) { for (ServiceComponentHost componentHost : entry.getValue()) { try { deleteHostComponent(entry.getKey(), componentHost); deleteStatusMetaData.addDeletedKey(componentHost.getHostName() + "/" + componentHost.getServiceComponentName()); - + //create cluster-master-service map to update all include/exclude files in one action + String componentName = componentHost.getServiceComponentName(); + if (masterToSlaveMappingForDecom.containsValue(componentName)) { + String masterComponentName = null; + for (Entry<String, String> entrySet : masterToSlaveMappingForDecom.entrySet()) { + if (entrySet.getValue().equals(componentName)) { + masterComponentName = entrySet.getKey(); + } + } + if (clusterServiceMasterForDecommissionMap.containsKey(componentHost.getClusterName())) { + clusterServiceMasterForDecommissionMap.get(componentHost.getClusterName()).put(componentHost.getServiceName(), masterComponentName); + } else { + Map<String, String> tempMap = new HashMap<>(); + tempMap.put(componentHost.getServiceName(), masterComponentName); + clusterServiceMasterForDecommissionMap.put(componentHost.getClusterName(), tempMap); + } + } } catch (Exception ex) { deleteStatusMetaData.addException(componentHost.getHostName() + "/" + componentHost.getServiceComponentName(), ex); } } } + for (String cluster : clusterServiceMasterForDecommissionMap.keySet()) { + createAndExecuteRefreshIncludeExcludeFilesActionForMasters(clusterServiceMasterForDecommissionMap.get(cluster), cluster); + } + //Do not break behavior for existing clients where delete request contains only 1 host component. //Response for these requests will have empty body with appropriate error code. if (deleteStatusMetaData.getDeletedKeys().size() + deleteStatusMetaData.getExceptionForKeys().size() == 1) { @@ -3616,7 +3670,6 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } private void deleteHostComponent(ServiceComponent serviceComponent, ServiceComponentHost componentHost) throws AmbariException { - String included_hostname = componentHost.getHostName(); String serviceName = serviceComponent.getServiceName(); String master_component_name = null; String slave_component_name = componentHost.getServiceComponentName(); @@ -3624,37 +3677,15 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle State slaveState = componentHost.getState(); //Delete hostcomponents serviceComponent.deleteServiceComponentHosts(componentHost.getHostName()); - // If deleted hostcomponents support decomission and were decommited and stopped - if (AmbariCustomCommandExecutionHelper.masterToSlaveMappingForDecom.containsValue(slave_component_name) + // If deleted hostcomponents support decomission and were decommited and stopped or in unknown state + if (masterToSlaveMappingForDecom.containsValue(slave_component_name) && desiredAdminState.equals(HostComponentAdminState.DECOMMISSIONED) - && slaveState.equals(State.INSTALLED)) { - - for (Entry<String, String> entrySet : AmbariCustomCommandExecutionHelper.masterToSlaveMappingForDecom.entrySet()) { + && (slaveState.equals(State.INSTALLED) || slaveState.equals(State.UNKNOWN))) { + for (Entry<String, String> entrySet : masterToSlaveMappingForDecom.entrySet()) { if (entrySet.getValue().equals(slave_component_name)) { master_component_name = entrySet.getKey(); } } - //Clear exclud file or draining list except HBASE - if (!serviceName.equals(Service.Type.HBASE.toString())) { - HashMap<String, String> requestProperties = new HashMap<>(); - requestProperties.put("context", "Remove host " + - included_hostname + " from exclude file"); - requestProperties.put("exclusive", "true"); - HashMap<String, String> params = new HashMap<>(); - params.put("included_hosts", included_hostname); - params.put("slave_type", slave_component_name); - params.put(AmbariCustomCommandExecutionHelper.UPDATE_EXCLUDE_FILE_ONLY, "true"); - - //Create filter for RECOMISSION command - RequestResourceFilter resourceFilter - = new RequestResourceFilter(serviceName, master_component_name, null); - //Create request for RECOMISSION command - ExecuteActionRequest actionRequest = new ExecuteActionRequest( - serviceComponent.getClusterName(), AmbariCustomCommandExecutionHelper.DECOMMISSION_COMMAND_NAME, null, - Collections.singletonList(resourceFilter), null, params, true); - //Send request - createAction(actionRequest, requestProperties); - } //Mark master component as needed to restart for remove host info from components UI Cluster cluster = clusters.getCluster(serviceComponent.getClusterName()); @@ -3695,6 +3726,40 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } } + /** + * Creates and triggers an action to update include and exclude files for the master components depending on current cluster topology and components state + * @param serviceMasterMap + * @param clusterName + * @throws AmbariException + */ + private void createAndExecuteRefreshIncludeExcludeFilesActionForMasters(Map<String, String> serviceMasterMap, String clusterName) throws AmbariException { + //Clear include/exclude files or draining list except HBASE + serviceMasterMap.remove(Service.Type.HBASE.toString()); + //exit if empty + if (serviceMasterMap.isEmpty()) { + return; + } + LOG.debug("Refresh include/exclude files action will be executed for " + serviceMasterMap); + HashMap<String, String> requestProperties = new HashMap<>(); + requestProperties.put("context", "Update Include and Exclude Files for " + serviceMasterMap.keySet().toString()); + requestProperties.put("exclusive", "true"); + HashMap<String, String> params = new HashMap<>(); + params.put(AmbariCustomCommandExecutionHelper.UPDATE_FILES_ONLY, "false"); + + //Create filter for command + List<RequestResourceFilter> resourceFilters = new ArrayList<>(serviceMasterMap.size()); + for (String serviceName : serviceMasterMap.keySet()) { + resourceFilters.add(new RequestResourceFilter(serviceName, serviceMasterMap.get(serviceName), null)); + } + + //Create request for command + ExecuteActionRequest actionRequest = new ExecuteActionRequest( + clusterName, AmbariCustomCommandExecutionHelper.DECOMMISSION_COMMAND_NAME, null, + resourceFilters, null, params, true); + //Send action + createAction(actionRequest, requestProperties); + } + @Override public void deleteMembers(java.util.Set<MemberRequest> requests) throws AmbariException { for (MemberRequest request : requests) { http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java index 4e2944f..8ef42ba 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java @@ -946,6 +946,7 @@ public class HostResourceProvider extends AbstractControllerResourceProvider { throw new AmbariException(reason.toString()); } } else { +// TODO why host with all components stopped can't be deleted? This functional is implemented and only this validation stops the request. if (!componentsToRemove.isEmpty()) { StringBuilder reason = new StringBuilder("Cannot remove host ") .append(hostName) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml index aad2db0..4eab367 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml @@ -99,6 +99,12 @@ excluded.</description> <on-ambari-upgrade add="true"/> </property> + <property> + <name>manage.include.files</name> + <value>false</value> + <description>If true Ambari will manage include file if dfs.hosts is configured.</description> + <on-ambari-upgrade add="false"/> + </property> <!-- <property> <name>dfs.hosts</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py index 15fda67..e054209 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py @@ -170,6 +170,14 @@ def hdfs(component=None): owner=params.hdfs_user, mode="f", ) + + if params.hdfs_include_file: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=params.hdfs_user, + mode="f", + ) + pass if params.service_map.has_key(component): service_name = params.service_map[component] ServiceConfig(service_name, http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py index 7226d22..cac6e9c 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py @@ -108,6 +108,14 @@ def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None, group=params.user_group ) + if params.hdfs_include_file: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=params.hdfs_user, + group=params.user_group + ) + pass + if do_format and not params.hdfs_namenode_format_disabled: format_namenode() pass @@ -437,7 +445,15 @@ def decommission(): group=user_group ) - if not params.update_exclude_file_only: + if params.hdfs_include_file: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=params.hdfs_user, + group=params.user_group + ) + pass + + if not params.update_files_only: Execute(nn_kinit_cmd, user=hdfs_user ) @@ -464,6 +480,13 @@ def decommission(): owner=hdfs_user ) + if params.hdfs_include_file: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=params.hdfs_user + ) + pass + if params.dfs_ha_enabled: # due to a bug in hdfs, refreshNodes will not run on both namenodes so we # need to execute each command scoped to a particular namenode http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index a9fc179..2854a00 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -163,7 +163,13 @@ falcon_user = config['configurations']['falcon-env']['falcon_user'] #exclude file hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", []) exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude'] -update_exclude_file_only = default("/commandParams/update_exclude_file_only",False) +slave_hosts = default("/clusterHostInfo/slave_hosts", []) +include_file_path = default("/configurations/hdfs-site/dfs.hosts", None) +hdfs_include_file = None +manage_include_files = default("/configurations/hdfs-site/manage.include.files", False) +if include_file_path and manage_include_files: + hdfs_include_file = list(set(slave_hosts) - set(hdfs_exclude_file)) +update_files_only = default("/commandParams/update_files_only",False) command_phase = default("/commandParams/phase","") klist_path_local = get_klist_path(default('/configurations/kerberos-env/executable_search_paths', None)) @@ -172,7 +178,6 @@ kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executab hostname = config["hostname"] public_hostname = config["public_hostname"] rm_host = default("/clusterHostInfo/rm_host", []) -slave_hosts = default("/clusterHostInfo/slave_hosts", []) oozie_servers = default("/clusterHostInfo/oozie_server", []) hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", []) hive_server_host = default("/clusterHostInfo/hive_server_host", []) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_windows.py index 1e47c29..b3ac578 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_windows.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_windows.py @@ -43,6 +43,13 @@ dfs_data_dir = config['configurations']['hdfs-site']['dfs.datanode.data.dir'] #decomission hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", []) exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude'] +include_file_path = default("/configurations/hdfs-site/dfs.hosts", None) +hdfs_include_file = None +manage_include_files = default("/configurations/hdfs-site/manage.include.files", False) +if include_file_path and manage_include_files: + slave_hosts = default("/clusterHostInfo/slave_hosts", []) + hdfs_include_file = list(set(slave_hosts) - set(hdfs_exclude_file)) +update_files_only = default("/commandParams/update_files_only",False) # HDFS High Availability properties dfs_ha_enabled = False dfs_ha_nameservices = default("/configurations/hdfs-site/dfs.internal.nameservices", None) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/include_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/include_hosts_list.j2 new file mode 100644 index 0000000..115a8a4 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/include_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in hdfs_include_file %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py index 15fda67..e054209 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py @@ -170,6 +170,14 @@ def hdfs(component=None): owner=params.hdfs_user, mode="f", ) + + if params.hdfs_include_file: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=params.hdfs_user, + mode="f", + ) + pass if params.service_map.has_key(component): service_name = params.service_map[component] ServiceConfig(service_name, http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs_namenode.py index 181b3c8..5a1f368 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs_namenode.py @@ -436,7 +436,15 @@ def decommission(): group=user_group ) - if not params.update_exclude_file_only: + if params.hdfs_include_file: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=params.hdfs_user, + group=params.user_group + ) + pass + + if not params.update_files_only: Execute(nn_kinit_cmd, user=hdfs_user ) @@ -463,6 +471,13 @@ def decommission(): owner=hdfs_user ) + if params.hdfs_include_file: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=params.hdfs_user + ) + pass + if params.dfs_ha_enabled: # due to a bug in hdfs, refreshNodes will not run on both namenodes so we # need to execute each command scoped to a particular namenode http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py index ad49d81..1581c2a 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py @@ -158,7 +158,13 @@ falcon_user = config['configurations']['falcon-env']['falcon_user'] #exclude file hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", []) exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude'] -update_exclude_file_only = default("/commandParams/update_exclude_file_only",False) +slave_hosts = default("/clusterHostInfo/slave_hosts", []) +include_file_path = default("/configurations/hdfs-site/dfs.hosts", None) +hdfs_include_file = None +manage_include_files = default("/configurations/hdfs-site/manage.include.files", False) +if include_file_path and manage_include_files: + hdfs_include_file = list(set(slave_hosts) - set(hdfs_exclude_file)) +update_files_only = default("/commandParams/update_files_only", False) command_phase = default("/commandParams/phase","") klist_path_local = get_klist_path(default('/configurations/kerberos-env/executable_search_paths', None)) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_windows.py index 70d95a6..6c492d8 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_windows.py +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_windows.py @@ -43,6 +43,13 @@ dfs_data_dir = config['configurations']['hdfs-site']['dfs.datanode.data.dir'] #decomission hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", []) exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude'] +include_file_path = default("/configurations/hdfs-site/dfs.hosts", None) +hdfs_include_file = None +manage_include_files = default("/configurations/hdfs-site/manage.include.files", False) +if include_file_path and manage_include_files: + slave_hosts = default("/clusterHostInfo/slave_hosts", []) + hdfs_include_file = list(set(slave_hosts) - set(hdfs_exclude_file)) +update_files_only = default("/commandParams/update_files_only",False) # HDFS High Availability properties dfs_ha_enabled = False dfs_ha_nameservices = default("/configurations/hdfs-site/dfs.internal.nameservices", None) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/templates/include_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/templates/include_hosts_list.j2 new file mode 100644 index 0000000..115a8a4 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/templates/include_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in hdfs_include_file %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml index d0d0ede..a65b801 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml @@ -408,6 +408,12 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>manage.include.files</name> + <value>false</value> + <description>If true Ambari will manage include file if yarn.resourcemanager.nodes.include-path is configured.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>yarn.http.policy</name> <value>HTTP_ONLY</value> <description> http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index f474a89..67931c6 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -241,11 +241,17 @@ user_group = config['configurations']['cluster-env']['user_group'] exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") +nm_hosts = default("/clusterHostInfo/nm_hosts", []) +#incude file +include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None) +include_hosts = None +manage_include_files = default("/configurations/yarn-site/manage.include.files", False) +if include_file_path and manage_include_files: + include_hosts = list(set(nm_hosts) - set(exclude_hosts)) + ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", [])) has_ats = not len(ats_host) == 0 -nm_hosts = default("/clusterHostInfo/nm_hosts", []) - # don't using len(nm_hosts) here, because check can take too much time on large clusters number_of_nm = 1 @@ -345,7 +351,7 @@ HdfsResource = functools.partial( immutable_paths = get_not_managed_resources(), dfs_type = dfs_type ) -update_exclude_file_only = default("/commandParams/update_exclude_file_only",False) +update_files_only = default("/commandParams/update_files_only",False) mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_windows.py index 52918d2e..c2a02d7 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_windows.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_windows.py @@ -59,4 +59,12 @@ hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar" exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") -update_exclude_file_only = config['commandParams']['update_exclude_file_only'] + +nm_hosts = default("/clusterHostInfo/nm_hosts", []) +#incude file +include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None) +include_hosts = None +manage_include_files = default("/configurations/yarn-site/manage.include.files", False) +if include_file_path and manage_include_files: + include_hosts = list(set(nm_hosts) - set(exclude_hosts)) +update_files_only = default("/commandParams/update_files_only", False) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py index 7d024b1..b929af0 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py @@ -91,7 +91,14 @@ class ResourcemanagerWindows(Resourcemanager): mode="f" ) - if params.update_exclude_file_only == False: + if params.include_hosts: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=yarn_user, + mode="f" + ) + + if params.update_files_only == False: Execute(yarn_refresh_cmd, user=yarn_user) @@ -159,7 +166,14 @@ class ResourcemanagerDefault(Resourcemanager): group=user_group ) - if params.update_exclude_file_only == False: + if params.include_hosts: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=yarn_user, + group=user_group + ) + + if params.update_files_only == False: Execute(yarn_refresh_cmd, environment= {'PATH' : params.execute_path }, user=yarn_user) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/include_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/include_hosts_list.j2 new file mode 100644 index 0000000..42e33c0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/include_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in include_hosts %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml index 64e0bcb..2a69d35 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml @@ -402,6 +402,12 @@ <on-ambari-upgrade add="false"/> </property> <property> + <name>manage.include.files</name> + <value>false</value> + <description>If true Ambari will manage include file if yarn.resourcemanager.nodes.include-path is configured.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>yarn.http.policy</name> <value>HTTP_ONLY</value> <description> http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py index a05d259..68d17f0 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py @@ -239,10 +239,17 @@ user_group = config['configurations']['cluster-env']['user_group'] exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") +nm_hosts = default("/clusterHostInfo/nm_hosts", []) +#incude file +include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None) +include_hosts = None +manage_include_files = default("/configurations/yarn-site/manage.include.files", False) +if include_file_path and manage_include_files: + include_hosts = list(set(nm_hosts) - set(exclude_hosts)) + ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", [])) has_ats = not len(ats_host) == 0 -nm_hosts = default("/clusterHostInfo/nm_hosts", []) # don't using len(nm_hosts) here, because check can take too much time on large clusters number_of_nm = 1 @@ -341,7 +348,7 @@ HdfsResource = functools.partial( immutable_paths = get_not_managed_resources(), dfs_type = dfs_type ) -update_exclude_file_only = default("/commandParams/update_exclude_file_only",False) +update_files_only = default("/commandParams/update_files_only", False) mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_windows.py index 52918d2e..c2a02d7 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_windows.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_windows.py @@ -59,4 +59,12 @@ hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar" exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") -update_exclude_file_only = config['commandParams']['update_exclude_file_only'] + +nm_hosts = default("/clusterHostInfo/nm_hosts", []) +#incude file +include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None) +include_hosts = None +manage_include_files = default("/configurations/yarn-site/manage.include.files", False) +if include_file_path and manage_include_files: + include_hosts = list(set(nm_hosts) - set(exclude_hosts)) +update_files_only = default("/commandParams/update_files_only", False) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py index f6d6315..961fe63 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py @@ -91,7 +91,14 @@ class ResourcemanagerWindows(Resourcemanager): mode="f" ) - if params.update_exclude_file_only == False: + if params.include_hosts: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=yarn_user, + mode="f" + ) + + if params.update_files_only == False: Execute(yarn_refresh_cmd, user=yarn_user) @@ -174,7 +181,14 @@ class ResourcemanagerDefault(Resourcemanager): group=user_group ) - if params.update_exclude_file_only == False: + if params.include_hosts: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=yarn_user, + mode="f" + ) + + if params.update_files_only == False: Execute(yarn_refresh_cmd, environment= {'PATH' : params.execute_path }, user=yarn_user) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/templates/include_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/templates/include_hosts_list.j2 new file mode 100644 index 0000000..42e33c0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/templates/include_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in include_hosts %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/configuration/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/configuration/hdfs-site.xml b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/configuration/hdfs-site.xml index 87684df..66d25cf 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/configuration/hdfs-site.xml +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/configuration/hdfs-site.xml @@ -72,6 +72,12 @@ excluded.</description> <on-ambari-upgrade add="true"/> </property> + <property> + <name>manage.include.files</name> + <value>false</value> + <description>If true Ambari will manage include file if dfs.hosts is configured.</description> + <on-ambari-upgrade add="false"/> + </property> <!-- <property> <name>dfs.hosts</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/hdfs_namenode.py index 6de7735..19751f6 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/hdfs_namenode.py @@ -143,8 +143,16 @@ def decommission(): owner=hdfs_user, group=user_group ) - - if not params.update_exclude_file_only: + + if params.hdfs_include_file: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=params.hdfs_user, + group=params.user_group + ) + pass + + if not params.update_files_only: Execute(nn_kinit_cmd, user=hdfs_user ) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py index 19e223c..9cf163a 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py @@ -55,13 +55,18 @@ falcon_user = config['configurations']['falcon-env']['falcon_user'] #exclude file hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", []) exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude'] -update_exclude_file_only = default("/commandParams/update_exclude_file_only",False) +slave_hosts = default("/clusterHostInfo/slave_hosts", []) +include_file_path = default("/configurations/hdfs-site/dfs.hosts", None) +hdfs_include_file = None +manage_include_files = default("/configurations/hdfs-site/manage.include.files", False) +if include_file_path and manage_include_files: + hdfs_include_file = list(set(slave_hosts) - set(hdfs_exclude_file)) +update_files_only = default("/commandParams/update_files_only",False) kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) #hosts hostname = config["hostname"] rm_host = default("/clusterHostInfo/rm_host", []) -slave_hosts = default("/clusterHostInfo/slave_hosts", []) oozie_servers = default("/clusterHostInfo/oozie_server", []) hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", []) hive_server_host = default("/clusterHostInfo/hive_server_host", []) @@ -241,4 +246,4 @@ ttnode_heapsize = "1024m" dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize'] mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce") mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce") -script_https_protocol = Script.get_force_https_protocol_name() \ No newline at end of file +script_https_protocol = Script.get_force_https_protocol_name() http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/templates/include_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/templates/include_hosts_list.j2 new file mode 100644 index 0000000..115a8a4 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/templates/include_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in hdfs_include_file %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/configuration/yarn-site.xml index 8e9b8b1..3cb5add 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/configuration/yarn-site.xml @@ -334,6 +334,12 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>manage.include.files</name> + <value>false</value> + <description>If true Ambari will manage include file if yarn.resourcemanager.nodes.include-path is configured.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>yarn.timeline-service.enabled</name> <value>true</value> <description>Indicate to clients whether timeline service is enabled or not. http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/params.py index 33496cfe..87b5992 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/params.py @@ -118,6 +118,14 @@ user_group = config['configurations']['cluster-env']['user_group'] exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") +nm_hosts = default("/clusterHostInfo/nm_hosts", []) +#incude file +include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None) +include_hosts = None +manage_include_files = default("/configurations/yarn-site/manage.include.files", False) +if include_file_path and manage_include_files: + include_hosts = list(set(nm_hosts) - set(exclude_hosts)) + hostname = config['hostname'] ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", [])) @@ -162,7 +170,7 @@ HdfsDirectory = functools.partial( kinit_path_local = kinit_path_local, bin_dir = hadoop_bin_dir ) -update_exclude_file_only = default("/commandParams/update_exclude_file_only",False) +update_files_only = default("/commandParams/update_files_only",False) mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/resourcemanager.py index 4d40d68..8bd76bf 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/resourcemanager.py @@ -89,7 +89,14 @@ class Resourcemanager(Script): group=user_group ) - if params.update_exclude_file_only == False: + if params.include_hosts: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=yarn_user, + group=user_group + ) + + if params.update_files_only == False: Execute(yarn_refresh_cmd, environment= {'PATH' : params.execute_path }, user=yarn_user) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/templates/include_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/templates/include_hosts_list.j2 new file mode 100644 index 0000000..42e33c0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/templates/include_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in include_hosts %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/configuration/yarn-site.xml index 93a35cd..f2da835 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/configuration/yarn-site.xml @@ -334,6 +334,12 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>manage.include.files</name> + <value>false</value> + <description>If true Ambari will manage include file if yarn.resourcemanager.nodes.include-path is configured.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>yarn.timeline-service.enabled</name> <value>true</value> <description>Indicate to clients whether timeline service is enabled or not. http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/params.py index 5a7e508..16db0e4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/params.py @@ -98,6 +98,13 @@ yarn_container_bin = "/usr/lib/hadoop-yarn/bin" #exclude file exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) exclude_file_path = config['configurations']['yarn-site']['yarn.resourcemanager.nodes.exclude-path'] +nm_hosts = default("/clusterHostInfo/nm_hosts", []) +#incude file +include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None) +include_hosts = None +manage_include_files = default("/configurations/yarn-site/manage.include.files", False) +if include_file_path and manage_include_files: + include_hosts = list(set(nm_hosts) - set(exclude_hosts)) hostname = config['hostname'] @@ -128,7 +135,7 @@ HdfsDirectory = functools.partial( keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local ) -update_exclude_file_only = config['commandParams']['update_exclude_file_only'] +update_files_only = default("/commandParams/update_files_only", False) hadoop_bin = "/usr/lib/hadoop/sbin" mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/resourcemanager.py index a286ae3..f92938b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/resourcemanager.py @@ -79,7 +79,14 @@ class Resourcemanager(Script): group=user_group ) - if params.update_exclude_file_only == False: + if params.include_hosts: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=yarn_user, + group=user_group + ) + + if params.update_files_only == False: Execute(yarn_refresh_cmd, user=yarn_user) pass http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2 new file mode 100644 index 0000000..c7ce416 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in exclude_hosts %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 new file mode 100644 index 0000000..42e33c0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in include_hosts %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/configuration/yarn-site.xml index 93a35cd..f2da835 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/configuration/yarn-site.xml @@ -334,6 +334,12 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>manage.include.files</name> + <value>false</value> + <description>If true Ambari will manage include file if yarn.resourcemanager.nodes.include-path is configured.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>yarn.timeline-service.enabled</name> <value>true</value> <description>Indicate to clients whether timeline service is enabled or not. http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py index 5a7e508..bd188e9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py @@ -98,7 +98,13 @@ yarn_container_bin = "/usr/lib/hadoop-yarn/bin" #exclude file exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) exclude_file_path = config['configurations']['yarn-site']['yarn.resourcemanager.nodes.exclude-path'] - +nm_hosts = default("/clusterHostInfo/nm_hosts", []) +#incude file +include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None) +include_hosts = None +manage_include_files = default("/configurations/yarn-site/manage.include.files", False) +if include_file_path and manage_include_files: + include_hosts = list(set(nm_hosts) - set(exclude_hosts)) hostname = config['hostname'] if security_enabled: @@ -128,7 +134,7 @@ HdfsDirectory = functools.partial( keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local ) -update_exclude_file_only = config['commandParams']['update_exclude_file_only'] +update_files_only = default("/commandParams/update_files_only", False) hadoop_bin = "/usr/lib/hadoop/sbin" mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py index a286ae3..f92938b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py @@ -79,7 +79,14 @@ class Resourcemanager(Script): group=user_group ) - if params.update_exclude_file_only == False: + if params.include_hosts: + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=yarn_user, + group=user_group + ) + + if params.update_files_only == False: Execute(yarn_refresh_cmd, user=yarn_user) pass http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 new file mode 100644 index 0000000..42e33c0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in include_hosts %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index b3a12f2..fdfca0f 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -6028,8 +6028,8 @@ public class AmbariManagementControllerTest { execCmd = storedTasks.get(0).getExecutionCommandWrapper ().getExecutionCommand(); Map<String, String> cmdParams = execCmd.getCommandParams(); - Assert.assertTrue(cmdParams.containsKey("update_exclude_file_only")); - Assert.assertTrue(cmdParams.get("update_exclude_file_only").equals("false")); + Assert.assertTrue(cmdParams.containsKey("update_files_only")); + Assert.assertTrue(cmdParams.get("update_files_only").equals("false")); Assert.assertNotNull(storedTasks); Assert.assertEquals(1, storedTasks.size()); Assert.assertEquals(HostComponentAdminState.DECOMMISSIONED, scHost.getComponentAdminState()); @@ -6084,8 +6084,8 @@ public class AmbariManagementControllerTest { Assert.assertTrue(hrc.getCommandDetail().contains(host1)); Assert.assertTrue(hrc.getCommandDetail().contains(host2)); cmdParams = hrc.getExecutionCommandWrapper().getExecutionCommand().getCommandParams(); - if(!cmdParams.containsKey("update_exclude_file_only") - || !cmdParams.get("update_exclude_file_only").equals("true")) { + if(!cmdParams.containsKey("update_files_only") + || !cmdParams.get("update_files_only").equals("true")) { countRefresh++; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py index a7233c7..f0f2587 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py @@ -1041,7 +1041,7 @@ class TestNamenode(RMFTestCase): bin_dir = '/usr/bin') self.assertNoMoreResources() - def test_decommission_update_exclude_file_only(self): + def test_decommission_update_files_only(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py", classname = "NameNode", command = "decommission", http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/altfs_plus_hdfs.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/altfs_plus_hdfs.json b/ambari-server/src/test/python/stacks/2.0.6/configs/altfs_plus_hdfs.json index ea00a37..f928073 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/altfs_plus_hdfs.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/altfs_plus_hdfs.json @@ -33,7 +33,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json index d267bc1..a16ec26 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json @@ -60,7 +60,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json index 71423c8..f3e8dc3 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json @@ -32,7 +32,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha.json index 009ff6d..7b0f78d 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha.json @@ -35,7 +35,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha_2.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha_2.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha_2.json index 2b078c3..01f0efc 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha_2.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha_2.json @@ -35,7 +35,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_non_hdfs.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_non_hdfs.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_non_hdfs.json index 571b737..0cbd322 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_non_hdfs.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_non_hdfs.json @@ -33,7 +33,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/default_no_install.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_no_install.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_no_install.json index 7fdb449..cfcf5e1 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_no_install.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_no_install.json @@ -35,7 +35,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/default_oozie_mysql.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_oozie_mysql.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_oozie_mysql.json index 7378b68..7db73ab 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_oozie_mysql.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_oozie_mysql.json @@ -36,7 +36,7 @@ "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", "mark_draining_only" : "false", - "update_exclude_file_only" : "false", + "update_files_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}],
