SLIDER-82 preparing ground for anti-affinity. * app state binding moves from multiple args to a single binding struct; this significantly simplifies test setup * references in roleHistory to available nodes are replace with "recent", as that is what they currently are
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f2f2c766 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f2f2c766 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f2f2c766 Branch: refs/heads/feature/SLIDER-82-pass-3.1 Commit: f2f2c766b887482ca4a6940e9c11e21f58660d9a Parents: a66f7db Author: Steve Loughran <[email protected]> Authored: Wed Nov 4 16:35:03 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Thu Nov 5 13:19:03 2015 +0000 ---------------------------------------------------------------------- .../slider/client/SliderYarnClientImpl.java | 9 +- .../slider/providers/PlacementPolicy.java | 7 +- .../server/appmaster/SliderAppMaster.java | 93 ++++++++++---------- .../slider/server/appmaster/state/AppState.java | 81 +++++++---------- .../appmaster/state/AppStateBindingInfo.java | 59 +++++++++++++ .../server/appmaster/state/RoleHistory.java | 79 +++++++++-------- .../TestMockAppStateDynamicHistory.groovy | 28 +----- .../TestMockAppStateDynamicRoles.groovy | 33 +++---- .../TestMockAppStateFlexDynamicRoles.groovy | 44 ++++----- .../TestMockAppStateRebuildOnAMRestart.groovy | 32 +++---- .../TestMockContainerResourceAllocations.groovy | 5 +- .../TestRoleHistoryContainerEvents.groovy | 2 +- ...stRoleHistoryFindNodesForNewInstances.groovy | 2 +- .../model/history/TestRoleHistoryRW.groovy | 4 +- .../TestRoleHistoryRequestTracking.groovy | 4 +- .../model/mock/BaseMockAppStateTest.groovy | 41 +++++---- .../appmaster/model/mock/MockAppState.groovy | 10 ++- .../web/rest/agent/TestAMAgentWebServices.java | 47 ++++------ .../management/TestAMManagementWebServices.java | 71 +++++---------- 19 files changed, 302 insertions(+), 349 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java index 3b7a65c..42759fd 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java @@ -55,8 +55,7 @@ import java.util.Set; * from the slider entry point service */ public class SliderYarnClientImpl extends YarnClientImpl { - protected static final Logger - log = LoggerFactory.getLogger(SliderYarnClientImpl.class); + protected static final Logger log = LoggerFactory.getLogger(SliderYarnClientImpl.class); /** * Keyword to use in the {@link #emergencyForceKill(String)} @@ -96,10 +95,10 @@ public class SliderYarnClientImpl extends YarnClientImpl { public List<ApplicationReport> listDeployedInstances(String user) throws YarnException, IOException { Preconditions.checkArgument(user != null, "Null User"); - Set<String> types = new HashSet<String>(1); + Set<String> types = new HashSet<>(1); types.add(SliderKeys.APP_TYPE); List<ApplicationReport> allApps = getApplications(types); - List<ApplicationReport> results = new ArrayList<ApplicationReport>(); + List<ApplicationReport> results = new ArrayList<>(); for (ApplicationReport report : allApps) { if (StringUtils.isEmpty(user) || user.equals(report.getUser())) { results.add(report); @@ -330,8 +329,6 @@ public class SliderYarnClientImpl extends YarnClientImpl { Preconditions.checkArgument(StringUtils.isNotEmpty(appname), "Null/empty application name"); Preconditions.checkArgument(desiredState != null, "Null desiredState"); - ApplicationReport found = null; - ApplicationReport foundAndLive = null; log.debug("Searching {} records for instance name {} in state '{}'", instances.size(), appname, desiredState); for (ApplicationReport app : instances) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java index 4e85a93..e0913a5 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java +++ b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java @@ -27,7 +27,12 @@ public class PlacementPolicy { /** * Default value: history used, anti-affinity hinted at on rebuild/flex up */ - public static final int DEFAULT = 0; + public static final int NONE = 0; + + /** + * Default value: history used, anti-affinity hinted at on rebuild/flex up + */ + public static final int DEFAULT = NONE; /** * Strict placement: when asking for an instance for which there is http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index df91d7f..b552290 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -59,13 +59,16 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; import org.apache.hadoop.yarn.client.api.async.NMClientAsync; import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.*; import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.registry.client.binding.RegistryPathUtils; @@ -84,6 +87,7 @@ import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.RoleKeys; import org.apache.slider.api.StatusKeys; import org.apache.slider.api.proto.SliderClusterAPI; +import org.apache.slider.client.SliderYarnClientImpl; import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.params.AbstractActionArgs; @@ -143,6 +147,7 @@ import org.apache.slider.server.appmaster.operations.AbstractRMOperation; import org.apache.slider.server.appmaster.rpc.SliderIPCService; import org.apache.slider.server.appmaster.security.SecurityConfiguration; import org.apache.slider.server.appmaster.state.AppState; +import org.apache.slider.server.appmaster.state.AppStateBindingInfo; import org.apache.slider.server.appmaster.state.ContainerAssignment; import org.apache.slider.server.appmaster.state.ProviderAppState; import org.apache.slider.server.appmaster.operations.RMOperationHandler; @@ -356,16 +361,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService private RegistryOperations registryOperations; /** - * Record of the max no. of cores allowed in this cluster - */ - private int containerMaxCores; - - /** - * limit container memory - */ - private int containerMaxMemory; - - /** * The stop request received...the exit details are extracted * from this */ @@ -415,6 +410,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService */ private boolean securityEnabled; private ContentCache contentCache; + private SliderYarnClientImpl yarnClient; + + /** + * resource limits + */ + private Resource maximumResourceCapability; /** * Service Constructor @@ -437,10 +438,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService // Load in the server configuration - if it is actually on the Classpath URL serverXmlUrl = ConfigHelper.getResourceUrl(SLIDER_SERVER_XML); if (serverXmlUrl != null) { - log.info("Loading {} at {}", SLIDER_SERVER_XML, serverXmlUrl); - Configuration serverConf = - ConfigHelper.loadFromResource(SLIDER_SERVER_XML); + Configuration serverConf = ConfigHelper.loadFromResource(SLIDER_SERVER_XML); ConfigHelper.mergeConfigurations(customConf, serverConf, SLIDER_SERVER_XML, true); } @@ -505,7 +504,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService addService(executorService); addService(actionQueues); - + addService(yarnClient = new SliderYarnClientImpl()); + //init all child services super.serviceInit(conf); } @@ -630,8 +630,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService Configuration serviceConf = getConfig(); - securityConfiguration = new SecurityConfiguration( - serviceConf, instanceDefinition, clustername); + securityConfiguration = new SecurityConfiguration(serviceConf, instanceDefinition, clustername); // obtain security state securityEnabled = securityConfiguration.isSecurityEnabled(); // set the global security flag for the instance definition @@ -659,8 +658,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService providerService = factory.createServerProvider(); // init the provider BUT DO NOT START IT YET initAndAddService(providerService); - providerRMOperationHandler = - new ProviderNotifyingOperationHandler(providerService); + providerRMOperationHandler = new ProviderNotifyingOperationHandler(providerService); // create a slider AM provider sliderAMProvider = new SliderAMProviderService(); @@ -682,12 +680,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService ApplicationId appid = appAttemptID.getApplicationId(); log.info("AM for ID {}", appid.getId()); - appInformation.put(StatusKeys.INFO_AM_CONTAINER_ID, - appMasterContainerID.toString()); - appInformation.put(StatusKeys.INFO_AM_APP_ID, - appid.toString()); - appInformation.put(StatusKeys.INFO_AM_ATTEMPT_ID, - appAttemptID.toString()); + appInformation.put(StatusKeys.INFO_AM_CONTAINER_ID, appMasterContainerID.toString()); + appInformation.put(StatusKeys.INFO_AM_APP_ID, appid.toString()); + appInformation.put(StatusKeys.INFO_AM_ATTEMPT_ID, appAttemptID.toString()); Map<String, String> envVars; List<Container> liveContainers; @@ -715,7 +710,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService if (securityEnabled) { // fix up the ACLs if they are not set - String acls = getConfig().get(KEY_PROTOCOL_ACL); + String acls = serviceConf.get(KEY_PROTOCOL_ACL); if (acls == null) { getConfig().set(KEY_PROTOCOL_ACL, "*"); } @@ -731,8 +726,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService appMasterHostname = rpcServiceAddress.getHostName(); appMasterRpcPort = rpcServiceAddress.getPort(); appMasterTrackingUrl = null; - log.info("AM Server is listening at {}:{}", appMasterHostname, - appMasterRpcPort); + log.info("AM Server is listening at {}:{}", appMasterHostname, appMasterRpcPort); appInformation.put(StatusKeys.INFO_AM_HOSTNAME, appMasterHostname); appInformation.set(StatusKeys.INFO_AM_RPC_PORT, appMasterRpcPort); @@ -776,24 +770,27 @@ public class SliderAppMaster extends AbstractSliderLaunchedService // ***************************************************** log.info("Connecting to RM at {},address tracking URL={}", appMasterRpcPort, appMasterTrackingUrl); - amRegistrationData = asyncRMClient - .registerApplicationMaster(appMasterHostname, + amRegistrationData = asyncRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); - Resource maxResources = - amRegistrationData.getMaximumResourceCapability(); - containerMaxMemory = maxResources.getMemory(); - containerMaxCores = maxResources.getVirtualCores(); - appState.setContainerLimits(maxResources.getMemory(), - maxResources.getVirtualCores()); + maximumResourceCapability = amRegistrationData.getMaximumResourceCapability(); + + int minMemory = serviceConf.getInt(RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); + // validate scheduler vcores allocation setting + int minCores = serviceConf.getInt(RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + int maxMemory = maximumResourceCapability.getMemory(); + int maxCores = maximumResourceCapability.getVirtualCores(); + appState.setContainerLimits(minMemory,maxMemory, minCores, maxCores ); // build the handler for RM request/release operations; this uses // the max value as part of its lookup - rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient, maxResources); + rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient, maximumResourceCapability); // set the RM-defined maximum cluster values - appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(containerMaxCores)); - appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(containerMaxMemory)); + appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(maxCores)); + appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(maxMemory)); processAMCredentials(securityConfiguration); @@ -837,15 +834,17 @@ public class SliderAppMaster extends AbstractSliderLaunchedService Path historyDir = new Path(clusterDirPath, HISTORY_DIR_NAME); //build the instance - appState.buildInstance(instanceDefinition, - serviceConf, - providerConf, - providerRoles, - fs.getFileSystem(), - historyDir, - liveContainers, - appInformation, - providerService.createContainerReleaseSelector()); + AppStateBindingInfo binding = new AppStateBindingInfo(); + binding.instanceDefinition = instanceDefinition; + binding.serviceConfig = serviceConf; + binding.publishedProviderConf = providerConf; + binding.roles = providerRoles; + binding.fs = fs.getFileSystem(); + binding.historyPath = historyDir; + binding.liveContainers = liveContainers; + binding.applicationInfo = appInformation; + binding.releaseSelector = providerService.createContainerReleaseSelector(); + appState.buildInstance(binding); providerService.rebuildContainerDetails(liveContainers, instanceDefinition.getName(), appState.getRolePriorityMap()); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index eadb1dc..1325148 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -21,7 +21,6 @@ package org.apache.slider.server.appmaster.state; import com.codahale.metrics.Counter; import com.codahale.metrics.MetricRegistry; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -265,15 +264,17 @@ public class AppState { /** - * Record of the max no. of cores allowed in this cluster + * limits of container core numbers in this queue */ private int containerMaxCores; + private int containerMinCores; /** - * limit container memory + * limits of container memory in this queue */ private int containerMaxMemory; - + private int containerMinMemory; + private RoleHistory roleHistory; private Configuration publishedProviderConf; private long startTimeThreshold; @@ -447,27 +448,29 @@ public class AppState { } /** - * Set the container limits -the max that can be asked for, - * which are used when the "max" values are requested + * Set the container limits -the min and max values for + * resource requests. All requests must be multiples of the min + * values. + * @param minMemory min memory MB * @param maxMemory maximum memory + * @param minCores min v core count * @param maxCores maximum cores */ - public void setContainerLimits(int maxMemory, int maxCores) { + public void setContainerLimits(int minMemory,int maxMemory, int minCores, int maxCores) { + containerMinCores = minCores; containerMaxCores = maxCores; + containerMinMemory = minMemory; containerMaxMemory = maxMemory; } - public ConfTreeOperations getResourcesSnapshot() { return resourcesSnapshot; } - public ConfTreeOperations getAppConfSnapshot() { return appConfSnapshot; } - public ConfTreeOperations getInternalsSnapshot() { return internalsSnapshot; } @@ -488,38 +491,17 @@ public class AppState { return unresolvedInstanceDefinition; } - /** - * Build up the application state - * @param instanceDefinition definition of the applicatin instance - * @param appmasterConfig - * @param publishedProviderConf any configuration info to be published by a provider - * @param providerRoles roles offered by a provider - * @param fs filesystem - * @param historyDir directory containing history files - * @param liveContainers list of live containers supplied on an AM restart - * @param applicationInfo app info to retain for web views - * @param releaseSelector selector of containers to release - */ - public synchronized void buildInstance(AggregateConf instanceDefinition, - Configuration appmasterConfig, - Configuration publishedProviderConf, - List<ProviderRole> providerRoles, - FileSystem fs, - Path historyDir, - List<Container> liveContainers, - Map<String, String> applicationInfo, - ContainerReleaseSelector releaseSelector) - throws BadClusterStateException, BadConfigException, IOException { - Preconditions.checkArgument(instanceDefinition != null); - Preconditions.checkArgument(releaseSelector != null); + public synchronized void buildInstance(AppStateBindingInfo binding) + throws BadClusterStateException, BadConfigException, IOException { + binding.validate(); log.debug("Building application state"); - this.publishedProviderConf = publishedProviderConf; - this.applicationInfo = applicationInfo != null ? applicationInfo - : new HashMap<String, String>(); + publishedProviderConf = binding.publishedProviderConf; + applicationInfo = binding.applicationInfo != null ? binding.applicationInfo + : new HashMap<String, String>(); clientProperties = new HashMap<>(); - containerReleaseSelector = releaseSelector; + containerReleaseSelector = binding.releaseSelector; Set<String> confKeys = ConfigHelper.sortedConfigKeys(publishedProviderConf); @@ -532,15 +514,15 @@ public class AppState { // set the cluster specification (once its dependency the client properties // is out the way - setInitialInstanceDefinition(instanceDefinition); + setInitialInstanceDefinition(binding.instanceDefinition); //build the initial role list - for (ProviderRole providerRole : providerRoles) { + List<ProviderRole> roleList = new ArrayList<>(binding.roles); + for (ProviderRole providerRole : roleList) { buildRole(providerRole); } - ConfTreeOperations resources = - instanceDefinition.getResourceOperations(); + ConfTreeOperations resources = instanceDefinition.getResourceOperations(); Set<String> roleNames = resources.getComponentNames(); for (String name : roleNames) { @@ -551,16 +533,14 @@ public class AppState { ProviderRole dynamicRole = createDynamicProviderRole(name, resComponent); buildRole(dynamicRole); - providerRoles.add(dynamicRole); + roleList.add(dynamicRole); } } //then pick up the requirements buildRoleRequirementsFromResources(); - //set the livespan - MapOperations globalResOpts = - instanceDefinition.getResourceOperations().getGlobalOptions(); + MapOperations globalResOpts = instanceDefinition.getResourceOperations().getGlobalOptions(); startTimeThreshold = globalResOpts.getOptionInt( InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE, @@ -576,16 +556,15 @@ public class AppState { // set up the role history - roleHistory = new RoleHistory(providerRoles); + roleHistory = new RoleHistory(roleList); roleHistory.register(metricsAndMonitoring); - roleHistory.onStart(fs, historyDir); + roleHistory.onStart(binding.fs, binding.historyPath); //rebuild any live containers - rebuildModelFromRestart(liveContainers); + rebuildModelFromRestart(binding.liveContainers); // any am config options to pick up - logServerURL = appmasterConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL, ""); - + logServerURL = binding.serviceConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL, ""); //mark as live applicationLive = true; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java new file mode 100644 index 0000000..184c8aa --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.state; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.providers.ProviderRole; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Binding information for application states; designed to be extensible + * so that tests don't have to be massivley reworked when new arguments + * are added. + */ +public class AppStateBindingInfo { + public AggregateConf instanceDefinition; + public Configuration serviceConfig = new Configuration(); + public Configuration publishedProviderConf = new Configuration(false); + public List<ProviderRole> roles = new ArrayList<>(); + public FileSystem fs; + public Path historyPath; + public List<Container> liveContainers = new ArrayList<>(0); + public Map<String, String> applicationInfo = new HashMap<>(); + public ContainerReleaseSelector releaseSelector = new SimpleReleaseSelector(); + + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument(instanceDefinition != null, "null instanceDefinition"); + Preconditions.checkArgument(serviceConfig != null, "null appmasterConfig"); + Preconditions.checkArgument(publishedProviderConf != null, "null publishedProviderConf"); + Preconditions.checkArgument(releaseSelector != null, "null releaseSelector"); + Preconditions.checkArgument(roles != null, "null providerRoles"); + Preconditions.checkArgument(fs != null, "null fs"); + Preconditions.checkArgument(historyPath != null, "null historyDir"); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java index df3983a..d9a6b34 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java @@ -31,7 +31,6 @@ import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.providers.ProviderRole; import org.apache.slider.server.appmaster.management.BoolMetric; -import org.apache.slider.server.appmaster.management.LongGauge; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.management.Timestamp; import org.apache.slider.server.appmaster.operations.AbstractRMOperation; @@ -52,6 +51,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; /** * The Role History. @@ -87,8 +87,8 @@ public class RoleHistory { private RoleHistoryWriter historyWriter = new RoleHistoryWriter(); /** - * When were the nodes updated in a {@link #onNodesUpdated(List)} call. - * If zero: never + * When were the nodes updated in a {@link #onNodesUpdated(List)} call? + * If zero: never. */ private final Timestamp nodesUpdatedTime = new Timestamp(0); private final BoolMetric nodeUpdateReceived = new BoolMetric(false); @@ -98,9 +98,10 @@ public class RoleHistory { /** * For each role, lists nodes that are available for data-local allocation, - ordered by more recently released - To accelerate node selection + * ordered by more recently released - to accelerate node selection. + * That is, they are "recently used nodes" */ - private Map<Integer, LinkedList<NodeInstance>> availableNodes; + private Map<Integer, LinkedList<NodeInstance>> recentNodes; /** * Track the failed nodes. Currently used to make wiser decision of container @@ -158,8 +159,7 @@ public class RoleHistory { throws BadConfigException { int index = providerRole.id; if (index < 0) { - throw new BadConfigException("Provider " + providerRole - + " id is out of range"); + throw new BadConfigException("Provider " + providerRole + " id is out of range"); } if (roleStats.get(index) != null) { throw new BadConfigException( @@ -206,7 +206,7 @@ public class RoleHistory { * Clear the lists of available nodes */ private synchronized void resetAvailableNodeLists() { - availableNodes = new HashMap<>(roleSize); + recentNodes = new ConcurrentHashMap<>(roleSize); } /** @@ -363,7 +363,7 @@ public class RoleHistory { public synchronized void insert(Collection<NodeInstance> nodes) { nodemap.insert(nodes); } - + /** * Get current time. overrideable for test subclasses * @return current time in millis @@ -435,8 +435,7 @@ public class RoleHistory { * @param historyDir path in FS for history * @return true if the history was thawed */ - public boolean onStart(FileSystem fs, Path historyDir) throws - BadConfigException { + public boolean onStart(FileSystem fs, Path historyDir) throws BadConfigException { assert filesystem == null; filesystem = fs; historyPath = historyDir; @@ -483,7 +482,7 @@ public class RoleHistory { } //start is then completed - buildAvailableNodeLists(); + buildRecentNodeLists(); } else { //fallback to bootstrap procedure onBootstrap(); @@ -496,7 +495,7 @@ public class RoleHistory { * (After the start), rebuild the availability data structures */ @VisibleForTesting - public synchronized void buildAvailableNodeLists() { + public synchronized void buildRecentNodeLists() { resetAvailableNodeLists(); // build the list of available nodes for (Map.Entry<String, NodeInstance> entry : nodemap.entrySet()) { @@ -505,13 +504,13 @@ public class RoleHistory { NodeEntry nodeEntry = ni.get(i); if (nodeEntry != null && nodeEntry.isAvailable()) { log.debug("Adding {} for role {}", ni, i); - getOrCreateNodesForRoleId(i).add(ni); + listRecentNodesForRoleId(i).add(ni); } } } // sort the resulting arrays for (int i = 0; i < roleSize; i++) { - sortAvailableNodeList(i); + sortRecentNodeList(i); } } @@ -521,30 +520,35 @@ public class RoleHistory { * @return potentially null list */ @VisibleForTesting - public List<NodeInstance> getNodesForRoleId(int id) { - return availableNodes.get(id); + public List<NodeInstance> getRecentNodesForRoleId(int id) { + return recentNodes.get(id); } - + /** - * Get the nodes for an ID -may be null + * Get a possibly emtpy list of suggested nodes for a role. * @param id role ID * @return list */ - private LinkedList<NodeInstance> getOrCreateNodesForRoleId(int id) { - LinkedList<NodeInstance> instances = availableNodes.get(id); + private LinkedList<NodeInstance> listRecentNodesForRoleId(int id) { + LinkedList<NodeInstance> instances = recentNodes.get(id); if (instances == null) { - instances = new LinkedList<>(); - availableNodes.put(id, instances); + synchronized (this) { + // recheck in the synchronized block and recreate + if (recentNodes.get(id) == null) { + recentNodes.put(id, new LinkedList<NodeInstance>()); + } + instances = recentNodes.get(id); + } } return instances; } - + /** - * Sort an available node list + * Sort a the recent node list for a single role * @param role role to sort */ - private void sortAvailableNodeList(int role) { - List<NodeInstance> nodesForRoleId = getNodesForRoleId(role); + private void sortRecentNodeList(int role) { + List<NodeInstance> nodesForRoleId = getRecentNodesForRoleId(role); if (nodesForRoleId != null) { Collections.sort(nodesForRoleId, new NodeInstance.Preferred(role)); } @@ -566,7 +570,7 @@ public class RoleHistory { NodeInstance nodeInstance = null; // Get the list of possible targets. // This is a live list: changes here are preserved - List<NodeInstance> targets = getNodesForRoleId(roleId); + List<NodeInstance> targets = getRecentNodesForRoleId(roleId); if (targets == null) { // nothing to allocate on return null; @@ -655,7 +659,7 @@ public class RoleHistory { public synchronized List<NodeInstance> listActiveNodes(int role) { return nodemap.listActiveNodes(role); } - + /** * Get the node entry of a container * @param container container to look up @@ -705,7 +709,7 @@ public class RoleHistory { * @return list of containers potentially reordered */ public synchronized List<Container> prepareAllocationList(List<Container> allocatedContainers) { - + //partition into requested and unrequested List<Container> requested = new ArrayList<>(allocatedContainers.size()); @@ -717,7 +721,7 @@ public class RoleHistory { requested.addAll(unrequested); return requested; } - + /** * A container has been allocated on a node -update the data structures * @param container container @@ -730,7 +734,7 @@ public class RoleHistory { int actualCount) { int role = ContainerPriority.extractRole(container); String hostname = RoleHistoryUtils.hostnameOf(container); - List<NodeInstance> nodeInstances = getOrCreateNodesForRoleId(role); + List<NodeInstance> nodeInstances = listRecentNodesForRoleId(role); ContainerAllocation outcome = outstandingRequests.onContainerAllocated(role, hostname, container); if (desiredCount <= actualCount) { @@ -741,7 +745,7 @@ public class RoleHistory { //add the list log.info("Adding {} hosts for role {}", hosts.size(), role); nodeInstances.addAll(hosts); - sortAvailableNodeList(role); + sortRecentNodeList(role); } } return outcome; @@ -892,7 +896,7 @@ public class RoleHistory { /** * If the node is marked as available; queue it for assignments. - * Unsynced: expects caller to be in a sync block. + * Unsynced: requires caller to be in a sync block. * @param container completed container * @param nodeEntry node * @param available available flag @@ -907,7 +911,7 @@ public class RoleHistory { NodeInstance ni = getOrCreateNodeInstance(container); int roleId = ContainerPriority.extractRole(container); log.debug("Node {} is now available for role id {}", ni, roleId); - getOrCreateNodesForRoleId(roleId).addFirst(ni); + listRecentNodesForRoleId(roleId).addFirst(ni); } return available; } @@ -918,8 +922,7 @@ public class RoleHistory { public synchronized void dump() { for (ProviderRole role : providerRoles) { log.info(role.toString()); - List<NodeInstance> instances = - getOrCreateNodesForRoleId(role.id); + List<NodeInstance> instances = listRecentNodesForRoleId(role.id); log.info(" available: " + instances.size() + " " + SliderUtils.joinWithInnerSeparator(" ", instances)); } @@ -952,7 +955,7 @@ public class RoleHistory { */ @VisibleForTesting public List<NodeInstance> cloneAvailableList(int role) { - return new LinkedList<>(getOrCreateNodesForRoleId(role)); + return new LinkedList<>(listRecentNodesForRoleId(role)); } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy index aa7bb11..c62eb72 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy @@ -34,6 +34,7 @@ import org.apache.slider.server.appmaster.model.mock.MockRoles import org.apache.slider.server.appmaster.model.mock.MockYarnEngine import org.apache.slider.server.appmaster.operations.ContainerRequestOperation import org.apache.slider.server.appmaster.state.AppState +import org.apache.slider.server.appmaster.state.AppStateBindingInfo import org.apache.slider.server.appmaster.state.NodeInstance import org.apache.slider.server.appmaster.state.RoleInstance import org.apache.slider.server.appmaster.state.SimpleReleaseSelector @@ -47,11 +48,6 @@ import org.junit.Test class TestMockAppStateDynamicHistory extends BaseMockAppStateTest implements MockRoles { - @Override - String getTestName() { - return "TestMockAppStateDynamicHistory" - } - /** * Small cluster with multiple containers per node, * to guarantee many container allocations on each node @@ -62,26 +58,6 @@ class TestMockAppStateDynamicHistory extends BaseMockAppStateTest return new MockYarnEngine(8, 1) } - @Override - void initApp() { - super.initApp() - appState = new MockAppState() - appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES) - - def instance = factory.newInstanceDefinition(0,0,0) - - appState.buildInstance( - instance, - new Configuration(), - new Configuration(false), - factory.ROLES, - fs, - historyPath, - null, - null, new SimpleReleaseSelector()) - } - - @Test public void testDynamicRoleHistory() throws Throwable { @@ -199,7 +175,7 @@ class TestMockAppStateDynamicHistory extends BaseMockAppStateTest assert !entry.live - def nodesForRoleId = roleHistory.getNodesForRoleId(role_priority_8) + def nodesForRoleId = roleHistory.getRecentNodesForRoleId(role_priority_8) assert nodesForRoleId // make sure new nodes will default to a different host in the engine http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy index ee4abd6..e35f028 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy @@ -23,6 +23,7 @@ import groovy.util.logging.Slf4j import org.apache.hadoop.conf.Configuration import org.apache.hadoop.yarn.api.records.ContainerId import org.apache.slider.api.ResourceKeys +import org.apache.slider.core.conf.AggregateConf import org.apache.slider.providers.PlacementPolicy import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest import org.apache.slider.server.appmaster.model.mock.MockAppState @@ -31,6 +32,7 @@ import org.apache.slider.server.appmaster.model.mock.MockYarnEngine import org.apache.slider.server.appmaster.operations.AbstractRMOperation import org.apache.slider.server.appmaster.operations.ContainerRequestOperation import org.apache.slider.server.appmaster.state.AppState +import org.apache.slider.server.appmaster.state.AppStateBindingInfo import org.apache.slider.server.appmaster.state.ContainerPriority import org.apache.slider.server.appmaster.state.RoleHistoryUtils import org.apache.slider.server.appmaster.state.RoleInstance @@ -65,40 +67,27 @@ class TestMockAppStateDynamicRoles extends BaseMockAppStateTest } @Override - void initApp() { - super.initApp() - appState = new MockAppState() - appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES) - def instance = factory.newInstanceDefinition(0,0,0) - + AggregateConf buildInstanceDefinition() { + def instance = factory.newInstanceDefinition(0, 0, 0) def opts = [ - (ResourceKeys.COMPONENT_PRIORITY): ROLE4, + (ResourceKeys.COMPONENT_PRIORITY) : ROLE4, (ResourceKeys.COMPONENT_INSTANCES): "1", ] - instance.resourceOperations.components[ROLE4]= opts + instance.resourceOperations.components[ROLE4] = opts def opts5 = [ - (ResourceKeys.COMPONENT_PRIORITY) : ROLE5, - (ResourceKeys.COMPONENT_INSTANCES): "1", + (ResourceKeys.COMPONENT_PRIORITY) : ROLE5, + (ResourceKeys.COMPONENT_INSTANCES) : "1", (ResourceKeys.COMPONENT_PLACEMENT_POLICY): Integer.toString(PlacementPolicy.STRICT), - (ResourceKeys.NODE_FAILURE_THRESHOLD): + (ResourceKeys.NODE_FAILURE_THRESHOLD) : Integer.toString(2), ] - instance.resourceOperations.components[ROLE5]= opts5 - - appState.buildInstance( - instance, - new Configuration(), - new Configuration(false), - factory.ROLES, - fs, - historyPath, - null, - null, new SimpleReleaseSelector()) + instance.resourceOperations.components[ROLE5] = opts5 + instance } @Test http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy index 5d880b4..7bc6fe4 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy @@ -20,21 +20,22 @@ package org.apache.slider.server.appmaster.model.appstate import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.slider.api.ResourceKeys +import org.apache.slider.core.conf.AggregateConf import org.apache.slider.core.conf.ConfTreeOperations import org.apache.slider.core.exceptions.BadConfigException import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest import org.apache.slider.server.appmaster.model.mock.MockAppState import org.apache.slider.server.appmaster.model.mock.MockRoles import org.apache.slider.server.appmaster.model.mock.MockYarnEngine +import org.apache.slider.server.appmaster.state.AppStateBindingInfo import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector import org.apache.slider.server.avro.RoleHistoryWriter import org.junit.Test /** - * Test that if you have >1 role, the right roles are chosen for release. + * Test that if you have more than one role, the right roles are chosen for release. */ @CompileStatic @Slf4j @@ -57,32 +58,25 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest } @Override - void initApp() { - super.initApp() - appState = new MockAppState() - appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES) + AppStateBindingInfo buildBindingInfo() { + def bindingInfo = super.buildBindingInfo() + bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector() + bindingInfo + } + @Override + AggregateConf buildInstanceDefinition() { def instance = factory.newInstanceDefinition(0, 0, 0) def opts = [ (ResourceKeys.COMPONENT_INSTANCES): "1", - (ResourceKeys.COMPONENT_PRIORITY): "6", + (ResourceKeys.COMPONENT_PRIORITY) : "6", ] instance.resourceOperations.components["dynamic-6"] = opts - - - appState.buildInstance(instance, - new Configuration(), - new Configuration(false), - factory.ROLES, - fs, - historyPath, - null, null, - new MostRecentContainerReleaseSelector()) + instance } - private ConfTreeOperations init() { createAndStartNodes(); def resources = appState.instanceDefinition.resources; @@ -181,16 +175,10 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest def historyWorkDir2 = new File("target/history" + testName + "-0002") def historyPath2 = new Path(historyWorkDir2.toURI()) appState = new MockAppState() - appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES) - appState.buildInstance( - factory.newInstanceDefinition(0, 0, 0), - new Configuration(), - new Configuration(false), - factory.ROLES, - fs, - historyPath2, - null, null, - new MostRecentContainerReleaseSelector()) + def binding2 = buildBindingInfo() + binding2.instanceDefinition = factory.newInstanceDefinition(0, 0, 0) + binding2.historyPath = historyPath2 + appState.buildInstance(binding2) // on this read there won't be the right number of roles def loadedRoleHistory = historyWriter.read(fs, history) assert 0 == appState.roleHistory.rebuild(loadedRoleHistory) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy index c310583..02052c2 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy @@ -20,14 +20,11 @@ package org.apache.slider.server.appmaster.model.appstate import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import org.apache.hadoop.conf.Configuration import org.apache.hadoop.yarn.api.records.Container import org.apache.slider.api.StatusKeys import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest import org.apache.slider.server.appmaster.model.mock.MockAppState import org.apache.slider.server.appmaster.model.mock.MockRoles -import org.apache.slider.server.appmaster.operations.AbstractRMOperation -import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector import org.apache.slider.server.appmaster.state.NodeEntry import org.apache.slider.server.appmaster.state.NodeInstance import org.apache.slider.server.appmaster.state.NodeMap @@ -35,7 +32,7 @@ import org.apache.slider.server.appmaster.state.RoleInstance import org.junit.Test /** - * Test that if you have >1 role, the right roles are chosen for release. + * Test that app state is rebuilt on a restart */ @CompileStatic @Slf4j @@ -74,16 +71,11 @@ class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest appState = new MockAppState() //and rebuild - appState.buildInstance( - factory.newInstanceDefinition(r0, r1, r2), - new Configuration(), - new Configuration(false), - factory.ROLES, - fs, - historyPath, - containers, - null, - new MostRecentContainerReleaseSelector()) + + def bindingInfo = buildBindingInfo() + bindingInfo.instanceDefinition = factory.newInstanceDefinition(r0, r1, r2) + bindingInfo.liveContainers = containers + appState.buildInstance(bindingInfo) assert appState.startedCountainerCount == clusterSize @@ -107,22 +99,18 @@ class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest assertNotNull("Null entry in original nodemap for " + hostname, orig) for (int i = 0; i < ROLE_COUNT; i++) { - - assert (nodeInstance.getActiveRoleInstances(i) == - orig.getActiveRoleInstances(i)) + assert (nodeInstance.getActiveRoleInstances(i) == orig.getActiveRoleInstances(i)) NodeEntry origRE = orig.getOrCreate(i) NodeEntry newRE = nodeInstance.getOrCreate(i) assert origRE.live == newRE.live - assert newRE.starting == 0 + assert 0 == newRE.starting } } - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() - assert ops.size() == 0 + assert 0 == appState.reviewRequestAndReleaseNodes().size() def status = appState.getClusterStatus() // verify the AM restart container count was set - String restarted = status.getInfo( - StatusKeys.INFO_CONTAINERS_AM_RESTART) + String restarted = status.getInfo(StatusKeys.INFO_CONTAINERS_AM_RESTART) assert restarted != null; //and that the count == 1 master + the region servers assert Integer.parseInt(restarted) == containers.size() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy index 4ba0afd..ad607cf 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy @@ -25,6 +25,7 @@ import org.apache.slider.api.ResourceKeys import org.apache.slider.core.conf.ConfTree import org.apache.slider.core.conf.ConfTreeOperations import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest +import org.apache.slider.server.appmaster.model.mock.MockAppState import org.apache.slider.server.appmaster.model.mock.MockRoles import org.apache.slider.server.appmaster.operations.AbstractRMOperation import org.apache.slider.server.appmaster.operations.ContainerRequestOperation @@ -71,7 +72,7 @@ class TestMockContainerResourceAllocations extends BaseMockAppStateTest { assert ops.size() == 1 ContainerRequestOperation operation = (ContainerRequestOperation) ops[0] Resource requirements = operation.request.capability - assert requirements.memory == RM_MAX_RAM + assert requirements.memory == MockAppState.RM_MAX_RAM assert requirements.virtualCores == 2 } @@ -89,7 +90,7 @@ class TestMockContainerResourceAllocations extends BaseMockAppStateTest { ContainerRequestOperation operation = (ContainerRequestOperation) ops[0] Resource requirements = operation.request.capability assert requirements.memory == 512 - assert requirements.virtualCores == RM_MAX_CORES + assert requirements.virtualCores == MockAppState.RM_MAX_CORES } @Test http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy index fa10145..8ab63aa 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy @@ -68,7 +68,7 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { public void setupRH() { roleHistory.onStart(fs, historyPath) roleHistory.insert(nodes) - roleHistory.buildAvailableNodeLists(); + roleHistory.buildRecentNodeLists(); resource = Resource.newInstance(ResourceKeys.DEF_YARN_CORES, ResourceKeys.DEF_YARN_MEMORY); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy index 79d23e5..c4768ec 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy @@ -63,7 +63,7 @@ class TestRoleHistoryFindNodesForNewInstances extends BaseMockAppStateTest { @Before public void setupNodeMap() { roleHistory.insert(nodes) - roleHistory.buildAvailableNodeLists(); + roleHistory.buildRecentNodeLists(); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy index c81c686..7afcfc1 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy @@ -26,11 +26,9 @@ import org.apache.slider.providers.PlacementPolicy import org.apache.slider.providers.ProviderRole import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest import org.apache.slider.server.appmaster.model.mock.MockFactory -import org.apache.slider.server.appmaster.model.mock.MockRoles import org.apache.slider.server.appmaster.state.NodeEntry import org.apache.slider.server.appmaster.state.NodeInstance import org.apache.slider.server.appmaster.state.RoleHistory -import org.apache.slider.server.avro.LoadedRoleHistory import org.apache.slider.server.avro.RoleHistoryWriter import org.junit.Test @@ -136,7 +134,7 @@ class TestRoleHistoryRW extends BaseMockAppStateTest { assert rh2.thawedDataTime == savetime // now start it - rh2.buildAvailableNodeLists(); + rh2.buildRecentNodeLists(); describe("starting") rh2.dump(); List<NodeInstance> available0 = rh2.cloneAvailableList(0) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy index 9847992..db795d0 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy @@ -67,7 +67,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { @Before public void setupNodeMap() { roleHistory.insert(nodes) - roleHistory.buildAvailableNodeLists(); + roleHistory.buildRecentNodeLists(); } @Test @@ -141,7 +141,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { recordAsFailed(age4Active1, key, 4) // trigger a list rebuild - roleHistory.buildAvailableNodeLists(); + roleHistory.buildRecentNodeLists(); assert !roleHistory.cloneAvailableList(key).isEmpty() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy index 29eefa5..33ea0a0 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy @@ -20,7 +20,6 @@ package org.apache.slider.server.appmaster.model.mock import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.FileSystem as HadoopFS import org.apache.hadoop.fs.Path import org.apache.hadoop.yarn.api.records.Container @@ -34,21 +33,19 @@ import org.apache.slider.core.conf.AggregateConf import org.apache.slider.core.main.LauncherExitCodes import org.apache.slider.server.appmaster.operations.AbstractRMOperation import org.apache.slider.server.appmaster.state.AppState +import org.apache.slider.server.appmaster.state.AppStateBindingInfo import org.apache.slider.server.appmaster.state.ContainerAssignment import org.apache.slider.server.appmaster.state.ContainerOutcome import org.apache.slider.server.appmaster.state.NodeEntry import org.apache.slider.server.appmaster.state.NodeInstance import org.apache.slider.server.appmaster.state.RoleInstance import org.apache.slider.server.appmaster.state.RoleStatus -import org.apache.slider.server.appmaster.state.SimpleReleaseSelector import org.apache.slider.test.SliderTestBase import org.junit.Before @CompileStatic @Slf4j abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles { - public static final int RM_MAX_RAM = 4096 - public static final int RM_MAX_CORES = 64 MockFactory factory = new MockFactory() MockAppState appState MockYarnEngine engine @@ -77,12 +74,14 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles return new MockYarnEngine(64, 1) } + /** + * Initialize the application. + * This uses the binding information supplied by {@link #buildBindingInfo()}. + */ @Before void initApp(){ String historyDirName = testName; - - YarnConfiguration conf = SliderUtils.createConfiguration() applicationId = new MockApplicationId(id: 1, clusterTimestamp: 0) applicationAttemptId = new MockApplicationAttemptId( @@ -94,29 +93,35 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles historyPath = new Path(historyWorkDir.toURI()) fs.delete(historyPath, true) appState = new MockAppState() - appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES) - appState.buildInstance( - buildInstanceDefinition(), - new Configuration(), - new Configuration(false), - factory.ROLES, - fs, - historyPath, - null, null, - new SimpleReleaseSelector()) + appState.buildInstance(buildBindingInfo()) } /** - * Override point, define the instance definition + * Build the binding info from the default constructor values, + * the roles from {@link #factory}, and an instance definition + * from {@link #buildInstanceDefinition()} * @return */ + AppStateBindingInfo buildBindingInfo() { + AppStateBindingInfo binding = new AppStateBindingInfo() + binding.instanceDefinition = buildInstanceDefinition(); + binding.roles = factory.ROLES + binding.fs = fs + binding.historyPath = historyPath + binding + } + + /** + * Override point, define the instance definition + * @return the instance definition + */ public AggregateConf buildInstanceDefinition() { factory.newInstanceDefinition(0, 0, 0) } /** * Get the test name ... defaults to method name - * @return + * @return the method name */ String getTestName() { methodName.methodName; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy index 6e21a38..5565e6b 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy @@ -21,12 +21,14 @@ import org.apache.slider.providers.ProviderRole import org.apache.slider.server.appmaster.management.MetricsAndMonitoring import org.apache.slider.server.appmaster.state.AbstractClusterServices import org.apache.slider.server.appmaster.state.AppState +import org.apache.slider.server.appmaster.state.AppStateBindingInfo /** * Extended app state that makes more things public */ class MockAppState extends AppState { - + public static final int RM_MAX_RAM = 4096 + public static final int RM_MAX_CORES = 64 public MockAppState(AbstractClusterServices recordFactory) { super(recordFactory, new MetricsAndMonitoring()); } @@ -38,6 +40,12 @@ class MockAppState extends AppState { */ public MockAppState() { super(new MockClusterServices(), new MetricsAndMonitoring()); + setContainerLimits(1, RM_MAX_RAM, 1, RM_MAX_CORES) + } + + MockAppState(AppStateBindingInfo bindingInfo) { + this() + buildInstance(bindingInfo) } public Map<String, ProviderRole> getRoleMap() { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java index 4c43168..7237ff4 100644 --- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java +++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java @@ -24,10 +24,10 @@ import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.client.config.ClientConfig; import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.api.json.JSONConfiguration; -import junit.framework.Assert; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.slider.common.SliderKeys; @@ -35,10 +35,12 @@ import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.conf.MapOperations; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; +import org.apache.slider.server.appmaster.model.mock.MockAppState; import org.apache.slider.server.appmaster.model.mock.MockFactory; import org.apache.slider.server.appmaster.model.mock.MockProviderService; import org.apache.slider.server.appmaster.model.mock.MockClusterServices; import org.apache.slider.server.appmaster.state.AppState; +import org.apache.slider.server.appmaster.state.AppStateBindingInfo; import org.apache.slider.server.appmaster.state.ProviderAppState; import org.apache.slider.server.appmaster.state.SimpleReleaseSelector; import org.apache.slider.server.appmaster.web.WebAppApi; @@ -71,26 +73,19 @@ public class TestAMAgentWebServices { public boolean verify(String hostname, javax.net.ssl.SSLSession sslSession) { - if (hostname.equals("localhost")) { - return true; - } - return false; + return hostname.equals("localhost"); } }); - } protected static final Logger log = LoggerFactory.getLogger(TestAMAgentWebServices.class); - public static final int RM_MAX_RAM = 4096; - public static final int RM_MAX_CORES = 64; public static final String AGENT_URL = "https://localhost:${PORT}/ws/v1/slider/agents/";; static MockFactory factory = new MockFactory(); - private static Configuration conf = new Configuration(); private static WebAppApi slider; private static FileSystem fs; @@ -117,28 +112,16 @@ public class TestAMAgentWebServices { YarnConfiguration conf = SliderUtils.createConfiguration(); fs = FileSystem.get(new URI("file:///"), conf); AppState appState = null; - try { - fs = FileSystem.get(new URI("file:///"), conf); - File - historyWorkDir = - new File("target/history", "TestAMAgentWebServices"); - org.apache.hadoop.fs.Path - historyPath = - new org.apache.hadoop.fs.Path(historyWorkDir.toURI()); - fs.delete(historyPath, true); - appState = new AppState(new MockClusterServices(), new MetricsAndMonitoring()); - appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES); - appState.buildInstance( - factory.newInstanceDefinition(0, 0, 0), - new Configuration(), - new Configuration(false), - factory.ROLES, - fs, - historyPath, - null, null, new SimpleReleaseSelector()); - } catch (Exception e) { - log.error("Failed to set up app {}", e, e); - } + File historyWorkDir = new File("target/history", "TestAMAgentWebServices"); + Path historyPath = new Path(historyWorkDir.toURI()); + fs.delete(historyPath, true); + appState = new MockAppState(new MockClusterServices()); + AppStateBindingInfo binding = new AppStateBindingInfo(); + binding.instanceDefinition = factory.newInstanceDefinition(0, 0, 0); + binding.roles = MockFactory.ROLES; + binding.fs = fs; + binding.historyPath = historyPath; + appState.buildInstance(binding); ProviderAppState providerAppState = new ProviderAppState("undefined", appState); @@ -173,7 +156,7 @@ public class TestAMAgentWebServices { WebResource webResource = client.resource(base_url + "test/register"); response = webResource.type(MediaType.APPLICATION_JSON) .post(RegistrationResponse.class, createDummyJSONRegister()); - Assert.assertEquals(RegistrationStatus.OK, response.getResponseStatus()); + assertEquals(RegistrationStatus.OK, response.getResponseStatus()); } protected Client createTestClient() { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2f2c766/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java index df7e002..70a6bcb 100644 --- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java +++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java @@ -31,7 +31,6 @@ import com.sun.jersey.test.framework.JerseyTest; import com.sun.jersey.test.framework.WebAppDescriptor; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.conf.AggregateConf; @@ -39,13 +38,13 @@ import org.apache.slider.core.conf.ConfTree; import org.apache.slider.core.exceptions.BadClusterStateException; import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.core.persist.JsonSerDeser; -import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; +import org.apache.slider.server.appmaster.model.mock.MockAppState; +import org.apache.slider.server.appmaster.model.mock.MockClusterServices; import org.apache.slider.server.appmaster.model.mock.MockFactory; import org.apache.slider.server.appmaster.model.mock.MockProviderService; -import org.apache.slider.server.appmaster.model.mock.MockClusterServices; import org.apache.slider.server.appmaster.state.AppState; +import org.apache.slider.server.appmaster.state.AppStateBindingInfo; import org.apache.slider.server.appmaster.state.ProviderAppState; -import org.apache.slider.server.appmaster.state.SimpleReleaseSelector; import org.apache.slider.server.appmaster.web.WebAppApi; import org.apache.slider.server.appmaster.web.WebAppApiImpl; import org.apache.slider.server.appmaster.web.rest.AMWebServices; @@ -67,15 +66,11 @@ import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; public class TestAMManagementWebServices extends JerseyTest { protected static final Logger log = LoggerFactory.getLogger(TestAMManagementWebServices.class); - public static final int RM_MAX_RAM = 4096; - public static final int RM_MAX_CORES = 64; - public static final String EXAMPLES = - "/org/apache/slider/core/conf/examples/"; + public static final String EXAMPLES = "/org/apache/slider/core/conf/examples/"; static MockFactory factory = new MockFactory(); private static Configuration conf = new Configuration(); private static WebAppApi slider; @@ -116,40 +111,26 @@ public class TestAMManagementWebServices extends JerseyTest { } protected AggregateConf getAggregateConf() { - JsonSerDeser<ConfTree> confTreeJsonSerDeser = - new JsonSerDeser<ConfTree>(ConfTree.class); - ConfTree internal = null; - ConfTree app_conf = null; - ConfTree resources = null; try { - internal = - confTreeJsonSerDeser.fromResource( - EXAMPLES +"internal.json"); - app_conf = - confTreeJsonSerDeser.fromResource( - EXAMPLES + "app_configuration.json"); - resources = - confTreeJsonSerDeser.fromResource( - EXAMPLES + "resources.json"); + JsonSerDeser<ConfTree> confTreeJsonSerDeser = new JsonSerDeser<>(ConfTree.class); + AggregateConf aggregateConf = new AggregateConf( + confTreeJsonSerDeser.fromResource(EXAMPLES + "internal.json"), + confTreeJsonSerDeser.fromResource(EXAMPLES + "app_configuration.json"), + confTreeJsonSerDeser.fromResource(EXAMPLES + "resources.json")); + aggregateConf.setName("test"); + return aggregateConf; } catch (IOException e) { - fail(e.getMessage()); + throw new AssertionError(e.getMessage(), e); } - AggregateConf aggregateConf = new AggregateConf( - resources, - app_conf, - internal); - aggregateConf.setName("test"); - return aggregateConf; } - } + @Before @Override public void setUp() throws Exception { super.setUp(); injector = createInjector(); - YarnConfiguration conf = SliderUtils.createConfiguration(); - fs = FileSystem.get(new URI("file:///"), conf); + fs = FileSystem.get(new URI("file:///"), SliderUtils.createConfiguration()); } private static Injector createInjector() { @@ -160,23 +141,17 @@ public class TestAMManagementWebServices extends JerseyTest { AppState appState = null; try { fs = FileSystem.get(new URI("file:///"), conf); - File - historyWorkDir = - new File("target/history", "TestAMManagementWebServices"); - org.apache.hadoop.fs.Path - historyPath = + File historyWorkDir = new File("target/history", "TestAMManagementWebServices"); + org.apache.hadoop.fs.Path historyPath = new org.apache.hadoop.fs.Path(historyWorkDir.toURI()); fs.delete(historyPath, true); - appState = new AppState(new MockClusterServices(), new MetricsAndMonitoring()); - appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES); - appState.buildInstance( - factory.newInstanceDefinition(0, 0, 0), - new Configuration(), - new Configuration(false), - factory.ROLES, - fs, - historyPath, - null, null, new SimpleReleaseSelector()); + appState = new MockAppState(new MockClusterServices()); + AppStateBindingInfo binding = new AppStateBindingInfo(); + binding.instanceDefinition = factory.newInstanceDefinition(0, 0, 0); + binding.roles = MockFactory.ROLES; + binding.fs = fs; + binding.historyPath = historyPath; + appState.buildInstance(binding); } catch (IOException | BadClusterStateException | URISyntaxException | BadConfigException e) { log.error("{}", e, e); }
