CLOUDSTACK-6047: Make aggregation command timeout configurable In case some environments has different performance or we found some commands would took too long to execute, one global configuration item is introduced to specify "time out in seconds per one command in aggregation commands".
By default it's 3 seconds. If admin feel it's too long, it can be adjust to as low as 1 seconds, which runs still well in my machine. Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/3578c713 Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/3578c713 Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/3578c713 Branch: refs/heads/4.4-forward Commit: 3578c7137f42fdcaadad9b263e18921e2fa094da Parents: c7d8aae Author: Sheng Yang <[email protected]> Authored: Fri Apr 18 23:27:12 2014 -0700 Committer: Sheng Yang <[email protected]> Committed: Fri Apr 18 23:27:12 2014 -0700 ---------------------------------------------------------------------- .../virtualnetwork/VirtualRoutingResource.java | 8 +++- .../discoverer/HypervServerDiscoverer.java | 3 ++ .../vmware/VmwareServerDiscoverer.java | 28 ++++++------- .../xen/discoverer/XcpServerDiscoverer.java | 43 ++++++++++---------- server/src/com/cloud/configuration/Config.java | 26 ++++++++---- .../kvm/discoverer/LibvirtServerDiscoverer.java | 24 +++++------ .../src/com/cloud/resource/DiscovererBase.java | 24 +++++------ setup/db/db/schema-430to440.sql | 2 + 8 files changed, 87 insertions(+), 71 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3578c713/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index 3b43853..0a7aa26 100755 --- a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -134,6 +134,7 @@ public class VirtualRoutingResource { private int _sleep; private int _retry; private int _port; + private int _eachTimeout; private String _cfgVersion = "1.0"; @@ -972,6 +973,9 @@ public class VirtualRoutingResource { value = (String)params.get("ssh.port"); _port = NumbersUtil.parseInt(value, 3922); + value = (String)params.get("router.aggregation.command.each.timeout"); + _eachTimeout = NumbersUtil.parseInt(value, 3); + if (_vrDeployer == null) { throw new ConfigurationException("Unable to find the resource for VirtualRouterDeployer!"); } @@ -1153,8 +1157,8 @@ public class VirtualRoutingResource { return new Answer(cmd, false, result.getDetails()); } - // 3 second for each answer should be enough, and 120s is the minimal timeout - int timeout = answerCounts * 3; + // 120s is the minimal timeout + int timeout = answerCounts * _eachTimeout; if (timeout < 120) { timeout = 120; } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3578c713/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/discoverer/HypervServerDiscoverer.java ---------------------------------------------------------------------- diff --git a/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/discoverer/HypervServerDiscoverer.java b/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/discoverer/HypervServerDiscoverer.java index f4c15f6..c928124 100644 --- a/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/discoverer/HypervServerDiscoverer.java +++ b/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/discoverer/HypervServerDiscoverer.java @@ -43,6 +43,7 @@ import com.cloud.agent.api.SetupAnswer; import com.cloud.agent.api.SetupCommand; import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupRoutingCommand; +import com.cloud.configuration.Config; import com.cloud.alert.AlertManager; import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenterVO; @@ -276,6 +277,8 @@ public class HypervServerDiscoverer extends DiscovererBase implements Discoverer params.putAll(details); + params.put("router.aggregation.command.each.timeout", _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString())); + HypervDirectConnectResource resource = new HypervDirectConnectResource(); resource.configure(agentIp, params); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3578c713/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java ---------------------------------------------------------------------- diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java index 51db0d9..e0c1931 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java @@ -16,20 +16,6 @@ // under the License. package com.cloud.hypervisor.vmware; -import java.net.URI; -import java.net.URLDecoder; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import javax.ejb.Local; -import javax.inject.Inject; -import javax.naming.ConfigurationException; - -import org.apache.cloudstack.api.ApiConstants; -import org.apache.log4j.Logger; - import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupRoutingCommand; import com.cloud.alert.AlertManager; @@ -76,6 +62,18 @@ import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.vmware.vim25.ClusterDasConfigInfo; import com.vmware.vim25.ManagedObjectReference; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.log4j.Logger; + +import javax.ejb.Local; +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.net.URI; +import java.net.URLDecoder; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; @Local(value = Discoverer.class) public class VmwareServerDiscoverer extends DiscovererBase implements Discoverer, ResourceStateAdapter { @@ -384,6 +382,8 @@ public class VmwareServerDiscoverer extends DiscovererBase implements Discoverer params.put("guestTrafficInfo", guestTrafficLabelObj); params.put("publicTrafficInfo", publicTrafficLabelObj); + params.put("router.aggregation.command.each.timeout", _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString())); + VmwareResource resource = new VmwareResource(); try { resource.configure("VMware", params); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3578c713/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/discoverer/XcpServerDiscoverer.java ---------------------------------------------------------------------- diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/discoverer/XcpServerDiscoverer.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/discoverer/XcpServerDiscoverer.java index 2cb6f77..0e7cf74 100755 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/discoverer/XcpServerDiscoverer.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/discoverer/XcpServerDiscoverer.java @@ -16,24 +16,6 @@ // under the License. package com.cloud.hypervisor.xen.discoverer; -import java.net.InetAddress; -import java.net.URI; -import java.net.UnknownHostException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.Set; - -import javax.ejb.Local; -import javax.inject.Inject; -import javax.naming.ConfigurationException; -import javax.persistence.EntityExistsException; - -import org.apache.log4j.Logger; -import org.apache.xmlrpc.XmlRpcException; - import com.cloud.agent.AgentManager; import com.cloud.agent.Listener; import com.cloud.agent.api.AgentControlAnswer; @@ -75,7 +57,6 @@ import com.cloud.hypervisor.xen.resource.XenServer620Resource; import com.cloud.hypervisor.xen.resource.XenServer620SP1Resource; import com.cloud.hypervisor.xen.resource.XenServerConnectionPool; import com.cloud.hypervisor.xen.resource.Xenserver625Resource; -import org.apache.cloudstack.hypervisor.xenserver.XenserverConfigs; import com.cloud.resource.Discoverer; import com.cloud.resource.DiscovererBase; import com.cloud.resource.ResourceStateAdapter; @@ -93,13 +74,30 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.exception.HypervisorVersionChangedException; import com.xensource.xenapi.Connection; import com.xensource.xenapi.Host; +import com.xensource.xenapi.HostPatch; import com.xensource.xenapi.Pool; -import com.xensource.xenapi.Session; import com.xensource.xenapi.PoolPatch; -import com.xensource.xenapi.HostPatch; +import com.xensource.xenapi.Session; import com.xensource.xenapi.Types.SessionAuthenticationFailed; -import com.xensource.xenapi.Types.XenAPIException; import com.xensource.xenapi.Types.UuidInvalid; +import com.xensource.xenapi.Types.XenAPIException; +import org.apache.cloudstack.hypervisor.xenserver.XenserverConfigs; +import org.apache.log4j.Logger; +import org.apache.xmlrpc.XmlRpcException; + +import javax.ejb.Local; +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import javax.persistence.EntityExistsException; +import java.net.InetAddress; +import java.net.URI; +import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; @Local(value = Discoverer.class) @@ -344,6 +342,7 @@ public class XcpServerDiscoverer extends DiscovererBase implements Discoverer, L boolean securityGroupEnabled = zone.isSecurityGroupEnabled(); params.put("securitygroupenabled", Boolean.toString(securityGroupEnabled)); + params.put("router.aggregation.command.each.timeout", _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString())); params.put("wait", Integer.toString(_wait)); details.put("wait", Integer.toString(_wait)); params.put("migratewait", _configDao.getValue(Config.MigrateWait.toString())); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3578c713/server/src/com/cloud/configuration/Config.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index a02050f..e45291d 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -16,15 +16,6 @@ // under the License. package com.cloud.configuration; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.StringTokenizer; - -import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; -import org.apache.cloudstack.framework.config.ConfigKey; - import com.cloud.agent.AgentManager; import com.cloud.consoleproxy.ConsoleProxyManager; import com.cloud.ha.HighAvailabilityManager; @@ -38,6 +29,14 @@ import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.template.TemplateManager; import com.cloud.vm.UserVmManager; import com.cloud.vm.snapshot.VMSnapshotManager; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; +import org.apache.cloudstack.framework.config.ConfigKey; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.StringTokenizer; public enum Config { @@ -1924,6 +1923,15 @@ public enum Config { "seconds between VRRP broadcast. It would 3 times broadcast fail to trigger fail-over mechanism of redundant router", null), + RouterAggregationCommandEachTimeout( + "Advanced", + NetworkOrchestrationService.class, + Integer.class, + "router.aggregation.command.each.timeout", + "3", + "timeout in seconds for each Virtual Router command being aggregated. The final aggregation command timeout would be determined by this timeout * commands counts ", + null), + ManagementServerVendor("Advanced", ManagementServer.class, String.class, "mgt.server.vendor", "ACS", "the vendor of management server", null); private final String _category; http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3578c713/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java b/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java index d127256..8eb6c89 100644 --- a/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java +++ b/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java @@ -16,18 +16,6 @@ // under the License. package com.cloud.hypervisor.kvm.discoverer; -import java.net.InetAddress; -import java.net.URI; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; - -import org.apache.log4j.Logger; - import com.cloud.agent.AgentManager; import com.cloud.agent.Listener; import com.cloud.agent.api.AgentControlAnswer; @@ -55,6 +43,16 @@ import com.cloud.resource.ResourceStateAdapter; import com.cloud.resource.ServerResource; import com.cloud.resource.UnableDeleteHostException; import com.cloud.utils.ssh.SSHCmdHelper; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.net.InetAddress; +import java.net.URI; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; public abstract class LibvirtServerDiscoverer extends DiscovererBase implements Discoverer, Listener, ResourceStateAdapter { private static final Logger s_logger = Logger.getLogger(LibvirtServerDiscoverer.class); @@ -211,6 +209,8 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements KvmDummyResourceBase kvmResource = new KvmDummyResourceBase(); Map<String, Object> params = new HashMap<String, Object>(); + params.put("router.aggregation.command.each.timeout", _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString())); + params.put("zone", Long.toString(dcId)); params.put("pod", Long.toString(podId)); params.put("cluster", Long.toString(clusterId)); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3578c713/server/src/com/cloud/resource/DiscovererBase.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/resource/DiscovererBase.java b/server/src/com/cloud/resource/DiscovererBase.java index ae2a9c9..871dc15 100644 --- a/server/src/com/cloud/resource/DiscovererBase.java +++ b/server/src/com/cloud/resource/DiscovererBase.java @@ -16,18 +16,6 @@ // under the License. package com.cloud.resource; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.net.URL; -import java.util.HashMap; -import java.util.Map; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; - -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.log4j.Logger; - import com.cloud.configuration.Config; import com.cloud.dc.ClusterVO; import com.cloud.dc.dao.ClusterDao; @@ -37,6 +25,16 @@ import com.cloud.host.dao.HostDao; import com.cloud.network.NetworkModel; import com.cloud.utils.component.AdapterBase; import com.cloud.utils.net.UrlUtil; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; public abstract class DiscovererBase extends AdapterBase implements Discoverer { protected Map<String, String> _params; @@ -139,6 +137,8 @@ public abstract class DiscovererBase extends AdapterBase implements Discoverer { params.put("migratewait", _configDao.getValue(Config.MigrateWait.toString())); params.put(Config.XenMaxNics.toString().toLowerCase(), _configDao.getValue(Config.XenMaxNics.toString())); params.put(Config.XenHeartBeatInterval.toString().toLowerCase(), _configDao.getValue(Config.XenHeartBeatInterval.toString())); + params.put("router.aggregation.command.each.timeout", _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString())); + return params; } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3578c713/setup/db/db/schema-430to440.sql ---------------------------------------------------------------------- diff --git a/setup/db/db/schema-430to440.sql b/setup/db/db/schema-430to440.sql index 36f9f67..262fa37 100644 --- a/setup/db/db/schema-430to440.sql +++ b/setup/db/db/schema-430to440.sql @@ -1651,3 +1651,5 @@ CREATE TABLE `cloud`.`op_vpc_distributed_router_sequence_no` ( PRIMARY KEY (`id`), UNIQUE `u_op_vpc_distributed_router_sequence_no_vpc_id`(`vpc_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +INSERT INTO `cloud`.`configuration`(category, instance, component, name, value, description, default_value) VALUES ('Advanced', 'DEFAULT', 'NetworkOrchestrationService', 'router.aggregation.command.each.timeout', '3', 'timeout in seconds for each Virtual Router command being aggregated. The final aggregation command timeout would be determined by this timeout * commands counts ', '3') ON DUPLICATE KEY UPDATE category='Advanced';
