This is an automated email from the ASF dual-hosted git repository. bhaisaab pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/master by this push: new 98dc4eb CLOUDSTACK-9782: New Background Polling Task Manager (#2218) 98dc4eb is described below commit 98dc4eb96a3db1bd02f7ea7dab284f0d76b9dfb5 Author: Rohit Yadav <bhais...@apache.org> AuthorDate: Thu Aug 3 11:53:45 2017 +0200 CLOUDSTACK-9782: New Background Polling Task Manager (#2218) CloudStack has several background polling tasks that are spread across the codebase, the aim of this work is to provide a single manager to handle submission, execution and handling of background tasks. With the framework implemented, existing oobm background task has been refactored to use this manager. Signed-off-by: Rohit Yadav <rohit.ya...@shapeblue.com> --- .../ChangeOutOfBandManagementPasswordCmd.java | 2 +- .../ConfigureOutOfBandManagementCmd.java | 2 +- .../IssueOutOfBandManagementPowerActionCmd.java | 2 +- .../OutOfBandManagementService.java | 9 +-- .../cloudstack/poll/BackgroundPollManager.java | 23 +++---- .../apache/cloudstack/poll/BackgroundPollTask.java | 14 +--- developer/developer-prefill.sql | 4 -- .../agent/manager/ClusteredAgentManagerImpl.java | 5 +- .../dao/OutOfBandManagementDao.java | 2 +- .../dao/OutOfBandManagementDaoImpl.java | 2 +- .../core/spring-server-core-managers-context.xml | 4 ++ server/src/com/cloud/server/StatsCollector.java | 54 ++------------- .../OutOfBandManagementServiceImpl.java | 61 ++++++++++++++--- ...BackgroundTask.java => PowerOperationTask.java} | 8 +-- .../cloudstack/poll/BackgroundPollManagerImpl.java | 79 ++++++++++++++++++++++ .../poll/BackgroundPollManagerImplTest.java | 79 ++++++++++++++++++++++ setup/dev/advanced.cfg | 13 ++-- test/integration/smoke/test_outofbandmanagement.py | 72 +++++++++++--------- 18 files changed, 296 insertions(+), 139 deletions(-) diff --git a/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ChangeOutOfBandManagementPasswordCmd.java b/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ChangeOutOfBandManagementPasswordCmd.java index ea2e3cd..0499a6e 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ChangeOutOfBandManagementPasswordCmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ChangeOutOfBandManagementPasswordCmd.java @@ -74,7 +74,7 @@ public class ChangeOutOfBandManagementPasswordCmd extends BaseAsyncCmd { CallContext.current().setEventDetails("Host Id: " + host.getId() + " Password: " + getPassword().charAt(0) + "****"); CallContext.current().putContextParameter(Host.class, host.getUuid()); - final OutOfBandManagementResponse response = outOfBandManagementService.changeOutOfBandManagementPassword(host, getPassword()); + final OutOfBandManagementResponse response = outOfBandManagementService.changePassword(host, getPassword()); response.setResponseName(getCommandName()); setResponseObject(response); } diff --git a/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ConfigureOutOfBandManagementCmd.java b/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ConfigureOutOfBandManagementCmd.java index db22410..d9c6c9f 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ConfigureOutOfBandManagementCmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ConfigureOutOfBandManagementCmd.java @@ -83,7 +83,7 @@ public class ConfigureOutOfBandManagementCmd extends BaseCmd { throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId()); } CallContext.current().putContextParameter(Host.class, host.getUuid()); - final OutOfBandManagementResponse response = outOfBandManagementService.configureOutOfBandManagement(host, getHostPMOptions()); + final OutOfBandManagementResponse response = outOfBandManagementService.configure(host, getHostPMOptions()); response.setId(host.getUuid()); response.setResponseName(getCommandName()); setResponseObject(response); diff --git a/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/IssueOutOfBandManagementPowerActionCmd.java b/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/IssueOutOfBandManagementPowerActionCmd.java index 8d6bdd3..7593a3c 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/IssueOutOfBandManagementPowerActionCmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/IssueOutOfBandManagementPowerActionCmd.java @@ -80,7 +80,7 @@ public class IssueOutOfBandManagementPowerActionCmd extends BaseAsyncCmd { CallContext.current().setEventDetails("Host Id: " + host.getId() + " Action: " + powerOperation.toString()); CallContext.current().putContextParameter(Host.class, host.getUuid()); - final OutOfBandManagementResponse response = outOfBandManagementService.executeOutOfBandManagementPowerOperation(host, powerOperation, getActionTimeout()); + final OutOfBandManagementResponse response = outOfBandManagementService.executePowerOperation(host, powerOperation, getActionTimeout()); response.setResponseName(getCommandName()); setResponseObject(response); } diff --git a/api/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementService.java b/api/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementService.java index 699b2c6..a8b73b7 100644 --- a/api/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementService.java +++ b/api/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementService.java @@ -30,9 +30,6 @@ public interface OutOfBandManagementService { ConfigKey<Long> ActionTimeout = new ConfigKey<Long>("Advanced", Long.class, "outofbandmanagement.action.timeout", "60", "The out of band management action timeout in seconds, configurable by cluster", true, ConfigKey.Scope.Cluster); - ConfigKey<Long> SyncThreadInterval = new ConfigKey<Long>("Advanced", Long.class, "outofbandmanagement.sync.interval", "300000", - "The interval (in milliseconds) when the out-of-band management background sync are retrieved", true, ConfigKey.Scope.Global); - ConfigKey<Integer> SyncThreadPoolSize = new ConfigKey<Integer>("Advanced", Integer.class, "outofbandmanagement.sync.poolsize", "50", "The out of band management background sync thread pool size", true, ConfigKey.Scope.Global); @@ -49,7 +46,7 @@ public interface OutOfBandManagementService { OutOfBandManagementResponse disableOutOfBandManagement(Cluster cluster); OutOfBandManagementResponse disableOutOfBandManagement(Host host); - OutOfBandManagementResponse configureOutOfBandManagement(Host host, ImmutableMap<OutOfBandManagement.Option, String> options); - OutOfBandManagementResponse executeOutOfBandManagementPowerOperation(Host host, OutOfBandManagement.PowerOperation operation, Long timeout); - OutOfBandManagementResponse changeOutOfBandManagementPassword(Host host, String password); + OutOfBandManagementResponse configure(Host host, ImmutableMap<OutOfBandManagement.Option, String> options); + OutOfBandManagementResponse executePowerOperation(Host host, OutOfBandManagement.PowerOperation operation, Long timeout); + OutOfBandManagementResponse changePassword(Host host, String password); } diff --git a/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java b/api/src/org/apache/cloudstack/poll/BackgroundPollManager.java similarity index 54% copy from engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java copy to api/src/org/apache/cloudstack/poll/BackgroundPollManager.java index 5985b81..b9d5500 100644 --- a/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java +++ b/api/src/org/apache/cloudstack/poll/BackgroundPollManager.java @@ -15,17 +15,16 @@ // specific language governing permissions and limitations // under the License. -package org.apache.cloudstack.outofbandmanagement.dao; +package org.apache.cloudstack.poll; -import com.cloud.utils.db.GenericDao; -import com.cloud.utils.fsm.StateDao; -import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement; -import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementVO; - -import java.util.List; - -public interface OutOfBandManagementDao extends GenericDao<OutOfBandManagementVO, Long>, StateDao<OutOfBandManagement.PowerState, OutOfBandManagement.PowerState.Event, OutOfBandManagement> { - OutOfBandManagement findByHost(long hostId); - List<OutOfBandManagementVO> findAllByManagementServer(long serverId); - void expireOutOfBandManagementOwnershipByServer(long serverId); +public interface BackgroundPollManager { + /** + * Submits a background poll task that need to run continuously in the background + * to poll external resources, update states, trigger actions etc. + * Tasks must be submitted by a manager in configure-phase, the list of submitted tasks + * are then submitted to the internal executor service during start-phase. + * @param task periodic background task + * @since 4.11 + */ + void submitTask(final BackgroundPollTask task); } diff --git a/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java b/api/src/org/apache/cloudstack/poll/BackgroundPollTask.java similarity index 54% copy from engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java copy to api/src/org/apache/cloudstack/poll/BackgroundPollTask.java index 5985b81..8eea147 100644 --- a/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java +++ b/api/src/org/apache/cloudstack/poll/BackgroundPollTask.java @@ -15,17 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.cloudstack.outofbandmanagement.dao; +package org.apache.cloudstack.poll; -import com.cloud.utils.db.GenericDao; -import com.cloud.utils.fsm.StateDao; -import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement; -import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementVO; - -import java.util.List; - -public interface OutOfBandManagementDao extends GenericDao<OutOfBandManagementVO, Long>, StateDao<OutOfBandManagement.PowerState, OutOfBandManagement.PowerState.Event, OutOfBandManagement> { - OutOfBandManagement findByHost(long hostId); - List<OutOfBandManagementVO> findAllByManagementServer(long serverId); - void expireOutOfBandManagementOwnershipByServer(long serverId); +public interface BackgroundPollTask extends Runnable { } diff --git a/developer/developer-prefill.sql b/developer/developer-prefill.sql index 3fd3a1b..7fd85c6 100644 --- a/developer/developer-prefill.sql +++ b/developer/developer-prefill.sql @@ -114,10 +114,6 @@ INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) VALUES ('Advanced', 'DEFAULT', 'management-server', 'ping.timeout', '1.5'); -INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) - VALUES ('Advanced', 'DEFAULT', 'management-server', - 'outofbandmanagement.sync.interval', '1000'); - -- Enable dynamic RBAC by default for fresh deployments INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) VALUES ('Advanced', 'DEFAULT', 'RoleService', diff --git a/engine/orchestration/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java b/engine/orchestration/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java index 9239adc..a89988c 100644 --- a/engine/orchestration/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java +++ b/engine/orchestration/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java @@ -49,6 +49,7 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.managed.context.ManagedContextTimerTask; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.cloudstack.utils.security.SSLUtils; +import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao; import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; @@ -120,6 +121,8 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust ConfigurationDao _configDao; @Inject ConfigDepot _configDepot; + @Inject + private OutOfBandManagementDao outOfBandManagementDao; protected ClusteredAgentManagerImpl() { super(); @@ -736,7 +739,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust s_logger.info("Marking hosts as disconnected on Management server" + vo.getMsid()); final long lastPing = (System.currentTimeMillis() >> 10) - getTimeout(); _hostDao.markHostsAsDisconnected(vo.getMsid(), lastPing); - outOfBandManagementDao.expireOutOfBandManagementOwnershipByServer(vo.getMsid()); + outOfBandManagementDao.expireServerOwnership(vo.getMsid()); s_logger.info("Deleting entries from op_host_transfer table for Management server " + vo.getMsid()); cleanupTransferMap(vo.getMsid()); } diff --git a/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java b/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java index 5985b81..8a4ee36 100644 --- a/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java +++ b/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java @@ -27,5 +27,5 @@ import java.util.List; public interface OutOfBandManagementDao extends GenericDao<OutOfBandManagementVO, Long>, StateDao<OutOfBandManagement.PowerState, OutOfBandManagement.PowerState.Event, OutOfBandManagement> { OutOfBandManagement findByHost(long hostId); List<OutOfBandManagementVO> findAllByManagementServer(long serverId); - void expireOutOfBandManagementOwnershipByServer(long serverId); + void expireServerOwnership(long serverId); } diff --git a/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDaoImpl.java b/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDaoImpl.java index b914b6b..3cdd28f 100644 --- a/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDaoImpl.java +++ b/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDaoImpl.java @@ -110,7 +110,7 @@ public class OutOfBandManagementDaoImpl extends GenericDaoBase<OutOfBandManageme } @Override - public void expireOutOfBandManagementOwnershipByServer(long serverId) { + public void expireServerOwnership(long serverId) { final String resetOwnerSql = "UPDATE oobm set mgmt_server_id=NULL, power_state=NULL where mgmt_server_id=?"; executeExpireOwnershipSql(resetOwnerSql, serverId); if (LOG.isDebugEnabled()) { diff --git a/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index 87097a4..56da591 100644 --- a/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -70,6 +70,10 @@ value="#{resourceDiscoverersRegistry.registered}" /> </bean> + <!-- the new background poll manager --> + <bean id="bgPollManager" class="org.apache.cloudstack.poll.BackgroundPollManagerImpl"> + </bean> + <bean id="highAvailabilityManagerExtImpl" class="com.cloud.ha.HighAvailabilityManagerExtImpl"> <property name="investigators" value="#{haInvestigatorsRegistry.registered}" /> <property name="fenceBuilders" value="#{haFenceBuildersRegistry.registered}" /> diff --git a/server/src/com/cloud/server/StatsCollector.java b/server/src/com/cloud/server/StatsCollector.java index 9725217..048b3b1 100644 --- a/server/src/com/cloud/server/StatsCollector.java +++ b/server/src/com/cloud/server/StatsCollector.java @@ -20,7 +20,6 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Calendar; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -34,15 +33,6 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; -import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement; -import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService; -import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementVO; -import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao; -import org.apache.cloudstack.utils.identity.ManagementServerNode; -import org.apache.cloudstack.utils.usage.UsageUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; @@ -56,6 +46,10 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.graphite.GraphiteClient; import org.apache.cloudstack.utils.graphite.GraphiteException; +import org.apache.cloudstack.utils.usage.UsageUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; @@ -177,8 +171,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc @Inject private HostDao _hostDao; @Inject - private OutOfBandManagementDao outOfBandManagementDao; - @Inject private UserVmDao _userVmDao; @Inject private VolumeDao _volsDao; @@ -195,8 +187,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc @Inject private ResourceManager _resourceMgr; @Inject - private OutOfBandManagementService outOfBandManagementService; - @Inject private ConfigurationDao _configDao; @Inject private EndPointSelector _epSelector; @@ -244,7 +234,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc private ConcurrentHashMap<Long, StorageStats> _storagePoolStats = new ConcurrentHashMap<Long, StorageStats>(); long hostStatsInterval = -1L; - long hostOutOfBandManagementStatsInterval = -1L; long hostAndVmStatsInterval = -1L; long storageStatsInterval = -1L; long volumeStatsInterval = -1L; @@ -290,7 +279,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc private void init(Map<String, String> configs) { _executor = Executors.newScheduledThreadPool(6, new NamedThreadFactory("StatsCollector")); - hostOutOfBandManagementStatsInterval = OutOfBandManagementService.SyncThreadInterval.value(); hostStatsInterval = NumbersUtil.parseLong(configs.get("host.stats.interval"), 60000L); hostAndVmStatsInterval = NumbersUtil.parseLong(configs.get("vm.stats.interval"), 60000L); storageStatsInterval = NumbersUtil.parseLong(configs.get("storage.stats.interval"), 60000L); @@ -337,10 +325,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc _executor.scheduleWithFixedDelay(new HostCollector(), 15000L, hostStatsInterval, TimeUnit.MILLISECONDS); } - if (hostOutOfBandManagementStatsInterval > 0) { - _executor.scheduleWithFixedDelay(new HostOutOfBandManagementStatsCollector(), 15000L, hostOutOfBandManagementStatsInterval, TimeUnit.MILLISECONDS); - } - if (hostAndVmStatsInterval > 0) { _executor.scheduleWithFixedDelay(new VmStatsCollector(), 15000L, hostAndVmStatsInterval, TimeUnit.MILLISECONDS); } @@ -477,36 +461,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc } } - class HostOutOfBandManagementStatsCollector extends ManagedContextRunnable { - @Override - protected void runInContext() { - try { - s_logger.trace("HostOutOfBandManagementStatsCollector is running..."); - List<OutOfBandManagementVO> outOfBandManagementHosts = outOfBandManagementDao.findAllByManagementServer(ManagementServerNode.getManagementServerId()); - if (outOfBandManagementHosts == null) { - return; - } - for (OutOfBandManagement outOfBandManagementHost : outOfBandManagementHosts) { - Host host = _hostDao.findById(outOfBandManagementHost.getHostId()); - if (host == null) { - continue; - } - if (outOfBandManagementService.isOutOfBandManagementEnabled(host)) { - outOfBandManagementService.submitBackgroundPowerSyncTask(host); - } else if (outOfBandManagementHost.getPowerState() != OutOfBandManagement.PowerState.Disabled) { - if (outOfBandManagementService.transitionPowerStateToDisabled(Collections.singletonList(host))) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Out-of-band management was disabled in zone/cluster/host, disabled power state for host id:" + host.getId()); - } - } - } - } - } catch (Throwable t) { - s_logger.error("Error trying to retrieve host out-of-band management stats", t); - } - } - } - class VmStatsCollector extends ManagedContextRunnable { @Override protected void runInContext() { diff --git a/server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java b/server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java index ca65ef7..4f0ac74 100644 --- a/server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java +++ b/server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java @@ -44,10 +44,13 @@ import org.apache.cloudstack.api.response.OutOfBandManagementResponse; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao; import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverChangePasswordCommand; import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverPowerCommand; import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverResponse; +import org.apache.cloudstack.poll.BackgroundPollManager; +import org.apache.cloudstack.poll.BackgroundPollTask; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -78,6 +81,8 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf private HostDao hostDao; @Inject private AlertManager alertMgr; + @Inject + private BackgroundPollManager backgroundPollManager; private String name; private long serviceId; @@ -202,12 +207,14 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf OutOfBandManagement.PowerState currentPowerState = outOfBandManagementHost.getPowerState(); try { OutOfBandManagement.PowerState newPowerState = OutOfBandManagement.PowerState.getStateMachine().getNextState(currentPowerState, event); - boolean result = outOfBandManagementDao.updateState(currentPowerState, event, newPowerState, outOfBandManagementHost, null); + boolean result = OutOfBandManagement.PowerState.getStateMachine().transitTo(outOfBandManagementHost, event, null, outOfBandManagementDao); if (result) { final String message = String.format("Transitioned out-of-band management power state from:%s to:%s due to event:%s for the host id:%d", currentPowerState, newPowerState, event, outOfBandManagementHost.getHostId()); LOG.debug(message); - ActionEventUtils.onActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), Domain.ROOT_DOMAIN, - EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_POWERSTATE_TRANSITION, message); + if (newPowerState == OutOfBandManagement.PowerState.Unknown) { + ActionEventUtils.onActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), Domain.ROOT_DOMAIN, + EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_POWERSTATE_TRANSITION, message); + } } return result; } catch (NoTransitionException ignored) { @@ -278,7 +285,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf public void submitBackgroundPowerSyncTask(final Host host) { if (host != null) { - backgroundSyncBlockingExecutor.submit(new OutOfBandManagementBackgroundTask(this, host, OutOfBandManagement.PowerOperation.STATUS)); + backgroundSyncBlockingExecutor.submit(new PowerOperationTask(this, host, OutOfBandManagement.PowerOperation.STATUS)); } } @@ -356,7 +363,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf @Override @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_CONFIGURE, eventDescription = "updating out-of-band management configuration") - public OutOfBandManagementResponse configureOutOfBandManagement(final Host host, final ImmutableMap<OutOfBandManagement.Option, String> options) { + public OutOfBandManagementResponse configure(final Host host, final ImmutableMap<OutOfBandManagement.Option, String> options) { OutOfBandManagement outOfBandManagementConfig = outOfBandManagementDao.findByHost(host.getId()); if (outOfBandManagementConfig == null) { outOfBandManagementConfig = outOfBandManagementDao.persist(new OutOfBandManagementVO(host.getId())); @@ -384,7 +391,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf @Override @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_ACTION, eventDescription = "issuing host out-of-band management action", async = true) - public OutOfBandManagementResponse executeOutOfBandManagementPowerOperation(final Host host, final OutOfBandManagement.PowerOperation powerOperation, final Long timeout) { + public OutOfBandManagementResponse executePowerOperation(final Host host, final OutOfBandManagement.PowerOperation powerOperation, final Long timeout) { checkOutOfBandManagementEnabledByZoneClusterHost(host); final OutOfBandManagement outOfBandManagementConfig = getConfigForHost(host); final ImmutableMap<OutOfBandManagement.Option, String> options = getOptions(outOfBandManagementConfig); @@ -428,7 +435,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf @Override @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_CHANGE_PASSWORD, eventDescription = "updating out-of-band management password") - public OutOfBandManagementResponse changeOutOfBandManagementPassword(final Host host, final String newPassword) { + public OutOfBandManagementResponse changePassword(final Host host, final String newPassword) { checkOutOfBandManagementEnabledByZoneClusterHost(host); if (Strings.isNullOrEmpty(newPassword)) { throw new CloudRuntimeException(String.format("Cannot change out-of-band management password as provided new-password is null or empty for the host %s.", host.getUuid())); @@ -503,7 +510,9 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<Runnable>(10 * poolSize, true), new ThreadPoolExecutor.CallerRunsPolicy()); - LOG.info("Starting out-of-band management background sync executor with thread pool-size=" + poolSize + " and background sync thread interval=" + SyncThreadInterval.value() + "s"); + backgroundPollManager.submitTask(new OutOfBandManagementPowerStatePollTask()); + + LOG.info("Starting out-of-band management background sync executor with thread pool-size=" + poolSize); return true; } @@ -516,7 +525,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf @Override public boolean stop() { backgroundSyncBlockingExecutor.shutdown(); - outOfBandManagementDao.expireOutOfBandManagementOwnershipByServer(getId()); + outOfBandManagementDao.expireServerOwnership(getId()); return true; } @@ -527,7 +536,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf @Override public ConfigKey<?>[] getConfigKeys() { - return new ConfigKey<?>[] {ActionTimeout, SyncThreadInterval, SyncThreadPoolSize}; + return new ConfigKey<?>[] {ActionTimeout, SyncThreadPoolSize}; } public List<OutOfBandManagementDriver> getOutOfBandManagementDrivers() { @@ -537,4 +546,36 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf public void setOutOfBandManagementDrivers(List<OutOfBandManagementDriver> outOfBandManagementDrivers) { this.outOfBandManagementDrivers = outOfBandManagementDrivers; } + + private final class OutOfBandManagementPowerStatePollTask extends ManagedContextRunnable implements BackgroundPollTask { + @Override + protected void runInContext() { + try { + if (LOG.isTraceEnabled()) { + LOG.trace("Host out-of-band management power state poll task is running..."); + } + final List<OutOfBandManagementVO> outOfBandManagementHosts = outOfBandManagementDao.findAllByManagementServer(ManagementServerNode.getManagementServerId()); + if (outOfBandManagementHosts == null || outOfBandManagementHosts.isEmpty()) { + return; + } + for (final OutOfBandManagement outOfBandManagementHost : outOfBandManagementHosts) { + final Host host = hostDao.findById(outOfBandManagementHost.getHostId()); + if (host == null) { + continue; + } + if (isOutOfBandManagementEnabled(host)) { + submitBackgroundPowerSyncTask(host); + } else if (outOfBandManagementHost.getPowerState() != OutOfBandManagement.PowerState.Disabled) { + if (transitionPowerStateToDisabled(Collections.singletonList(host))) { + if (LOG.isDebugEnabled()) { + LOG.debug("Out-of-band management was disabled in zone/cluster/host, disabled power state for host id:" + host.getId()); + } + } + } + } + } catch (Throwable t) { + LOG.error("Error trying to retrieve host out-of-band management stats", t); + } + } + } } diff --git a/server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementBackgroundTask.java b/server/src/org/apache/cloudstack/outofbandmanagement/PowerOperationTask.java similarity index 80% rename from server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementBackgroundTask.java rename to server/src/org/apache/cloudstack/outofbandmanagement/PowerOperationTask.java index 38e139e..9c346d7 100644 --- a/server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementBackgroundTask.java +++ b/server/src/org/apache/cloudstack/outofbandmanagement/PowerOperationTask.java @@ -20,14 +20,14 @@ package org.apache.cloudstack.outofbandmanagement; import com.cloud.host.Host; import org.apache.log4j.Logger; -public class OutOfBandManagementBackgroundTask implements Runnable { - public static final Logger LOG = Logger.getLogger(OutOfBandManagementBackgroundTask.class); +public class PowerOperationTask implements Runnable { + public static final Logger LOG = Logger.getLogger(PowerOperationTask.class); final private OutOfBandManagementService service; final private Host host; final private OutOfBandManagement.PowerOperation powerOperation; - public OutOfBandManagementBackgroundTask(OutOfBandManagementService service, Host host, OutOfBandManagement.PowerOperation powerOperation) { + public PowerOperationTask(OutOfBandManagementService service, Host host, OutOfBandManagement.PowerOperation powerOperation) { this.service = service; this.host = host; this.powerOperation = powerOperation; @@ -41,7 +41,7 @@ public class OutOfBandManagementBackgroundTask implements Runnable { @Override public void run() { try { - service.executeOutOfBandManagementPowerOperation(host, powerOperation, null); + service.executePowerOperation(host, powerOperation, null); } catch (Exception e) { LOG.warn(String.format("Out-of-band management background task operation=%s for host id=%d failed with: %s", powerOperation.name(), host.getId(), e.getMessage())); diff --git a/server/src/org/apache/cloudstack/poll/BackgroundPollManagerImpl.java b/server/src/org/apache/cloudstack/poll/BackgroundPollManagerImpl.java new file mode 100644 index 0000000..c0a7f1c --- /dev/null +++ b/server/src/org/apache/cloudstack/poll/BackgroundPollManagerImpl.java @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.poll; + +import com.cloud.utils.component.Manager; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.exception.CloudRuntimeException; +import com.google.common.base.Preconditions; +import org.apache.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +public final class BackgroundPollManagerImpl extends ManagerBase implements BackgroundPollManager, Manager { + public static final Logger LOG = Logger.getLogger(BackgroundPollManagerImpl.class); + + private ScheduledExecutorService backgroundPollTaskScheduler; + private List<BackgroundPollTask> submittedTasks = new ArrayList<>(); + private volatile boolean isConfiguredAndStarted = false; + + public long getInitialDelay() { + return 5000L; + } + + public long getRoundDelay() { + return 4000L; + } + + @Override + public boolean start() { + if (isConfiguredAndStarted) { + return true; + } + backgroundPollTaskScheduler = Executors.newScheduledThreadPool(submittedTasks.size() + 1, new NamedThreadFactory("BackgroundTaskPollManager")); + for (final BackgroundPollTask task : submittedTasks) { + backgroundPollTaskScheduler.scheduleWithFixedDelay(task, getInitialDelay(), getRoundDelay(), TimeUnit.MILLISECONDS); + LOG.debug("Scheduled background poll task: " + task.getClass().getName()); + } + isConfiguredAndStarted = true; + return true; + } + + @Override + public boolean stop() { + if (isConfiguredAndStarted) { + backgroundPollTaskScheduler.shutdown(); + } + return true; + } + + @Override + public void submitTask(final BackgroundPollTask task) { + Preconditions.checkNotNull(task); + if (isConfiguredAndStarted) { + throw new CloudRuntimeException("Background Poll Manager cannot accept poll task as it has been configured and started."); + } + LOG.debug("Background Poll Manager received task: " + task.getClass().getSimpleName()); + submittedTasks.add(task); + } +} diff --git a/server/test/org/apache/cloudstack/poll/BackgroundPollManagerImplTest.java b/server/test/org/apache/cloudstack/poll/BackgroundPollManagerImplTest.java new file mode 100644 index 0000000..3304aba --- /dev/null +++ b/server/test/org/apache/cloudstack/poll/BackgroundPollManagerImplTest.java @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.poll; + +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class BackgroundPollManagerImplTest { + + private BackgroundPollManagerImpl pollManager; + private DummyPollTask pollTask; + + private class DummyPollTask extends ManagedContextRunnable implements BackgroundPollTask { + private boolean didIRun = false; + private long counter = 0; + + public boolean didItRan() { + return didIRun; + } + + public long getCounter() { + return counter; + } + + @Override + protected void runInContext() { + didIRun = true; + counter++; + } + } + + @Before + public void setUp() throws Exception { + pollManager = new BackgroundPollManagerImpl(); + pollTask = new DummyPollTask(); + } + + @After + public void tearDown() throws Exception { + pollManager.stop(); + } + + @Test + public void testSubmitValidTask() throws Exception { + Assert.assertFalse(pollTask.didItRan()); + Assert.assertTrue(pollTask.getCounter() == 0); + + pollManager.submitTask(pollTask); + pollManager.start(); + Thread.sleep(pollManager.getInitialDelay()*2); + + Assert.assertTrue(pollTask.didItRan()); + Assert.assertTrue(pollTask.getCounter() > 0); + } + + @Test(expected = NullPointerException.class) + public void testSubmitNullTask() throws Exception { + pollManager.submitTask(null); + } + +} \ No newline at end of file diff --git a/setup/dev/advanced.cfg b/setup/dev/advanced.cfg index 0742f24..bdb25e5 100644 --- a/setup/dev/advanced.cfg +++ b/setup/dev/advanced.cfg @@ -113,6 +113,11 @@ "username": "root", "url": "http://sim/c1/h0", "password": "password" + }, + { + "username": "root", + "url": "http://sim/c1/h1", + "password": "password" } ], "clustertype": "CloudManaged", @@ -120,6 +125,10 @@ { "url": "nfs://10.147.28.6:/export/home/sandbox/primary2", "name": "PS2" + }, + { + "url": "nfs://10.147.28.6:/export/home/sandbox/primary3", + "name": "PS2" } ] } @@ -223,10 +232,6 @@ { "name": "ping.timeout", "value": "1.5" - }, - { - "name": "outofbandmanagement.sync.interval", - "value": "1000" } ], "mgtSvr": [ diff --git a/test/integration/smoke/test_outofbandmanagement.py b/test/integration/smoke/test_outofbandmanagement.py index e91d49a..a52e3b1 100644 --- a/test/integration/smoke/test_outofbandmanagement.py +++ b/test/integration/smoke/test_outofbandmanagement.py @@ -211,8 +211,10 @@ class TestOutOfBandManagement(cloudstackTestCase): cmd.driver = 'randomDriverThatDoesNotExist' try: response = self.apiclient.configureOutOfBandManagement(cmd) + except Exception: + pass + else: self.fail("Expected an exception to be thrown, failing") - except Exception: pass @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") @@ -239,20 +241,26 @@ class TestOutOfBandManagement(cloudstackTestCase): cmd.hostid = -1 try: response = self.apiclient.enableOutOfBandManagementForHost(cmd) + except Exception: + pass + else: self.fail("Expected an exception to be thrown, failing") - except Exception: pass try: cmd = enableOutOfBandManagementForCluster.enableOutOfBandManagementForClusterCmd() response = self.apiclient.enableOutOfBandManagementForCluster(cmd) + except Exception: + pass + else: self.fail("Expected an exception to be thrown, failing") - except Exception: pass try: cmd = enableOutOfBandManagementForZone.enableOutOfBandManagementForZoneCmd() response = self.apiclient.enableOutOfBandManagementForZone(cmd) + except Exception: + pass + else: self.fail("Expected an exception to be thrown, failing") - except Exception: pass @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") @@ -265,20 +273,26 @@ class TestOutOfBandManagement(cloudstackTestCase): cmd.hostid = -1 try: response = self.apiclient.disableOutOfBandManagementForHost(cmd) + except Exception: + pass + else: self.fail("Expected an exception to be thrown, failing") - except Exception: pass try: cmd = disableOutOfBandManagementForCluster.disableOutOfBandManagementForClusterCmd() response = self.apiclient.disableOutOfBandManagementForCluster(cmd) + except Exception: + pass + else: self.fail("Expected an exception to be thrown, failing") - except Exception: pass try: cmd = disableOutOfBandManagementForZone.disableOutOfBandManagementForZoneCmd() response = self.apiclient.disableOutOfBandManagementForZone(cmd) + except Exception: + pass + else: self.fail("Expected an exception to be thrown, failing") - except Exception: pass @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") @@ -340,8 +354,10 @@ class TestOutOfBandManagement(cloudstackTestCase): try: self.issuePowerActionCmd('STATUS') - self.fail("Exception was expected, oobm is disabled at zone level") - except Exception: pass + except Exception: + pass + else: + self.fail("Expected an exception to be thrown, failing") # Enable at zone level cmd = enableOutOfBandManagementForZone.enableOutOfBandManagementForZoneCmd() @@ -350,18 +366,16 @@ class TestOutOfBandManagement(cloudstackTestCase): try: self.issuePowerActionCmd('STATUS') - self.fail("Exception was expected, oobm is disabled at cluster level") - except Exception: pass + except Exception: + pass + else: + self.fail("Expected an exception to be thrown, failing") # Check background thread syncs state to Disabled response = self.getHost(hostId=host.id).outofbandmanagement self.assertEqual(response.powerstate, 'Disabled') self.dbclient.execute("update oobm set power_state='On' where port=%d" % self.getIpmiServerPort()) - interval = list_configurations( - self.apiclient, - name='outofbandmanagement.sync.interval' - )[0].value - self.checkSyncToState('Disabled', interval) + self.checkSyncToState('Disabled', 2) # Enable at cluster level cmd = enableOutOfBandManagementForCluster.enableOutOfBandManagementForClusterCmd() @@ -370,8 +384,10 @@ class TestOutOfBandManagement(cloudstackTestCase): try: self.issuePowerActionCmd('STATUS') - self.fail("Exception was expected, oobm is disabled at host level") - except Exception: pass + except Exception: + pass + else: + self.fail("Expected an exception to be thrown, failing") # Enable at host level cmd = enableOutOfBandManagementForHost.enableOutOfBandManagementForHostCmd() @@ -452,25 +468,21 @@ class TestOutOfBandManagement(cloudstackTestCase): Tests out-of-band management background powerstate sync """ self.debug("Testing oobm background sync") - interval = list_configurations( - self.apiclient, - name='outofbandmanagement.sync.interval' - )[0].value self.configureAndEnableOobm() bmc = IpmiServerContext().bmc bmc.powerstate = 'on' - self.checkSyncToState('On', interval) + self.checkSyncToState('On', 2) bmc.powerstate = 'off' - self.checkSyncToState('Off', interval) + self.checkSyncToState('Off', 2) # Check for unknown state (ipmi server not reachable) cmd = self.getOobmConfigCmd() cmd.port = 1 response = self.apiclient.configureOutOfBandManagement(cmd) - self.checkSyncToState('Unknown', interval) + self.checkSyncToState('Unknown', 2) @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") @@ -532,15 +544,11 @@ class TestOutOfBandManagement(cloudstackTestCase): self.fail("Management server failed to expire ownership of fenced peer") self.debug("Testing oobm background sync should claim new ownership") - interval = list_configurations( - self.apiclient, - name='outofbandmanagement.sync.interval' - )[0].value bmc = IpmiServerContext().bmc bmc.powerstate = 'on' - self.checkSyncToState('On', interval) + self.checkSyncToState('On', 2) result = self.dbclient.execute("select mgmt_server_id from oobm where port=%d" % (self.getIpmiServerPort())) newOwnerId = result[0][0] @@ -585,8 +593,10 @@ class TestOutOfBandManagement(cloudstackTestCase): try: response = self.issuePowerActionCmd('STATUS') + except Exception: + pass + else: self.fail("Expected an exception to be thrown, failing") - except Exception: pass alerts = Alert.list(self.apiclient, keyword="auth-error", listall=True) -- To stop receiving notification emails like this one, please contact ['"commits@cloudstack.apache.org" <commits@cloudstack.apache.org>'].