This is an automated email from the ASF dual-hosted git repository.
bhaisaab pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/master by this push:
new 98dc4eb CLOUDSTACK-9782: New Background Polling Task Manager (#2218)
98dc4eb is described below
commit 98dc4eb96a3db1bd02f7ea7dab284f0d76b9dfb5
Author: Rohit Yadav <[email protected]>
AuthorDate: Thu Aug 3 11:53:45 2017 +0200
CLOUDSTACK-9782: New Background Polling Task Manager (#2218)
CloudStack has several background polling tasks that are spread across
the codebase, the aim of this work is to provide a single manager to
handle submission, execution and handling of background tasks. With
the framework implemented, existing oobm background task has been
refactored to use this manager.
Signed-off-by: Rohit Yadav <[email protected]>
---
.../ChangeOutOfBandManagementPasswordCmd.java | 2 +-
.../ConfigureOutOfBandManagementCmd.java | 2 +-
.../IssueOutOfBandManagementPowerActionCmd.java | 2 +-
.../OutOfBandManagementService.java | 9 +--
.../cloudstack/poll/BackgroundPollManager.java | 23 +++----
.../apache/cloudstack/poll/BackgroundPollTask.java | 14 +---
developer/developer-prefill.sql | 4 --
.../agent/manager/ClusteredAgentManagerImpl.java | 5 +-
.../dao/OutOfBandManagementDao.java | 2 +-
.../dao/OutOfBandManagementDaoImpl.java | 2 +-
.../core/spring-server-core-managers-context.xml | 4 ++
server/src/com/cloud/server/StatsCollector.java | 54 ++-------------
.../OutOfBandManagementServiceImpl.java | 61 ++++++++++++++---
...BackgroundTask.java => PowerOperationTask.java} | 8 +--
.../cloudstack/poll/BackgroundPollManagerImpl.java | 79 ++++++++++++++++++++++
.../poll/BackgroundPollManagerImplTest.java | 79 ++++++++++++++++++++++
setup/dev/advanced.cfg | 13 ++--
test/integration/smoke/test_outofbandmanagement.py | 72 +++++++++++---------
18 files changed, 296 insertions(+), 139 deletions(-)
diff --git
a/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ChangeOutOfBandManagementPasswordCmd.java
b/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ChangeOutOfBandManagementPasswordCmd.java
index ea2e3cd..0499a6e 100644
---
a/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ChangeOutOfBandManagementPasswordCmd.java
+++
b/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ChangeOutOfBandManagementPasswordCmd.java
@@ -74,7 +74,7 @@ public class ChangeOutOfBandManagementPasswordCmd extends
BaseAsyncCmd {
CallContext.current().setEventDetails("Host Id: " + host.getId() + "
Password: " + getPassword().charAt(0) + "****");
CallContext.current().putContextParameter(Host.class, host.getUuid());
- final OutOfBandManagementResponse response =
outOfBandManagementService.changeOutOfBandManagementPassword(host,
getPassword());
+ final OutOfBandManagementResponse response =
outOfBandManagementService.changePassword(host, getPassword());
response.setResponseName(getCommandName());
setResponseObject(response);
}
diff --git
a/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ConfigureOutOfBandManagementCmd.java
b/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ConfigureOutOfBandManagementCmd.java
index db22410..d9c6c9f 100644
---
a/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ConfigureOutOfBandManagementCmd.java
+++
b/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ConfigureOutOfBandManagementCmd.java
@@ -83,7 +83,7 @@ public class ConfigureOutOfBandManagementCmd extends BaseCmd {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to
find host by ID: " + getHostId());
}
CallContext.current().putContextParameter(Host.class, host.getUuid());
- final OutOfBandManagementResponse response =
outOfBandManagementService.configureOutOfBandManagement(host,
getHostPMOptions());
+ final OutOfBandManagementResponse response =
outOfBandManagementService.configure(host, getHostPMOptions());
response.setId(host.getUuid());
response.setResponseName(getCommandName());
setResponseObject(response);
diff --git
a/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/IssueOutOfBandManagementPowerActionCmd.java
b/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/IssueOutOfBandManagementPowerActionCmd.java
index 8d6bdd3..7593a3c 100644
---
a/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/IssueOutOfBandManagementPowerActionCmd.java
+++
b/api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/IssueOutOfBandManagementPowerActionCmd.java
@@ -80,7 +80,7 @@ public class IssueOutOfBandManagementPowerActionCmd extends
BaseAsyncCmd {
CallContext.current().setEventDetails("Host Id: " + host.getId() + "
Action: " + powerOperation.toString());
CallContext.current().putContextParameter(Host.class, host.getUuid());
- final OutOfBandManagementResponse response =
outOfBandManagementService.executeOutOfBandManagementPowerOperation(host,
powerOperation, getActionTimeout());
+ final OutOfBandManagementResponse response =
outOfBandManagementService.executePowerOperation(host, powerOperation,
getActionTimeout());
response.setResponseName(getCommandName());
setResponseObject(response);
}
diff --git
a/api/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementService.java
b/api/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementService.java
index 699b2c6..a8b73b7 100644
---
a/api/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementService.java
+++
b/api/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementService.java
@@ -30,9 +30,6 @@ public interface OutOfBandManagementService {
ConfigKey<Long> ActionTimeout = new ConfigKey<Long>("Advanced",
Long.class, "outofbandmanagement.action.timeout", "60",
"The out of band management action timeout in seconds,
configurable by cluster", true, ConfigKey.Scope.Cluster);
- ConfigKey<Long> SyncThreadInterval = new ConfigKey<Long>("Advanced",
Long.class, "outofbandmanagement.sync.interval", "300000",
- "The interval (in milliseconds) when the out-of-band management
background sync are retrieved", true, ConfigKey.Scope.Global);
-
ConfigKey<Integer> SyncThreadPoolSize = new ConfigKey<Integer>("Advanced",
Integer.class, "outofbandmanagement.sync.poolsize", "50",
"The out of band management background sync thread pool size",
true, ConfigKey.Scope.Global);
@@ -49,7 +46,7 @@ public interface OutOfBandManagementService {
OutOfBandManagementResponse disableOutOfBandManagement(Cluster cluster);
OutOfBandManagementResponse disableOutOfBandManagement(Host host);
- OutOfBandManagementResponse configureOutOfBandManagement(Host host,
ImmutableMap<OutOfBandManagement.Option, String> options);
- OutOfBandManagementResponse executeOutOfBandManagementPowerOperation(Host
host, OutOfBandManagement.PowerOperation operation, Long timeout);
- OutOfBandManagementResponse changeOutOfBandManagementPassword(Host host,
String password);
+ OutOfBandManagementResponse configure(Host host,
ImmutableMap<OutOfBandManagement.Option, String> options);
+ OutOfBandManagementResponse executePowerOperation(Host host,
OutOfBandManagement.PowerOperation operation, Long timeout);
+ OutOfBandManagementResponse changePassword(Host host, String password);
}
diff --git
a/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java
b/api/src/org/apache/cloudstack/poll/BackgroundPollManager.java
similarity index 54%
copy from
engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java
copy to api/src/org/apache/cloudstack/poll/BackgroundPollManager.java
index 5985b81..b9d5500 100644
---
a/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java
+++ b/api/src/org/apache/cloudstack/poll/BackgroundPollManager.java
@@ -15,17 +15,16 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.cloudstack.outofbandmanagement.dao;
+package org.apache.cloudstack.poll;
-import com.cloud.utils.db.GenericDao;
-import com.cloud.utils.fsm.StateDao;
-import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
-import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementVO;
-
-import java.util.List;
-
-public interface OutOfBandManagementDao extends
GenericDao<OutOfBandManagementVO, Long>,
StateDao<OutOfBandManagement.PowerState, OutOfBandManagement.PowerState.Event,
OutOfBandManagement> {
- OutOfBandManagement findByHost(long hostId);
- List<OutOfBandManagementVO> findAllByManagementServer(long serverId);
- void expireOutOfBandManagementOwnershipByServer(long serverId);
+public interface BackgroundPollManager {
+ /**
+ * Submits a background poll task that need to run continuously in the
background
+ * to poll external resources, update states, trigger actions etc.
+ * Tasks must be submitted by a manager in configure-phase, the list of
submitted tasks
+ * are then submitted to the internal executor service during start-phase.
+ * @param task periodic background task
+ * @since 4.11
+ */
+ void submitTask(final BackgroundPollTask task);
}
diff --git
a/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java
b/api/src/org/apache/cloudstack/poll/BackgroundPollTask.java
similarity index 54%
copy from
engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java
copy to api/src/org/apache/cloudstack/poll/BackgroundPollTask.java
index 5985b81..8eea147 100644
---
a/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java
+++ b/api/src/org/apache/cloudstack/poll/BackgroundPollTask.java
@@ -15,17 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.cloudstack.outofbandmanagement.dao;
+package org.apache.cloudstack.poll;
-import com.cloud.utils.db.GenericDao;
-import com.cloud.utils.fsm.StateDao;
-import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
-import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementVO;
-
-import java.util.List;
-
-public interface OutOfBandManagementDao extends
GenericDao<OutOfBandManagementVO, Long>,
StateDao<OutOfBandManagement.PowerState, OutOfBandManagement.PowerState.Event,
OutOfBandManagement> {
- OutOfBandManagement findByHost(long hostId);
- List<OutOfBandManagementVO> findAllByManagementServer(long serverId);
- void expireOutOfBandManagementOwnershipByServer(long serverId);
+public interface BackgroundPollTask extends Runnable {
}
diff --git a/developer/developer-prefill.sql b/developer/developer-prefill.sql
index 3fd3a1b..7fd85c6 100644
--- a/developer/developer-prefill.sql
+++ b/developer/developer-prefill.sql
@@ -114,10 +114,6 @@ INSERT INTO `cloud`.`configuration` (category, instance,
component, name, value)
VALUES ('Advanced', 'DEFAULT', 'management-server',
'ping.timeout', '1.5');
-INSERT INTO `cloud`.`configuration` (category, instance, component, name,
value)
- VALUES ('Advanced', 'DEFAULT', 'management-server',
- 'outofbandmanagement.sync.interval', '1000');
-
-- Enable dynamic RBAC by default for fresh deployments
INSERT INTO `cloud`.`configuration` (category, instance, component, name,
value)
VALUES ('Advanced', 'DEFAULT', 'RoleService',
diff --git
a/engine/orchestration/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java
b/engine/orchestration/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java
index 9239adc..a89988c 100644
---
a/engine/orchestration/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java
+++
b/engine/orchestration/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java
@@ -49,6 +49,7 @@ import
org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.managed.context.ManagedContextTimerTask;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.cloudstack.utils.security.SSLUtils;
+import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
import org.apache.log4j.Logger;
import com.cloud.agent.api.Answer;
@@ -120,6 +121,8 @@ public class ClusteredAgentManagerImpl extends
AgentManagerImpl implements Clust
ConfigurationDao _configDao;
@Inject
ConfigDepot _configDepot;
+ @Inject
+ private OutOfBandManagementDao outOfBandManagementDao;
protected ClusteredAgentManagerImpl() {
super();
@@ -736,7 +739,7 @@ public class ClusteredAgentManagerImpl extends
AgentManagerImpl implements Clust
s_logger.info("Marking hosts as disconnected on Management server"
+ vo.getMsid());
final long lastPing = (System.currentTimeMillis() >> 10) -
getTimeout();
_hostDao.markHostsAsDisconnected(vo.getMsid(), lastPing);
-
outOfBandManagementDao.expireOutOfBandManagementOwnershipByServer(vo.getMsid());
+ outOfBandManagementDao.expireServerOwnership(vo.getMsid());
s_logger.info("Deleting entries from op_host_transfer table for
Management server " + vo.getMsid());
cleanupTransferMap(vo.getMsid());
}
diff --git
a/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java
b/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java
index 5985b81..8a4ee36 100644
---
a/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java
+++
b/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java
@@ -27,5 +27,5 @@ import java.util.List;
public interface OutOfBandManagementDao extends
GenericDao<OutOfBandManagementVO, Long>,
StateDao<OutOfBandManagement.PowerState, OutOfBandManagement.PowerState.Event,
OutOfBandManagement> {
OutOfBandManagement findByHost(long hostId);
List<OutOfBandManagementVO> findAllByManagementServer(long serverId);
- void expireOutOfBandManagementOwnershipByServer(long serverId);
+ void expireServerOwnership(long serverId);
}
diff --git
a/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDaoImpl.java
b/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDaoImpl.java
index b914b6b..3cdd28f 100644
---
a/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDaoImpl.java
+++
b/engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDaoImpl.java
@@ -110,7 +110,7 @@ public class OutOfBandManagementDaoImpl extends
GenericDaoBase<OutOfBandManageme
}
@Override
- public void expireOutOfBandManagementOwnershipByServer(long serverId) {
+ public void expireServerOwnership(long serverId) {
final String resetOwnerSql = "UPDATE oobm set mgmt_server_id=NULL,
power_state=NULL where mgmt_server_id=?";
executeExpireOwnershipSql(resetOwnerSql, serverId);
if (LOG.isDebugEnabled()) {
diff --git
a/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
b/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
index 87097a4..56da591 100644
---
a/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
+++
b/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
@@ -70,6 +70,10 @@
value="#{resourceDiscoverersRegistry.registered}" />
</bean>
+ <!-- the new background poll manager -->
+ <bean id="bgPollManager"
class="org.apache.cloudstack.poll.BackgroundPollManagerImpl">
+ </bean>
+
<bean id="highAvailabilityManagerExtImpl"
class="com.cloud.ha.HighAvailabilityManagerExtImpl">
<property name="investigators"
value="#{haInvestigatorsRegistry.registered}" />
<property name="fenceBuilders"
value="#{haFenceBuildersRegistry.registered}" />
diff --git a/server/src/com/cloud/server/StatsCollector.java
b/server/src/com/cloud/server/StatsCollector.java
index 9725217..048b3b1 100644
--- a/server/src/com/cloud/server/StatsCollector.java
+++ b/server/src/com/cloud/server/StatsCollector.java
@@ -20,7 +20,6 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Calendar;
-import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@@ -34,15 +33,6 @@ import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
-import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
-import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService;
-import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementVO;
-import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
-import org.apache.cloudstack.utils.identity.ManagementServerNode;
-import org.apache.cloudstack.utils.usage.UsageUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
@@ -56,6 +46,10 @@ import
org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.utils.graphite.GraphiteClient;
import org.apache.cloudstack.utils.graphite.GraphiteException;
+import org.apache.cloudstack.utils.usage.UsageUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
@@ -177,8 +171,6 @@ public class StatsCollector extends ManagerBase implements
ComponentMethodInterc
@Inject
private HostDao _hostDao;
@Inject
- private OutOfBandManagementDao outOfBandManagementDao;
- @Inject
private UserVmDao _userVmDao;
@Inject
private VolumeDao _volsDao;
@@ -195,8 +187,6 @@ public class StatsCollector extends ManagerBase implements
ComponentMethodInterc
@Inject
private ResourceManager _resourceMgr;
@Inject
- private OutOfBandManagementService outOfBandManagementService;
- @Inject
private ConfigurationDao _configDao;
@Inject
private EndPointSelector _epSelector;
@@ -244,7 +234,6 @@ public class StatsCollector extends ManagerBase implements
ComponentMethodInterc
private ConcurrentHashMap<Long, StorageStats> _storagePoolStats = new
ConcurrentHashMap<Long, StorageStats>();
long hostStatsInterval = -1L;
- long hostOutOfBandManagementStatsInterval = -1L;
long hostAndVmStatsInterval = -1L;
long storageStatsInterval = -1L;
long volumeStatsInterval = -1L;
@@ -290,7 +279,6 @@ public class StatsCollector extends ManagerBase implements
ComponentMethodInterc
private void init(Map<String, String> configs) {
_executor = Executors.newScheduledThreadPool(6, new
NamedThreadFactory("StatsCollector"));
- hostOutOfBandManagementStatsInterval =
OutOfBandManagementService.SyncThreadInterval.value();
hostStatsInterval =
NumbersUtil.parseLong(configs.get("host.stats.interval"), 60000L);
hostAndVmStatsInterval =
NumbersUtil.parseLong(configs.get("vm.stats.interval"), 60000L);
storageStatsInterval =
NumbersUtil.parseLong(configs.get("storage.stats.interval"), 60000L);
@@ -337,10 +325,6 @@ public class StatsCollector extends ManagerBase implements
ComponentMethodInterc
_executor.scheduleWithFixedDelay(new HostCollector(), 15000L,
hostStatsInterval, TimeUnit.MILLISECONDS);
}
- if (hostOutOfBandManagementStatsInterval > 0) {
- _executor.scheduleWithFixedDelay(new
HostOutOfBandManagementStatsCollector(), 15000L,
hostOutOfBandManagementStatsInterval, TimeUnit.MILLISECONDS);
- }
-
if (hostAndVmStatsInterval > 0) {
_executor.scheduleWithFixedDelay(new VmStatsCollector(), 15000L,
hostAndVmStatsInterval, TimeUnit.MILLISECONDS);
}
@@ -477,36 +461,6 @@ public class StatsCollector extends ManagerBase implements
ComponentMethodInterc
}
}
- class HostOutOfBandManagementStatsCollector extends ManagedContextRunnable
{
- @Override
- protected void runInContext() {
- try {
- s_logger.trace("HostOutOfBandManagementStatsCollector is
running...");
- List<OutOfBandManagementVO> outOfBandManagementHosts =
outOfBandManagementDao.findAllByManagementServer(ManagementServerNode.getManagementServerId());
- if (outOfBandManagementHosts == null) {
- return;
- }
- for (OutOfBandManagement outOfBandManagementHost :
outOfBandManagementHosts) {
- Host host =
_hostDao.findById(outOfBandManagementHost.getHostId());
- if (host == null) {
- continue;
- }
- if
(outOfBandManagementService.isOutOfBandManagementEnabled(host)) {
-
outOfBandManagementService.submitBackgroundPowerSyncTask(host);
- } else if (outOfBandManagementHost.getPowerState() !=
OutOfBandManagement.PowerState.Disabled) {
- if
(outOfBandManagementService.transitionPowerStateToDisabled(Collections.singletonList(host)))
{
- if (s_logger.isDebugEnabled()) {
- s_logger.debug("Out-of-band management was
disabled in zone/cluster/host, disabled power state for host id:" +
host.getId());
- }
- }
- }
- }
- } catch (Throwable t) {
- s_logger.error("Error trying to retrieve host out-of-band
management stats", t);
- }
- }
- }
-
class VmStatsCollector extends ManagedContextRunnable {
@Override
protected void runInContext() {
diff --git
a/server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java
b/server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java
index ca65ef7..4f0ac74 100644
---
a/server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java
+++
b/server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java
@@ -44,10 +44,13 @@ import
org.apache.cloudstack.api.response.OutOfBandManagementResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
+import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
import
org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverChangePasswordCommand;
import
org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverPowerCommand;
import
org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverResponse;
+import org.apache.cloudstack.poll.BackgroundPollManager;
+import org.apache.cloudstack.poll.BackgroundPollTask;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
@@ -78,6 +81,8 @@ public class OutOfBandManagementServiceImpl extends
ManagerBase implements OutOf
private HostDao hostDao;
@Inject
private AlertManager alertMgr;
+ @Inject
+ private BackgroundPollManager backgroundPollManager;
private String name;
private long serviceId;
@@ -202,12 +207,14 @@ public class OutOfBandManagementServiceImpl extends
ManagerBase implements OutOf
OutOfBandManagement.PowerState currentPowerState =
outOfBandManagementHost.getPowerState();
try {
OutOfBandManagement.PowerState newPowerState =
OutOfBandManagement.PowerState.getStateMachine().getNextState(currentPowerState,
event);
- boolean result =
outOfBandManagementDao.updateState(currentPowerState, event, newPowerState,
outOfBandManagementHost, null);
+ boolean result =
OutOfBandManagement.PowerState.getStateMachine().transitTo(outOfBandManagementHost,
event, null, outOfBandManagementDao);
if (result) {
final String message = String.format("Transitioned out-of-band
management power state from:%s to:%s due to event:%s for the host id:%d",
currentPowerState, newPowerState, event, outOfBandManagementHost.getHostId());
LOG.debug(message);
-
ActionEventUtils.onActionEvent(CallContext.current().getCallingUserId(),
CallContext.current().getCallingAccountId(), Domain.ROOT_DOMAIN,
-
EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_POWERSTATE_TRANSITION, message);
+ if (newPowerState == OutOfBandManagement.PowerState.Unknown) {
+
ActionEventUtils.onActionEvent(CallContext.current().getCallingUserId(),
CallContext.current().getCallingAccountId(), Domain.ROOT_DOMAIN,
+
EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_POWERSTATE_TRANSITION, message);
+ }
}
return result;
} catch (NoTransitionException ignored) {
@@ -278,7 +285,7 @@ public class OutOfBandManagementServiceImpl extends
ManagerBase implements OutOf
public void submitBackgroundPowerSyncTask(final Host host) {
if (host != null) {
- backgroundSyncBlockingExecutor.submit(new
OutOfBandManagementBackgroundTask(this, host,
OutOfBandManagement.PowerOperation.STATUS));
+ backgroundSyncBlockingExecutor.submit(new PowerOperationTask(this,
host, OutOfBandManagement.PowerOperation.STATUS));
}
}
@@ -356,7 +363,7 @@ public class OutOfBandManagementServiceImpl extends
ManagerBase implements OutOf
@Override
@ActionEvent(eventType =
EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_CONFIGURE, eventDescription =
"updating out-of-band management configuration")
- public OutOfBandManagementResponse configureOutOfBandManagement(final Host
host, final ImmutableMap<OutOfBandManagement.Option, String> options) {
+ public OutOfBandManagementResponse configure(final Host host, final
ImmutableMap<OutOfBandManagement.Option, String> options) {
OutOfBandManagement outOfBandManagementConfig =
outOfBandManagementDao.findByHost(host.getId());
if (outOfBandManagementConfig == null) {
outOfBandManagementConfig = outOfBandManagementDao.persist(new
OutOfBandManagementVO(host.getId()));
@@ -384,7 +391,7 @@ public class OutOfBandManagementServiceImpl extends
ManagerBase implements OutOf
@Override
@ActionEvent(eventType =
EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_ACTION, eventDescription = "issuing
host out-of-band management action", async = true)
- public OutOfBandManagementResponse
executeOutOfBandManagementPowerOperation(final Host host, final
OutOfBandManagement.PowerOperation powerOperation, final Long timeout) {
+ public OutOfBandManagementResponse executePowerOperation(final Host host,
final OutOfBandManagement.PowerOperation powerOperation, final Long timeout) {
checkOutOfBandManagementEnabledByZoneClusterHost(host);
final OutOfBandManagement outOfBandManagementConfig =
getConfigForHost(host);
final ImmutableMap<OutOfBandManagement.Option, String> options =
getOptions(outOfBandManagementConfig);
@@ -428,7 +435,7 @@ public class OutOfBandManagementServiceImpl extends
ManagerBase implements OutOf
@Override
@ActionEvent(eventType =
EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_CHANGE_PASSWORD, eventDescription =
"updating out-of-band management password")
- public OutOfBandManagementResponse changeOutOfBandManagementPassword(final
Host host, final String newPassword) {
+ public OutOfBandManagementResponse changePassword(final Host host, final
String newPassword) {
checkOutOfBandManagementEnabledByZoneClusterHost(host);
if (Strings.isNullOrEmpty(newPassword)) {
throw new CloudRuntimeException(String.format("Cannot change
out-of-band management password as provided new-password is null or empty for
the host %s.", host.getUuid()));
@@ -503,7 +510,9 @@ public class OutOfBandManagementServiceImpl extends
ManagerBase implements OutOf
0L, TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<Runnable>(10 * poolSize, true), new
ThreadPoolExecutor.CallerRunsPolicy());
- LOG.info("Starting out-of-band management background sync executor
with thread pool-size=" + poolSize + " and background sync thread interval=" +
SyncThreadInterval.value() + "s");
+ backgroundPollManager.submitTask(new
OutOfBandManagementPowerStatePollTask());
+
+ LOG.info("Starting out-of-band management background sync executor
with thread pool-size=" + poolSize);
return true;
}
@@ -516,7 +525,7 @@ public class OutOfBandManagementServiceImpl extends
ManagerBase implements OutOf
@Override
public boolean stop() {
backgroundSyncBlockingExecutor.shutdown();
-
outOfBandManagementDao.expireOutOfBandManagementOwnershipByServer(getId());
+ outOfBandManagementDao.expireServerOwnership(getId());
return true;
}
@@ -527,7 +536,7 @@ public class OutOfBandManagementServiceImpl extends
ManagerBase implements OutOf
@Override
public ConfigKey<?>[] getConfigKeys() {
- return new ConfigKey<?>[] {ActionTimeout, SyncThreadInterval,
SyncThreadPoolSize};
+ return new ConfigKey<?>[] {ActionTimeout, SyncThreadPoolSize};
}
public List<OutOfBandManagementDriver> getOutOfBandManagementDrivers() {
@@ -537,4 +546,36 @@ public class OutOfBandManagementServiceImpl extends
ManagerBase implements OutOf
public void setOutOfBandManagementDrivers(List<OutOfBandManagementDriver>
outOfBandManagementDrivers) {
this.outOfBandManagementDrivers = outOfBandManagementDrivers;
}
+
+ private final class OutOfBandManagementPowerStatePollTask extends
ManagedContextRunnable implements BackgroundPollTask {
+ @Override
+ protected void runInContext() {
+ try {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Host out-of-band management power state poll
task is running...");
+ }
+ final List<OutOfBandManagementVO> outOfBandManagementHosts =
outOfBandManagementDao.findAllByManagementServer(ManagementServerNode.getManagementServerId());
+ if (outOfBandManagementHosts == null ||
outOfBandManagementHosts.isEmpty()) {
+ return;
+ }
+ for (final OutOfBandManagement outOfBandManagementHost :
outOfBandManagementHosts) {
+ final Host host =
hostDao.findById(outOfBandManagementHost.getHostId());
+ if (host == null) {
+ continue;
+ }
+ if (isOutOfBandManagementEnabled(host)) {
+ submitBackgroundPowerSyncTask(host);
+ } else if (outOfBandManagementHost.getPowerState() !=
OutOfBandManagement.PowerState.Disabled) {
+ if
(transitionPowerStateToDisabled(Collections.singletonList(host))) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Out-of-band management was disabled
in zone/cluster/host, disabled power state for host id:" + host.getId());
+ }
+ }
+ }
+ }
+ } catch (Throwable t) {
+ LOG.error("Error trying to retrieve host out-of-band
management stats", t);
+ }
+ }
+ }
}
diff --git
a/server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementBackgroundTask.java
b/server/src/org/apache/cloudstack/outofbandmanagement/PowerOperationTask.java
similarity index 80%
rename from
server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementBackgroundTask.java
rename to
server/src/org/apache/cloudstack/outofbandmanagement/PowerOperationTask.java
index 38e139e..9c346d7 100644
---
a/server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementBackgroundTask.java
+++
b/server/src/org/apache/cloudstack/outofbandmanagement/PowerOperationTask.java
@@ -20,14 +20,14 @@ package org.apache.cloudstack.outofbandmanagement;
import com.cloud.host.Host;
import org.apache.log4j.Logger;
-public class OutOfBandManagementBackgroundTask implements Runnable {
- public static final Logger LOG =
Logger.getLogger(OutOfBandManagementBackgroundTask.class);
+public class PowerOperationTask implements Runnable {
+ public static final Logger LOG =
Logger.getLogger(PowerOperationTask.class);
final private OutOfBandManagementService service;
final private Host host;
final private OutOfBandManagement.PowerOperation powerOperation;
- public OutOfBandManagementBackgroundTask(OutOfBandManagementService
service, Host host, OutOfBandManagement.PowerOperation powerOperation) {
+ public PowerOperationTask(OutOfBandManagementService service, Host host,
OutOfBandManagement.PowerOperation powerOperation) {
this.service = service;
this.host = host;
this.powerOperation = powerOperation;
@@ -41,7 +41,7 @@ public class OutOfBandManagementBackgroundTask implements
Runnable {
@Override
public void run() {
try {
- service.executeOutOfBandManagementPowerOperation(host,
powerOperation, null);
+ service.executePowerOperation(host, powerOperation, null);
} catch (Exception e) {
LOG.warn(String.format("Out-of-band management background task
operation=%s for host id=%d failed with: %s",
powerOperation.name(), host.getId(), e.getMessage()));
diff --git
a/server/src/org/apache/cloudstack/poll/BackgroundPollManagerImpl.java
b/server/src/org/apache/cloudstack/poll/BackgroundPollManagerImpl.java
new file mode 100644
index 0000000..c0a7f1c
--- /dev/null
+++ b/server/src/org/apache/cloudstack/poll/BackgroundPollManagerImpl.java
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.poll;
+
+import com.cloud.utils.component.Manager;
+import com.cloud.utils.component.ManagerBase;
+import com.cloud.utils.concurrency.NamedThreadFactory;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.google.common.base.Preconditions;
+import org.apache.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public final class BackgroundPollManagerImpl extends ManagerBase implements
BackgroundPollManager, Manager {
+ public static final Logger LOG =
Logger.getLogger(BackgroundPollManagerImpl.class);
+
+ private ScheduledExecutorService backgroundPollTaskScheduler;
+ private List<BackgroundPollTask> submittedTasks = new ArrayList<>();
+ private volatile boolean isConfiguredAndStarted = false;
+
+ public long getInitialDelay() {
+ return 5000L;
+ }
+
+ public long getRoundDelay() {
+ return 4000L;
+ }
+
+ @Override
+ public boolean start() {
+ if (isConfiguredAndStarted) {
+ return true;
+ }
+ backgroundPollTaskScheduler =
Executors.newScheduledThreadPool(submittedTasks.size() + 1, new
NamedThreadFactory("BackgroundTaskPollManager"));
+ for (final BackgroundPollTask task : submittedTasks) {
+ backgroundPollTaskScheduler.scheduleWithFixedDelay(task,
getInitialDelay(), getRoundDelay(), TimeUnit.MILLISECONDS);
+ LOG.debug("Scheduled background poll task: " +
task.getClass().getName());
+ }
+ isConfiguredAndStarted = true;
+ return true;
+ }
+
+ @Override
+ public boolean stop() {
+ if (isConfiguredAndStarted) {
+ backgroundPollTaskScheduler.shutdown();
+ }
+ return true;
+ }
+
+ @Override
+ public void submitTask(final BackgroundPollTask task) {
+ Preconditions.checkNotNull(task);
+ if (isConfiguredAndStarted) {
+ throw new CloudRuntimeException("Background Poll Manager cannot
accept poll task as it has been configured and started.");
+ }
+ LOG.debug("Background Poll Manager received task: " +
task.getClass().getSimpleName());
+ submittedTasks.add(task);
+ }
+}
diff --git
a/server/test/org/apache/cloudstack/poll/BackgroundPollManagerImplTest.java
b/server/test/org/apache/cloudstack/poll/BackgroundPollManagerImplTest.java
new file mode 100644
index 0000000..3304aba
--- /dev/null
+++ b/server/test/org/apache/cloudstack/poll/BackgroundPollManagerImplTest.java
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.poll;
+
+import org.apache.cloudstack.managed.context.ManagedContextRunnable;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class BackgroundPollManagerImplTest {
+
+ private BackgroundPollManagerImpl pollManager;
+ private DummyPollTask pollTask;
+
+ private class DummyPollTask extends ManagedContextRunnable implements
BackgroundPollTask {
+ private boolean didIRun = false;
+ private long counter = 0;
+
+ public boolean didItRan() {
+ return didIRun;
+ }
+
+ public long getCounter() {
+ return counter;
+ }
+
+ @Override
+ protected void runInContext() {
+ didIRun = true;
+ counter++;
+ }
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ pollManager = new BackgroundPollManagerImpl();
+ pollTask = new DummyPollTask();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ pollManager.stop();
+ }
+
+ @Test
+ public void testSubmitValidTask() throws Exception {
+ Assert.assertFalse(pollTask.didItRan());
+ Assert.assertTrue(pollTask.getCounter() == 0);
+
+ pollManager.submitTask(pollTask);
+ pollManager.start();
+ Thread.sleep(pollManager.getInitialDelay()*2);
+
+ Assert.assertTrue(pollTask.didItRan());
+ Assert.assertTrue(pollTask.getCounter() > 0);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testSubmitNullTask() throws Exception {
+ pollManager.submitTask(null);
+ }
+
+}
\ No newline at end of file
diff --git a/setup/dev/advanced.cfg b/setup/dev/advanced.cfg
index 0742f24..bdb25e5 100644
--- a/setup/dev/advanced.cfg
+++ b/setup/dev/advanced.cfg
@@ -113,6 +113,11 @@
"username": "root",
"url": "http://sim/c1/h0";,
"password": "password"
+ },
+ {
+ "username": "root",
+ "url": "http://sim/c1/h1";,
+ "password": "password"
}
],
"clustertype": "CloudManaged",
@@ -120,6 +125,10 @@
{
"url":
"nfs://10.147.28.6:/export/home/sandbox/primary2",
"name": "PS2"
+ },
+ {
+ "url":
"nfs://10.147.28.6:/export/home/sandbox/primary3",
+ "name": "PS2"
}
]
}
@@ -223,10 +232,6 @@
{
"name": "ping.timeout",
"value": "1.5"
- },
- {
- "name": "outofbandmanagement.sync.interval",
- "value": "1000"
}
],
"mgtSvr": [
diff --git a/test/integration/smoke/test_outofbandmanagement.py
b/test/integration/smoke/test_outofbandmanagement.py
index e91d49a..a52e3b1 100644
--- a/test/integration/smoke/test_outofbandmanagement.py
+++ b/test/integration/smoke/test_outofbandmanagement.py
@@ -211,8 +211,10 @@ class TestOutOfBandManagement(cloudstackTestCase):
cmd.driver = 'randomDriverThatDoesNotExist'
try:
response = self.apiclient.configureOutOfBandManagement(cmd)
+ except Exception:
+ pass
+ else:
self.fail("Expected an exception to be thrown, failing")
- except Exception: pass
@attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic",
"sg"], required_hardware="false")
@@ -239,20 +241,26 @@ class TestOutOfBandManagement(cloudstackTestCase):
cmd.hostid = -1
try:
response = self.apiclient.enableOutOfBandManagementForHost(cmd)
+ except Exception:
+ pass
+ else:
self.fail("Expected an exception to be thrown, failing")
- except Exception: pass
try:
cmd =
enableOutOfBandManagementForCluster.enableOutOfBandManagementForClusterCmd()
response = self.apiclient.enableOutOfBandManagementForCluster(cmd)
+ except Exception:
+ pass
+ else:
self.fail("Expected an exception to be thrown, failing")
- except Exception: pass
try:
cmd =
enableOutOfBandManagementForZone.enableOutOfBandManagementForZoneCmd()
response = self.apiclient.enableOutOfBandManagementForZone(cmd)
+ except Exception:
+ pass
+ else:
self.fail("Expected an exception to be thrown, failing")
- except Exception: pass
@attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic",
"sg"], required_hardware="false")
@@ -265,20 +273,26 @@ class TestOutOfBandManagement(cloudstackTestCase):
cmd.hostid = -1
try:
response = self.apiclient.disableOutOfBandManagementForHost(cmd)
+ except Exception:
+ pass
+ else:
self.fail("Expected an exception to be thrown, failing")
- except Exception: pass
try:
cmd =
disableOutOfBandManagementForCluster.disableOutOfBandManagementForClusterCmd()
response = self.apiclient.disableOutOfBandManagementForCluster(cmd)
+ except Exception:
+ pass
+ else:
self.fail("Expected an exception to be thrown, failing")
- except Exception: pass
try:
cmd =
disableOutOfBandManagementForZone.disableOutOfBandManagementForZoneCmd()
response = self.apiclient.disableOutOfBandManagementForZone(cmd)
+ except Exception:
+ pass
+ else:
self.fail("Expected an exception to be thrown, failing")
- except Exception: pass
@attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic",
"sg"], required_hardware="false")
@@ -340,8 +354,10 @@ class TestOutOfBandManagement(cloudstackTestCase):
try:
self.issuePowerActionCmd('STATUS')
- self.fail("Exception was expected, oobm is disabled at zone level")
- except Exception: pass
+ except Exception:
+ pass
+ else:
+ self.fail("Expected an exception to be thrown, failing")
# Enable at zone level
cmd =
enableOutOfBandManagementForZone.enableOutOfBandManagementForZoneCmd()
@@ -350,18 +366,16 @@ class TestOutOfBandManagement(cloudstackTestCase):
try:
self.issuePowerActionCmd('STATUS')
- self.fail("Exception was expected, oobm is disabled at cluster
level")
- except Exception: pass
+ except Exception:
+ pass
+ else:
+ self.fail("Expected an exception to be thrown, failing")
# Check background thread syncs state to Disabled
response = self.getHost(hostId=host.id).outofbandmanagement
self.assertEqual(response.powerstate, 'Disabled')
self.dbclient.execute("update oobm set power_state='On' where port=%d"
% self.getIpmiServerPort())
- interval = list_configurations(
- self.apiclient,
- name='outofbandmanagement.sync.interval'
- )[0].value
- self.checkSyncToState('Disabled', interval)
+ self.checkSyncToState('Disabled', 2)
# Enable at cluster level
cmd =
enableOutOfBandManagementForCluster.enableOutOfBandManagementForClusterCmd()
@@ -370,8 +384,10 @@ class TestOutOfBandManagement(cloudstackTestCase):
try:
self.issuePowerActionCmd('STATUS')
- self.fail("Exception was expected, oobm is disabled at host level")
- except Exception: pass
+ except Exception:
+ pass
+ else:
+ self.fail("Expected an exception to be thrown, failing")
# Enable at host level
cmd =
enableOutOfBandManagementForHost.enableOutOfBandManagementForHostCmd()
@@ -452,25 +468,21 @@ class TestOutOfBandManagement(cloudstackTestCase):
Tests out-of-band management background powerstate sync
"""
self.debug("Testing oobm background sync")
- interval = list_configurations(
- self.apiclient,
- name='outofbandmanagement.sync.interval'
- )[0].value
self.configureAndEnableOobm()
bmc = IpmiServerContext().bmc
bmc.powerstate = 'on'
- self.checkSyncToState('On', interval)
+ self.checkSyncToState('On', 2)
bmc.powerstate = 'off'
- self.checkSyncToState('Off', interval)
+ self.checkSyncToState('Off', 2)
# Check for unknown state (ipmi server not reachable)
cmd = self.getOobmConfigCmd()
cmd.port = 1
response = self.apiclient.configureOutOfBandManagement(cmd)
- self.checkSyncToState('Unknown', interval)
+ self.checkSyncToState('Unknown', 2)
@attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic",
"sg"], required_hardware="false")
@@ -532,15 +544,11 @@ class TestOutOfBandManagement(cloudstackTestCase):
self.fail("Management server failed to expire ownership of fenced
peer")
self.debug("Testing oobm background sync should claim new ownership")
- interval = list_configurations(
- self.apiclient,
- name='outofbandmanagement.sync.interval'
- )[0].value
bmc = IpmiServerContext().bmc
bmc.powerstate = 'on'
- self.checkSyncToState('On', interval)
+ self.checkSyncToState('On', 2)
result = self.dbclient.execute("select mgmt_server_id from oobm where
port=%d" % (self.getIpmiServerPort()))
newOwnerId = result[0][0]
@@ -585,8 +593,10 @@ class TestOutOfBandManagement(cloudstackTestCase):
try:
response = self.issuePowerActionCmd('STATUS')
+ except Exception:
+ pass
+ else:
self.fail("Expected an exception to be thrown, failing")
- except Exception: pass
alerts = Alert.list(self.apiclient, keyword="auth-error",
listall=True)
--
To stop receiving notification emails like this one, please contact
['"[email protected]" <[email protected]>'].
