Martin Peřina has uploaded a new change for review. Change subject: core: Introduce ConcurrentAgentsFenceActionExecutor ......................................................................
core: Introduce ConcurrentAgentsFenceActionExecutor Introduces ConcurrentAgentsFenceActionExecutor which is responsible to execute fence action on multiple fence agents concurrently. It uses SingleAgentFenceExecutor to execute action on each agent. Change-Id: I42e7fc0b530a341e9137c4d1efda6ae3f7ab2752 Bug-Url: https://bugzilla.redhat.com/1182510 Signed-off-by: Martin Perina <[email protected]> --- A backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/pm/ConcurrentAgentsFenceActionExecutor.java A backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/pm/ConcurrentAgentsFenceActionExecutorTest.java 2 files changed, 581 insertions(+), 0 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/64/38964/1 diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/pm/ConcurrentAgentsFenceActionExecutor.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/pm/ConcurrentAgentsFenceActionExecutor.java new file mode 100644 index 0000000..fc2c72be --- /dev/null +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/pm/ConcurrentAgentsFenceActionExecutor.java @@ -0,0 +1,308 @@ +package org.ovirt.engine.core.bll.pm; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.Future; + +import org.ovirt.engine.core.common.businessentities.FenceAgent; +import org.ovirt.engine.core.common.businessentities.FencingPolicy; +import org.ovirt.engine.core.common.businessentities.VDS; +import org.ovirt.engine.core.common.businessentities.pm.FenceActionType; +import org.ovirt.engine.core.common.businessentities.pm.FenceOperationResult; +import org.ovirt.engine.core.common.businessentities.pm.FenceOperationResult.Status; +import org.ovirt.engine.core.common.businessentities.pm.HostPowerStatus; +import org.ovirt.engine.core.utils.threadpool.ThreadPoolUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages fence execution for the host using multiple concurrent fence agents + */ +public class ConcurrentAgentsFenceActionExecutor implements FenceActionExecutor { + private static final Logger log = LoggerFactory.getLogger(ConcurrentAgentsFenceActionExecutor.class); + + private final VDS fencedHost; + private final List<FenceAgent> fenceAgents; + private final FencingPolicy fencingPolicy; + + /** + * Processor which manages action execution + */ + protected BaseTaskProcessor tasksProcessor; + + public ConcurrentAgentsFenceActionExecutor( + VDS fencedHost, + List<FenceAgent> fenceAgents, + FencingPolicy fencingPolicy + ) { + this.fencedHost = fencedHost; + this.fenceAgents = fenceAgents; + this.fencingPolicy = fencingPolicy; + } + + @Override + public FenceOperationResult fence(FenceActionType fenceAction) { + setupParams(fenceAction); + return fenceConcurrently(fenceAction); + } + + /** + * Setup parameters for specified fence action + */ + protected void setupParams(FenceActionType fenceAction) { + switch (fenceAction) { + case START: + tasksProcessor = new StartActionTaskProcessor(fencedHost); + break; + + case STOP: + tasksProcessor = new StopActionTaskProcessor(fencedHost); + break; + + case STATUS: + tasksProcessor = new StatusActionTaskProcessor(fencedHost); + break; + } + } + + /** + * Creates instance of fence executor for specified agent + */ + protected FenceActionExecutor createFenceActionExecutor(FenceAgent fenceAgent) { + return new SingleAgentFenceActionExecutor(fencedHost, fenceAgent, fencingPolicy); + } + + /** + * Creates task which executes specified fence action using specified fence executor + */ + protected Callable<FenceOperationResult> createTask( + final FenceActionExecutor executor, + final FenceActionType fenceAction + ) { + return (new Callable<FenceOperationResult>() { + @Override + public FenceOperationResult call() { + return executor.fence(fenceAction); + } + }); + } + + /** + * Creates list of tasks to execute specified fence actions on all fence agents + */ + protected List<Callable<FenceOperationResult>> createTasks(FenceActionType fenceAction) { + List<Callable<FenceOperationResult>> tasks = new ArrayList<>(); + for (FenceAgent fenceAgent : fenceAgents) { + tasks.add(createTask(createFenceActionExecutor(fenceAgent), fenceAction)); + } + return tasks; + } + + /** + * Executes specified fence action on all fence agents concurrently + */ + protected FenceOperationResult fenceConcurrently(FenceActionType fenceAction) { + List<FenceOperationResult> results = new ArrayList<>(fenceAgents.size()); + FenceOperationResult taskResult; + + ExecutorCompletionService<FenceOperationResult> tasksExecutor = ThreadPoolUtil.createCompletionService(); + List<Future<FenceOperationResult>> futures = + ThreadPoolUtil.submitTasks(tasksExecutor, createTasks(fenceAction)); + + for (int i = 0; i < fenceAgents.size(); ++i) { + try { + taskResult = tasksExecutor.take().get(); + } catch (ExecutionException | InterruptedException e) { + taskResult = new FenceOperationResult( + Status.ERROR, + HostPowerStatus.UNKNOWN, + e.getMessage()); + log.error("Error getting task result: {}", e.getMessage()); + log.debug("Exception", e); + } + + results.add(taskResult); + if (tasksProcessor.isGoalReached(taskResult)) { + // action goal is reach, cancel all remaining tasks + for (Future<FenceOperationResult> future : futures) { + future.cancel(true); + } + break; + } + } + return tasksProcessor.createActionResult(results); + } + + /** + * Base class to manage processing of fence action for concurrent agents + */ + protected static abstract class BaseTaskProcessor { + protected final VDS fencedHost; + + public BaseTaskProcessor(VDS fencedHost) { + this.fencedHost = fencedHost; + } + + /** + * Checks if fence action goal is reached + * @param result + * result of a finished task + * @return {@code true} if fence action goal was reached, otherwise {@code false} + */ + abstract public boolean isGoalReached(FenceOperationResult result); + + /** + * Creates a result the whole fence action on concurrent agents + * @param taskResults + * list of results of all tasks + * @return result the whole fence action on concurrent agents + */ + abstract public FenceOperationResult createActionResult(List<FenceOperationResult> taskResults); + } + + /** + * Manages processing of fence status action for concurrent agents + */ + protected static class StatusActionTaskProcessor extends BaseTaskProcessor { + public StatusActionTaskProcessor(VDS fencedHost) { + super(fencedHost); + } + + /** + * If at least one agent reports status {@code HostPowerStatus.ON}, the the host is on (so the goal of status + * action is reached), otherwise and we have to continue with processing + */ + @Override + public boolean isGoalReached(FenceOperationResult result) { + return result.getPowerStatus() == HostPowerStatus.ON; + } + + @Override + public FenceOperationResult createActionResult(List<FenceOperationResult> taskResults) { + FenceOperationResult successfulResult = null; + int statusOffReported = 0; + int errorReported = 0; + for (FenceOperationResult result : taskResults) { + if (result.getPowerStatus() == HostPowerStatus.ON) { + // one task reported power status on, so the host should be on + return result; + } + + if (result.getStatus() == Status.SUCCESS) { + // save successful task result, so we know that at least one status attempt was successful with + // power status off, so we can return it + successfulResult = result; + if (result.getPowerStatus() == HostPowerStatus.OFF) { + // note that we received off status + statusOffReported++; + } + } else { + errorReported++; + } + } + + if (statusOffReported > 0) { + if (errorReported > 0) { + // we received at least one error and at least one power off status reported, so we cannot determine + // if host is really powered off + return new FenceOperationResult( + Status.ERROR, + HostPowerStatus.UNKNOWN, + String.format( + "Unable to determine host '%s' power status: at least one agent failed to get" + + "status and at least one agent reported host is powered off.", + fencedHost)); + } else { + // no errors received, report successful result + return successfulResult; + } + } + + // all tasks returned error, so the whole status action failed + return new FenceOperationResult( + Status.ERROR, + HostPowerStatus.UNKNOWN, + "All agents failed to get host power status."); + } + } + + /** + * Manages processing of fence start action for concurrent agents + */ + protected static class StartActionTaskProcessor extends BaseTaskProcessor { + public StartActionTaskProcessor(VDS fencedHost) { + super(fencedHost); + } + + /** + * If at least one agent reports status {@code HostPowerStatus.ON}, the the host is on (so the goal of start + * action is reached), otherwise and we have to continue with processing + */ + @Override + public boolean isGoalReached(FenceOperationResult result) { + return result.getPowerStatus() == HostPowerStatus.ON; + } + + @Override + public FenceOperationResult createActionResult(List<FenceOperationResult> taskResults) { + for (FenceOperationResult result : taskResults) { + if (result.getPowerStatus() == HostPowerStatus.ON) { + return result; + } + } + + // no task reported status ON, so whole start operation has to fail + return new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN); + } + } + + /** + * Manages processing of fence stop action for concurrent agents + */ + protected static class StopActionTaskProcessor extends BaseTaskProcessor { + public StopActionTaskProcessor(VDS fencedHost) { + super(fencedHost); + } + + /** + * For stop action all agents have to report status {@code HostPowerStatus.OFF}, so we have to continue with + * processing until all agents reported its result + */ + @Override + public boolean isGoalReached(FenceOperationResult result) { + return false; + } + + @Override + public FenceOperationResult createActionResult(List<FenceOperationResult> taskResults) { + int skippedDueToPolicy = 0; + FenceOperationResult skippedDueToPolicyResult = null; + for (FenceOperationResult result : taskResults) { + if (result.getStatus() == Status.SKIPPED_DUE_TO_POLICY) { + skippedDueToPolicy++; + skippedDueToPolicyResult = result; + } else if (result.getStatus() != Status.SUCCESS) { + // stop action on one agent failed, so the whole action has to fail also + return result; + } + } + + if (skippedDueToPolicy == taskResults.size()) { + // all agents reported skipped due to policy, return it as a whole operation result + return skippedDueToPolicyResult; + } else if (skippedDueToPolicy > 0) { + // only some agents reported skipped due to policy, return error + return new FenceOperationResult( + Status.ERROR, + HostPowerStatus.UNKNOWN, + "Fence action was skipped due to fencing policy only on some of fence agents, but not all"); + } + + // all tasks returned status off, so the whole action is successful + return new FenceOperationResult(Status.SUCCESS, HostPowerStatus.OFF); + } + } +} diff --git a/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/pm/ConcurrentAgentsFenceActionExecutorTest.java b/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/pm/ConcurrentAgentsFenceActionExecutorTest.java new file mode 100644 index 0000000..980854e --- /dev/null +++ b/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/pm/ConcurrentAgentsFenceActionExecutorTest.java @@ -0,0 +1,273 @@ +package org.ovirt.engine.core.bll.pm; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import org.ovirt.engine.core.common.businessentities.FenceAgent; +import org.ovirt.engine.core.common.businessentities.FencingPolicy; +import org.ovirt.engine.core.common.businessentities.VDS; +import org.ovirt.engine.core.common.businessentities.pm.FenceActionType; +import org.ovirt.engine.core.common.businessentities.pm.FenceOperationResult; +import org.ovirt.engine.core.common.businessentities.pm.FenceOperationResult.Status; +import org.ovirt.engine.core.common.businessentities.pm.HostPowerStatus; +import org.ovirt.engine.core.common.config.ConfigValues; +import org.ovirt.engine.core.utils.MockConfigRule; + +@RunWith(MockitoJUnitRunner.class) +public class ConcurrentAgentsFenceActionExecutorTest { + @ClassRule + public static MockConfigRule configRule = + new MockConfigRule( + MockConfigRule.mockConfig(ConfigValues.DefaultMinThreadPoolSize, 2), + MockConfigRule.mockConfig(ConfigValues.DefaultMaxThreadPoolSize, 10), + MockConfigRule.mockConfig(ConfigValues.DefaultMaxThreadWaitQueueSize, 5)); + + @Mock + SingleAgentFenceActionExecutor singleExecutor1; + + @Mock + SingleAgentFenceActionExecutor singleExecutor2; + + @Mock + VDS fencedHost; + + FenceAgent agent1; + + FenceAgent agent2; + + ConcurrentAgentsFenceActionExecutor executor; + + @Before + public void setup() { + agent1 = new FenceAgent(); + agent2 = new FenceAgent(); + + List<FenceAgent> fenceAgents = new ArrayList<>(); + fenceAgents.add(agent1); + fenceAgents.add(agent2); + + executor = spy(new ConcurrentAgentsFenceActionExecutor( + fencedHost, + fenceAgents, + new FencingPolicy())); + + doReturn(singleExecutor1).when(executor).createFenceActionExecutor(eq(agent1)); + doReturn(singleExecutor2).when(executor).createFenceActionExecutor(eq(agent2)); + doReturn("host1").when(fencedHost).getHostName(); + } + + /** + * Test status action with power on result, when both agents reports power on + */ + @Test + public void statusOnWhenAllReportsOn() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON); + + mockSingleAgentResult(singleExecutor1, new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON)); + mockSingleAgentResult(singleExecutor2, new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON)); + + FenceOperationResult result = executor.fence(FenceActionType.STATUS); + + validateResult(expectedResult, result); + } + + /** + * Test status action with power on result, when one agent reports power on + */ + @Test + public void statusOnWhenOneReportsOn() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON); + + mockSingleAgentResult(singleExecutor1, new FenceOperationResult(Status.SUCCESS, HostPowerStatus.OFF)); + mockSingleAgentResult(singleExecutor2, new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON)); + + FenceOperationResult result = executor.fence(FenceActionType.STATUS); + + validateResult(expectedResult, result); + } + + /** + * Test status action with power on result, when one agent reports power on and other reports error + */ + @Test + public void statusOnWhenOneReportsOnAndOtherFailed() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON); + + mockSingleAgentResult(singleExecutor1, new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN)); + mockSingleAgentResult(singleExecutor2, new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON)); + + FenceOperationResult result = executor.fence(FenceActionType.STATUS); + + validateResult(expectedResult, result); + } + + /** + * Test status action with power off result, when all agents reports power off + */ + @Test + public void statusOffWhenAllReportsOff() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.OFF); + + mockSingleAgentResult(singleExecutor1, new FenceOperationResult(Status.SUCCESS, HostPowerStatus.OFF)); + mockSingleAgentResult(singleExecutor2, new FenceOperationResult(Status.SUCCESS, HostPowerStatus.OFF)); + + FenceOperationResult result = executor.fence(FenceActionType.STATUS); + + validateResult(expectedResult, result); + } + + /** + * Test failed status action, when one agent reports power off and other reports error + */ + @Test + public void failedStatusWhenOneReportsOffAndOtherFailed() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN); + + mockSingleAgentResult(singleExecutor1, new FenceOperationResult(Status.SUCCESS, HostPowerStatus.OFF)); + mockSingleAgentResult(singleExecutor2, new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN)); + + FenceOperationResult result = executor.fence(FenceActionType.STATUS); + + validateResult(expectedResult, result); + } + + /** + * Test failed status action, when all agents failed to get status + */ + @Test + public void failedStatusWhenAllAgentsFailed() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN); + + mockSingleAgentResult(singleExecutor1, new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN)); + mockSingleAgentResult(singleExecutor2, new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN)); + + FenceOperationResult result = executor.fence(FenceActionType.STATUS); + + validateResult(expectedResult, result); + } + + /** + * Test successful start action, when all agents were successful + */ + @Test + public void successfulStartWhenAllAgentsSuccessful() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON); + + mockSingleAgentResult(singleExecutor1, new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON)); + mockSingleAgentResult(singleExecutor2, new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON)); + + FenceOperationResult result = executor.fence(FenceActionType.START); + + validateResult(expectedResult, result); + } + + /** + * Test successful start action, when one agent was successful and another failed + */ + @Test + public void successfulStartWhenOneSuccessfulAndAnotherFailed() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON); + + mockSingleAgentResult(singleExecutor1, new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN)); + mockSingleAgentResult(singleExecutor2, new FenceOperationResult(Status.SUCCESS, HostPowerStatus.ON)); + + FenceOperationResult result = executor.fence(FenceActionType.START); + + validateResult(expectedResult, result); + } + + /** + * Test failed start action, when all agents failed + */ + @Test + public void failedStartWhenAllAgentsFailed() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN); + + mockSingleAgentResult(singleExecutor1, new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN)); + mockSingleAgentResult(singleExecutor2, new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN)); + + FenceOperationResult result = executor.fence(FenceActionType.START); + + validateResult(expectedResult, result); + } + + /** + * Test successful stop action, when all agents were successful + */ + @Test + public void successfulStopWhenAllAgentsSuccessful() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.SUCCESS, HostPowerStatus.OFF); + + mockSingleAgentResult(singleExecutor1, new FenceOperationResult(Status.SUCCESS, HostPowerStatus.OFF)); + mockSingleAgentResult(singleExecutor2, new FenceOperationResult(Status.SUCCESS, HostPowerStatus.OFF)); + + FenceOperationResult result = executor.fence(FenceActionType.STOP); + + validateResult(expectedResult, result); + } + + /** + * Test failed stop action, when one agent was successful and another failed + */ + @Test + public void failedStopWhenOneSuccessfulAndAnotherFailed() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN); + + mockSingleAgentResult(singleExecutor1, new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN)); + mockSingleAgentResult(singleExecutor2, new FenceOperationResult(Status.SUCCESS, HostPowerStatus.OFF)); + + FenceOperationResult result = executor.fence(FenceActionType.STOP); + + validateResult(expectedResult, result); + } + + /** + * Test failed stop action, when all agents failed + */ + @Test + public void failedStopWhenAllAgentsFailed() { + FenceOperationResult expectedResult = + new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN); + + mockSingleAgentResult(singleExecutor1, new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN)); + mockSingleAgentResult(singleExecutor2, new FenceOperationResult(Status.ERROR, HostPowerStatus.UNKNOWN)); + + FenceOperationResult result = executor.fence(FenceActionType.STOP); + + validateResult(expectedResult, result); + } + + protected void mockSingleAgentResult(SingleAgentFenceActionExecutor executor, FenceOperationResult result) { + doReturn(result).when(executor).fence(any(FenceActionType.class)); + } + + protected void validateResult(FenceOperationResult expected, FenceOperationResult actual) { + assertNotNull(actual); + assertEquals(expected.getStatus(), actual.getStatus()); + assertEquals(expected.getPowerStatus(), actual.getPowerStatus()); + } +} -- To view, visit https://gerrit.ovirt.org/38964 To unsubscribe, visit https://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I42e7fc0b530a341e9137c4d1efda6ae3f7ab2752 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Martin Peřina <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
