Repository: incubator-slider Updated Branches: refs/heads/develop 2c4ceb6ba -> 313f9eea7
SLIDER-1199 Blacklist nodes that exceed the node failure threshold for a role Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/313f9eea Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/313f9eea Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/313f9eea Branch: refs/heads/develop Commit: 313f9eea75e8ae09d7267434dd48dd6456bc068b Parents: 2c4ceb6 Author: Billie Rinaldi <bil...@apache.org> Authored: Wed Feb 8 19:28:27 2017 -0800 Committer: Billie Rinaldi <bil...@apache.org> Committed: Wed Feb 8 20:04:46 2017 -0800 ---------------------------------------------------------------------- .../providers/AbstractProviderService.java | 6 ++ .../server/appmaster/SliderAppMaster.java | 8 +- .../appmaster/actions/ResetFailureWindow.java | 14 ++- .../operations/AsyncRMOperationHandler.java | 6 ++ .../ProviderNotifyingOperationHandler.java | 8 ++ .../operations/RMOperationHandlerActions.java | 8 ++ .../operations/UpdateBlacklistOperation.java | 42 ++++++++ .../slider/server/appmaster/state/AppState.java | 19 ++++ .../server/appmaster/state/NodeInstance.java | 10 ++ .../server/appmaster/state/RoleHistory.java | 34 +++++++ .../TestMockAppStateContainerFailure.groovy | 22 ++-- .../TestRoleHistoryUpdateBlacklist.groovy | 101 +++++++++++++++++++ .../server/appmaster/model/mock/MockAM.groovy | 23 +++++ .../appmaster/model/mock/MockFactory.groovy | 12 ++- .../model/mock/MockProviderService.groovy | 6 ++ .../model/mock/MockRMOperationHandler.groovy | 9 ++ 16 files changed, 311 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/313f9eea/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java index 92766f5..030fe1a 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java @@ -410,6 +410,12 @@ public abstract class AbstractProviderService } @Override + public void updateBlacklist(List<String> blacklistAdditions, + List<String> blacklistRemovals) { + // no-op + } + + @Override public void execute(List<AbstractRMOperation> operations) { for (AbstractRMOperation operation : operations) { operation.execute(this); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/313f9eea/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index bb9e418..8e45fe4 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -1874,7 +1874,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService */ private void scheduleFailureWindowResets(ConfTree resources) throws BadConfigException { - ResetFailureWindow reset = new ResetFailureWindow(); + ResetFailureWindow reset = new ResetFailureWindow(rmOperationHandler); ConfTreeOperations ops = new ConfTreeOperations(resources); MapOperations globals = ops.getGlobalOptions(); long seconds = globals.getTimeRange(ResourceKeys.CONTAINER_FAILURE_WINDOW, @@ -2112,6 +2112,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService rmOperationHandler.cancelSingleRequest(request); } + @Override + public void updateBlacklist(List<String> blacklistAdditions, + List<String> blacklistRemovals) { + rmOperationHandler.updateBlacklist(blacklistAdditions, blacklistRemovals); + } + /* =================================================================== */ /* END */ /* =================================================================== */ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/313f9eea/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ResetFailureWindow.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ResetFailureWindow.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ResetFailureWindow.java index 28bcf55..36f58dd 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ResetFailureWindow.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ResetFailureWindow.java @@ -19,21 +19,31 @@ package org.apache.slider.server.appmaster.actions; import org.apache.slider.server.appmaster.SliderAppMaster; +import org.apache.slider.server.appmaster.operations.AbstractRMOperation; +import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions; import org.apache.slider.server.appmaster.state.AppState; /** * Requests the AM to reset the failure window */ public class ResetFailureWindow extends AsyncAction { + private final RMOperationHandlerActions operationHandler; - public ResetFailureWindow() { + public ResetFailureWindow(RMOperationHandlerActions operationHandler) { super("ResetFailureWindow"); + this.operationHandler = operationHandler; } @Override public void execute(SliderAppMaster appMaster, QueueAccess queueService, AppState appState) throws Exception { - appState.resetFailureCounts(); + synchronized (appMaster) { + appState.resetFailureCounts(); + AbstractRMOperation blacklistOperation = appState.updateBlacklist(); + if (blacklistOperation != null) { + blacklistOperation.execute(operationHandler); + } + } } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/313f9eea/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java index 03231ef..7173354 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java @@ -107,4 +107,10 @@ public class AsyncRMOperationHandler extends RMOperationHandler { public void addContainerRequest(AMRMClient.ContainerRequest req) { client.addContainerRequest(req); } + + @Override + public void updateBlacklist(List<String> blacklistAdditions, + List<String> blacklistRemovals) { + client.updateBlacklist(blacklistAdditions, blacklistRemovals); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/313f9eea/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java index 184a36a..972cc30 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java @@ -23,6 +23,8 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.slider.providers.ProviderService; +import java.util.List; + public class ProviderNotifyingOperationHandler extends RMOperationHandler { private final ProviderService providerService; @@ -52,4 +54,10 @@ public class ProviderNotifyingOperationHandler extends RMOperationHandler { public void cancelSingleRequest(AMRMClient.ContainerRequest request) { providerService.cancelSingleRequest(request); } + + @Override + public void updateBlacklist(List<String> blacklistAdditions, + List<String> blacklistRemovals) { + providerService.updateBlacklist(blacklistAdditions, blacklistRemovals); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/313f9eea/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java index b7794ed..b8566d8 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java @@ -53,6 +53,14 @@ public interface RMOperationHandlerActions { int cancelContainerRequests(Priority priority1, Priority priority2, int count); /** + * Blacklist resources + * @param blacklistAdditions resources to add to the blacklist + * @param blacklistRemovals resources to remove from the blacklist + */ + void updateBlacklist(List<String> blacklistAdditions, + List<String> blacklistRemovals); + + /** * Execute an entire list of operations * @param operations ops */ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/313f9eea/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/UpdateBlacklistOperation.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/UpdateBlacklistOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/UpdateBlacklistOperation.java new file mode 100644 index 0000000..7bc060d --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/UpdateBlacklistOperation.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.appmaster.operations; + +import java.util.List; + +public class UpdateBlacklistOperation extends AbstractRMOperation { + private final List<String> blacklistAdditions; + private final List<String> blacklistRemovals; + + public UpdateBlacklistOperation(List<String> blacklistAdditions, + List<String> blacklistRemovals) { + this.blacklistAdditions = blacklistAdditions; + this.blacklistRemovals = blacklistRemovals; + } + + @Override + public void execute(RMOperationHandlerActions handler) { + handler.updateBlacklist(blacklistAdditions, blacklistRemovals); + } + + @Override + public String toString() { + return "blacklist additions: " + blacklistAdditions + + ", blacklist removals: " + blacklistRemovals; + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/313f9eea/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 3db9388..0ca9f78 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -75,6 +75,7 @@ import org.apache.slider.server.appmaster.management.MetricsConstants; import org.apache.slider.server.appmaster.operations.AbstractRMOperation; import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation; import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; +import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -449,6 +450,11 @@ public class AppState { return roleHistory; } + @VisibleForTesting + public void setRoleHistory(RoleHistory roleHistory) { + this.roleHistory = roleHistory; + } + /** * Get the path used for history files * @return the directory used for history files @@ -1976,6 +1982,15 @@ public class AppState { return results; } + public synchronized AbstractRMOperation updateBlacklist() { + UpdateBlacklistOperation blacklistOperation = + roleHistory.updateBlacklist(getRoleStatusMap().values()); + if (blacklistOperation != null) { + log.info("Updating {}", blacklistOperation); + } + return blacklistOperation; + } + /** * Look at where the current node state is -and whether it should be changed */ @@ -1983,6 +1998,10 @@ public class AppState { throws SliderInternalStateException, TriggerClusterTeardownException { log.debug("in reviewRequestAndReleaseNodes()"); List<AbstractRMOperation> allOperations = new ArrayList<>(); + AbstractRMOperation blacklistOperation = updateBlacklist(); + if (blacklistOperation != null) { + allOperations.add(blacklistOperation); + } for (RoleStatus roleStatus : getRoleStatusMap().values()) { if (!roleStatus.isExcludeFromFlexing()) { List<AbstractRMOperation> operations = reviewOneRole(roleStatus); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/313f9eea/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java index cc17cf0..ea07698 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java @@ -42,6 +42,8 @@ public class NodeInstance { public final String hostname; + private boolean blacklisted = false; + /** * last state of node. Starts off as {@link NodeState#RUNNING}, * on the assumption that it is live. @@ -81,6 +83,14 @@ public class NodeInstance { nodeEntries = new ArrayList<>(roles); } + public synchronized void setBlacklisted(boolean blacklisted) { + this.blacklisted = blacklisted; + } + + public boolean isBlacklisted() { + return blacklisted; + } + /** * Update the node status. * The return code is true if the node state changed enough to http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/313f9eea/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java index 4e8a4d7..38c70f3 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java @@ -34,6 +34,7 @@ import org.apache.slider.server.appmaster.management.BoolMetric; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.management.Timestamp; import org.apache.slider.server.appmaster.operations.AbstractRMOperation; +import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation; import org.apache.slider.server.avro.LoadedRoleHistory; import org.apache.slider.server.avro.NodeEntryRecord; import org.apache.slider.server.avro.RoleHistoryHeader; @@ -49,6 +50,7 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; /** @@ -546,6 +548,38 @@ public class RoleHistory { } } + public synchronized UpdateBlacklistOperation updateBlacklist( + Collection<RoleStatus> roleStatuses) { + List<String> blacklistAdditions = new ArrayList<>(); + List<String> blacklistRemovals = new ArrayList<>(); + for (Entry<String, NodeInstance> nodeInstanceEntry : nodemap.entrySet()) { + boolean shouldBeBlacklisted = false; + String nodeHost = nodeInstanceEntry.getKey(); + NodeInstance nodeInstance = nodeInstanceEntry.getValue(); + for (RoleStatus roleStatus : roleStatuses) { + if (nodeInstance.exceedsFailureThreshold(roleStatus)) { + shouldBeBlacklisted = true; + break; + } + } + if (shouldBeBlacklisted) { + if (!nodeInstance.isBlacklisted()) { + blacklistAdditions.add(nodeHost); + nodeInstance.setBlacklisted(true); + } + } else { + if (nodeInstance.isBlacklisted()) { + blacklistRemovals.add(nodeHost); + nodeInstance.setBlacklisted(false); + } + } + } + if (blacklistAdditions.isEmpty() && blacklistRemovals.isEmpty()) { + return null; + } + return new UpdateBlacklistOperation(blacklistAdditions, blacklistRemovals); + } + /** * Find a node for use * @param role role http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/313f9eea/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy index 0eb5456..f1a4027 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy @@ -27,7 +27,9 @@ import org.apache.slider.core.exceptions.SliderException import org.apache.slider.core.exceptions.TriggerClusterTeardownException import org.apache.slider.server.appmaster.actions.ResetFailureWindow import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest +import org.apache.slider.server.appmaster.model.mock.MockAM import org.apache.slider.server.appmaster.model.mock.MockAppState +import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler import org.apache.slider.server.appmaster.model.mock.MockRoles import org.apache.slider.server.appmaster.model.mock.MockYarnEngine import org.apache.slider.server.appmaster.state.AppState @@ -46,6 +48,8 @@ import org.junit.Test @Slf4j class TestMockAppStateContainerFailure extends BaseMockAppStateTest implements MockRoles { + MockRMOperationHandler operationHandler = new MockRMOperationHandler() + MockAM mockAM = new MockAM() @Override String getTestName() { @@ -214,14 +218,14 @@ class TestMockAppStateContainerFailure extends BaseMockAppStateTest @Test public void testRoleStatusFailureWindow() throws Throwable { - ResetFailureWindow resetter = new ResetFailureWindow(); + ResetFailureWindow resetter = new ResetFailureWindow(operationHandler); // initial reset - resetter.execute(null, null, appState) + resetter.execute(mockAM, null, appState) role0Status.desired = 1 for (int i = 0; i < 100; i++) { - resetter.execute(null, null, appState) + resetter.execute(mockAM, null, appState) List<RoleInstance> instances = createAndSubmitNodes() assert instances.size() == 1 @@ -250,8 +254,8 @@ class TestMockAppStateContainerFailure extends BaseMockAppStateTest assert 0L == status.preempted assert 0L == status.nodeFailed - ResetFailureWindow resetter = new ResetFailureWindow(); - resetter.execute(null, null, appState) + ResetFailureWindow resetter = new ResetFailureWindow(operationHandler); + resetter.execute(mockAM, null, appState) assert 1 == status.failed assert 0L == status.failedRecently } @@ -267,8 +271,8 @@ class TestMockAppStateContainerFailure extends BaseMockAppStateTest assert 0L == status.preempted assert 0L == status.nodeFailed - ResetFailureWindow resetter = new ResetFailureWindow(); - resetter.execute(null, null, appState) + ResetFailureWindow resetter = new ResetFailureWindow(operationHandler); + resetter.execute(mockAM, null, appState) assert 1 == status.failed assert 0L == status.failedRecently assert 1L == status.limitsExceeded @@ -285,8 +289,8 @@ class TestMockAppStateContainerFailure extends BaseMockAppStateTest assert 0L == status.failedRecently assert 0L == status.nodeFailed - ResetFailureWindow resetter = new ResetFailureWindow(); - resetter.execute(null, null, appState) + ResetFailureWindow resetter = new ResetFailureWindow(operationHandler); + resetter.execute(mockAM, null, appState) assert 1L == status.preempted } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/313f9eea/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.groovy new file mode 100644 index 0000000..8e26003 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.groovy @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.history + +import org.apache.slider.server.appmaster.actions.ResetFailureWindow +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest +import org.apache.slider.server.appmaster.model.mock.MockAM +import org.apache.slider.server.appmaster.model.mock.MockFactory +import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler +import org.apache.slider.server.appmaster.model.mock.MockRoleHistory +import org.apache.slider.server.appmaster.operations.AbstractRMOperation +import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation +import org.apache.slider.server.appmaster.state.NodeInstance +import org.apache.slider.server.appmaster.state.RoleHistory +import org.apache.slider.server.appmaster.state.RoleStatus +import org.junit.Before +import org.junit.Test + +class TestRoleHistoryUpdateBlacklist extends BaseMockAppStateTest { + RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES) + Collection<RoleStatus> roleStatuses = [new RoleStatus(MockFactory.PROVIDER_ROLE0)] + NodeInstance ni = nodeInstance(1, 0, 0, 0) + List<NodeInstance> nodes = [ni] + + @Override + String getTestName() { + return "TestUpdateBlacklist" + } + + @Before + public void setupNodeMap() { + roleHistory.insert(nodes) + roleHistory.buildRecentNodeLists(); + appState.roleHistory = roleHistory + } + + @Test + public void testUpdateBlacklist() { + assert !ni.isBlacklisted() + + // at threshold, blacklist is unmodified + recordAsFailed(ni, 0, MockFactory.NODE_FAILURE_THRESHOLD) + def op = roleHistory.updateBlacklist(roleStatuses) + assert null == op + assert !ni.isBlacklisted() + + // threshold is reached, node goes on blacklist + recordAsFailed(ni, 0, 1) + op = roleHistory.updateBlacklist(roleStatuses) + assert null != op + assert ni.isBlacklisted() + + // blacklist remains unmodified + op = roleHistory.updateBlacklist(roleStatuses) + assert null == op + assert ni.isBlacklisted() + + // failure threshold reset, node goes off blacklist + ni.resetFailedRecently() + op = roleHistory.updateBlacklist(roleStatuses) + assert null != op + assert !ni.isBlacklisted() + } + + @Test + public void testBlacklistOperations() { + recordAsFailed(ni, 0, MockFactory.NODE_FAILURE_THRESHOLD + 1) + + List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() + assertListLength(ops, 1) + AbstractRMOperation op = ops[0] + assert op instanceof UpdateBlacklistOperation + assert ni.isBlacklisted() + + MockRMOperationHandler handler = new MockRMOperationHandler() + assert 0 == handler.blacklisted + handler.execute(ops) + assert 1 == handler.blacklisted + + ResetFailureWindow resetter = new ResetFailureWindow(handler); + resetter.execute(new MockAM(), null, appState) + assert 0 == handler.blacklisted + assert !ni.isBlacklisted() + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/313f9eea/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAM.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAM.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAM.groovy new file mode 100644 index 0000000..e24c6b3 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAM.groovy @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.appmaster.model.mock + +import org.apache.slider.server.appmaster.SliderAppMaster + +class MockAM extends SliderAppMaster { +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/313f9eea/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy index 3ba6e31..bad44c6 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy @@ -45,6 +45,8 @@ import org.apache.slider.providers.ProviderRole @Slf4j class MockFactory implements MockRoles { + public static final NODE_FAILURE_THRESHOLD = 2; + public static MockFactory instance = new MockFactory(); /* @@ -58,7 +60,7 @@ class MockFactory implements MockRoles { MockRoles.ROLE0, 0, PlacementPolicy.DEFAULT, - 2, + NODE_FAILURE_THRESHOLD, 1, ResourceKeys.DEF_YARN_LABEL_EXPRESSION) /** @@ -68,7 +70,7 @@ class MockFactory implements MockRoles { MockRoles.ROLE1, 1, PlacementPolicy.STRICT, - 2, + NODE_FAILURE_THRESHOLD, 1, ResourceKeys.DEF_YARN_LABEL_EXPRESSION) @@ -79,7 +81,7 @@ class MockFactory implements MockRoles { MockRoles.ROLE2, 2, PlacementPolicy.ANYWHERE, - 2, + NODE_FAILURE_THRESHOLD, 2, ResourceKeys.DEF_YARN_LABEL_EXPRESSION) @@ -90,7 +92,7 @@ class MockFactory implements MockRoles { MockRoles.ROLE2, 2, PlacementPolicy.ANTI_AFFINITY_REQUIRED, - 2, + NODE_FAILURE_THRESHOLD, 2, null) @@ -101,7 +103,7 @@ class MockFactory implements MockRoles { MockRoles.ROLE1, 1, PlacementPolicy.ANTI_AFFINITY_REQUIRED, - 2, + NODE_FAILURE_THRESHOLD, 1, MockRoles.LABEL_GPU) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/313f9eea/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy index 1ea2277..1580787 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy @@ -283,6 +283,12 @@ class MockProviderService implements ProviderService { } @Override + void updateBlacklist(List<String> blacklistAdditions, + List<String> blacklistRemovals) { + + } + + @Override void execute(List<AbstractRMOperation> operations) { } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/313f9eea/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy index c803b54..921c283 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy @@ -35,6 +35,8 @@ class MockRMOperationHandler extends RMOperationHandler { int availableToCancel = 0; // count of cancelled values. This must be explicitly set int cancelled + // number blacklisted + int blacklisted = 0 @Override public void releaseAssignedContainer(ContainerId containerId) { @@ -70,6 +72,13 @@ class MockRMOperationHandler extends RMOperationHandler { } } + @Override + void updateBlacklist(List<String> blacklistAdditions, List<String> + blacklistRemovals) { + blacklisted += blacklistAdditions.size() + blacklisted -= blacklistRemovals.size() + } + /** * clear the history */
