Repository: asterixdb Updated Branches: refs/heads/master 198ffbca2 -> 52ca73695
[CLUS][MTD] += ClusterStateManager pending removal - storage format changes: no - interface changes: no Add ClusterStateManager API to enable marking nodes as pending removal, so they are not considered when creating the node groups for new datasets Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1844 Sonar-Qube: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> BAD: Jenkins <[email protected]> Reviewed-by: Yingyi Bu <[email protected]> Reviewed-by: abdullah alamoudi <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/52ca7369 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/52ca7369 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/52ca7369 Branch: refs/heads/master Commit: 52ca73695ad4d0297341196c768d83404bdb663b Parents: 198ffbc Author: Michael Blow <[email protected]> Authored: Wed Jun 21 22:11:58 2017 -0400 Committer: Yingyi Bu <[email protected]> Committed: Thu Jun 22 20:04:38 2017 -0700 ---------------------------------------------------------------------- .../asterix/app/translator/QueryTranslator.java | 10 +++-- .../metadata/dataset/hints/DatasetHints.java | 9 ++--- .../runtime/utils/ClusterStateManager.java | 41 +++++++++++++++++++- 3 files changed, 49 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/52ca7369/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index f7327b4..8a7d757 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -709,17 +709,19 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen protected static String configureNodegroupForDataset(ICcApplicationContext appCtx, Map<String, String> hints, String dataverseName, String datasetName, MetadataProvider metadataProvider) throws Exception { - Set<String> allNodes = ClusterStateManager.INSTANCE.getParticipantNodes(); + Set<String> allNodes = ClusterStateManager.INSTANCE.getParticipantNodes(true); Set<String> selectedNodes = new LinkedHashSet<>(); String hintValue = hints.get(DatasetNodegroupCardinalityHint.NAME); if (hintValue == null) { selectedNodes.addAll(allNodes); } else { int nodegroupCardinality; - boolean valid = DatasetHints.validate(appCtx, DatasetNodegroupCardinalityHint.NAME, - hints.get(DatasetNodegroupCardinalityHint.NAME)).first; + final Pair<Boolean, String> validation = DatasetHints.validate(appCtx, DatasetNodegroupCardinalityHint.NAME, + hints.get(DatasetNodegroupCardinalityHint.NAME)); + boolean valid = validation.first; if (!valid) { - throw new CompilationException("Incorrect use of hint:" + DatasetNodegroupCardinalityHint.NAME); + throw new CompilationException("Incorrect use of hint '" + DatasetNodegroupCardinalityHint.NAME + + "': " + validation.second); } else { nodegroupCardinality = Integer.parseInt(hints.get(DatasetNodegroupCardinalityHint.NAME)); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/52ca7369/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java index 277587f..709f655 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java @@ -22,6 +22,7 @@ import java.util.HashSet; import java.util.Set; import org.apache.asterix.common.dataflow.ICcApplicationContext; +import org.apache.asterix.runtime.utils.ClusterStateManager; import org.apache.hyracks.algebricks.common.utils.Pair; /** @@ -106,22 +107,20 @@ public class DatasetHints { @Override public Pair<Boolean, String> validateValue(ICcApplicationContext appCtx, String value) { - boolean valid = true; int intValue; try { intValue = Integer.parseInt(value); if (intValue < 0) { return new Pair<>(false, "Value must be >= 0"); } - int numNodesInCluster = appCtx.getMetadataProperties().getNodeNames().size(); + int numNodesInCluster = ClusterStateManager.INSTANCE.getParticipantNodes(true).size(); if (numNodesInCluster < intValue) { return new Pair<>(false, - "Value must be greater or equal to the existing number of nodes in cluster (" + "Value must be less than or equal to the available number of nodes in cluster (" + numNodesInCluster + ")"); } } catch (NumberFormatException nfe) { - valid = false; - return new Pair<>(valid, "Inappropriate value"); + return new Pair<>(false, "Inappropriate value"); } return new Pair<>(true, null); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/52ca7369/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java index 9c96d32..da341af 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java @@ -65,6 +65,7 @@ public class ClusterStateManager implements IClusterStateManager { private static final Logger LOGGER = Logger.getLogger(ClusterStateManager.class.getName()); public static final ClusterStateManager INSTANCE = new ClusterStateManager(); private final Map<String, Map<IOption, Object>> activeNcConfiguration = new HashMap<>(); + private Set<String> pendingRemoval = new HashSet<>(); private final Cluster cluster; private ClusterState state = ClusterState.UNUSABLE; @@ -73,8 +74,8 @@ public class ClusterStateManager implements IClusterStateManager { private boolean globalRecoveryCompleted = false; - private Map<String, ClusterPartition[]> node2PartitionsMap = null; - private SortedMap<Integer, ClusterPartition> clusterPartitions = null; + private Map<String, ClusterPartition[]> node2PartitionsMap; + private SortedMap<Integer, ClusterPartition> clusterPartitions; private String currentMetadataNode = null; private boolean metadataNodeActive = false; @@ -101,6 +102,7 @@ public class ClusterStateManager implements IClusterStateManager { } failedNodes.add(nodeId); ftStrategy.notifyNodeFailure(nodeId); + pendingRemoval.remove(nodeId); } public synchronized void addNCConfiguration(String nodeId, Map<IOption, Object> configuration) @@ -238,6 +240,14 @@ public class ClusterStateManager implements IClusterStateManager { return participantNodes; } + public synchronized Set<String> getParticipantNodes(boolean excludePendingRemoval) { + Set<String> participantNodes = getParticipantNodes(); + if (excludePendingRemoval) { + participantNodes.removeAll(pendingRemoval); + } + return participantNodes; + } + public synchronized AlgebricksAbsolutePartitionConstraint getClusterLocations() { if (clusterPartitionConstraint == null) { resetClusterPartitionConstraint(); @@ -387,4 +397,31 @@ public class ClusterStateManager implements IClusterStateManager { } } } + + public synchronized void removePending(String nodeId) { + if (LOGGER.isLoggable(Level.INFO)) { + LOGGER.info("Registering intention to remove node id " + nodeId); + } + if (!activeNcConfiguration.containsKey(nodeId)) { + LOGGER.warning("Cannot register unknown node " + nodeId + " for pending removal"); + } + pendingRemoval.add(nodeId); + } + + public synchronized boolean cancelRemovePending(String nodeId) { + if (LOGGER.isLoggable(Level.INFO)) { + LOGGER.info("Deregistering intention to remove node id " + nodeId); + } + if (!pendingRemoval.remove(nodeId)) { + LOGGER.warning("Cannot deregister intention to remove node id " + nodeId + " that was not registered"); + return false; + } else { + return true; + } + } + + public synchronized Set<String> getNodesPendingRemoval() { + return new HashSet<>(pendingRemoval); + } + }
