This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new e19a13f SOLR-15694: Node roles
e19a13f is described below
commit e19a13f8146a713fe883c9a17809dc542638cc85
Author: Ishan Chattopadhyaya <[email protected]>
AuthorDate: Sat Jan 8 00:20:16 2022 +0530
SOLR-15694: Node roles
---
solr/CHANGES.txt | 2 +
.../apache/solr/cloud/OverseerNodePrioritizer.java | 27 ++-
.../java/org/apache/solr/cloud/ZkController.java | 42 +++--
.../apache/solr/cloud/api/collections/Assign.java | 38 ++--
.../cloud/api/collections/CreateCollectionCmd.java | 2 +-
.../cloud/api/collections/OverseerRoleCmd.java | 9 +
.../solr/cloud/api/collections/RestoreCmd.java | 2 +-
.../src/java/org/apache/solr/cluster/Cluster.java | 5 +
.../placement/impl/PlacementRequestImpl.java | 8 +-
.../impl/SimpleClusterAbstractionsImpl.java | 13 +-
.../java/org/apache/solr/core/CoreContainer.java | 10 ++
.../src/java/org/apache/solr/core/NodeRoles.java | 137 ++++++++++++++
.../java/org/apache/solr/handler/ClusterAPI.java | 191 +++++++++++++++-----
.../test/org/apache/solr/cloud/NodeRolesTest.java | 122 +++++++++++++
.../org/apache/solr/cloud/OverseerRolesTest.java | 4 +-
.../placement/ClusterAbstractionsForTest.java | 5 +
solr/solr-ref-guide/src/node-roles.adoc | 196 +++++++++++++++++++++
solr/solr-ref-guide/src/solrcloud-clusters.adoc | 4 +
.../apache/solr/common/cloud/ZkStateReader.java | 11 +-
19 files changed, 747 insertions(+), 81 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 29f47d0..66711b2 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -44,6 +44,8 @@ New Features
* SOLR-14608: Faster sorting for the /export handler (Joel Bernstein)
+* SOLR-15694: Node roles (Ishan Chattopadhyaya, noble)
+
Improvements
----------------------
* LUCENE-8984: MoreLikeThis MLT is biased for uncommon fields (Andy Hind via
Anshum Gupta)
diff --git
a/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java
b/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java
index c02403e..2a02562 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java
@@ -17,15 +17,19 @@
package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.solr.client.solrj.impl.ZkDistribStateManager;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.NodeRoles;
+import org.apache.solr.handler.ClusterAPI;
import org.apache.solr.handler.component.ShardHandler;
import org.apache.solr.handler.component.ShardHandlerFactory;
import org.apache.solr.handler.component.ShardRequest;
@@ -59,11 +63,26 @@ public class OverseerNodePrioritizer {
public synchronized void prioritizeOverseerNodes(String overseerId) throws
Exception {
SolrZkClient zk = zkStateReader.getZkClient();
- if(!zk.exists(ZkStateReader.ROLES,true))return;
- Map<?,?> m = (Map<?,?>) Utils.fromJSON(zk.getData(ZkStateReader.ROLES,
null, new Stat(), true));
+ List<String> overseerDesignates = new ArrayList<>();
+ if (zk.exists(ZkStateReader.ROLES,true)) {
+ Map<?,?> m = (Map<?,?>) Utils.fromJSON(zk.getData(ZkStateReader.ROLES,
null, new Stat(), true));
+ @SuppressWarnings("unchecked")
+ List<String> l = (List<String>) m.get("overseer");
+ if (l != null) {
+ overseerDesignates.addAll(l);
+ }
+ }
- List<?> overseerDesignates = (List<?>) m.get("overseer");
- if(overseerDesignates==null || overseerDesignates.isEmpty()) return;
+ List<String> preferredOverseers =
ClusterAPI.getNodesByRole(NodeRoles.Role.OVERSEER, NodeRoles.MODE_PREFERRED,
+ new ZkDistribStateManager(zkStateReader.getZkClient()));
+ for (String preferred: preferredOverseers) {
+ if (overseerDesignates.contains(preferred)) {
+ log.warn("Node {} has been configured to be a preferred overseer using
both ADDROLE API command " +
+ "as well as using Node Roles (i.e. -Dsolr.node.roles start up
property). Only the latter is recommended.", preferred);
+ }
+ }
+ overseerDesignates.addAll(preferredOverseers);
+ if (overseerDesignates.isEmpty()) return;
String ldr = OverseerTaskProcessor.getLeaderNode(zk);
if(overseerDesignates.contains(ldr)) return;
log.info("prioritizing overseer nodes at {} overseer designates are {}",
overseerId, overseerDesignates);
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index 75a4e83..723a49f 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -64,7 +64,6 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.*;
import org.apache.solr.common.cloud.Replica.Type;
-import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
@@ -79,6 +78,7 @@ import org.apache.solr.core.CloudConfig;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.NodeRoles;
import org.apache.solr.core.SolrCoreInitializationException;
import org.apache.solr.handler.component.HttpShardHandler;
import org.apache.solr.logging.MDCLoggingContext;
@@ -105,6 +105,7 @@ import static
org.apache.solr.common.cloud.ZkStateReader.ELECTION_NODE_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import static
org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE;
import static org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE;
/**
@@ -357,7 +358,9 @@ public class ZkController implements Closeable {
overseerElector.setup(context);
- overseerElector.joinElection(context, true);
+ if (cc.nodeRoles.isOverseerAllowedOrPreferred()) {
+ overseerElector.joinElection(context, true);
+ }
}
cc.cancelCoreRecoveries();
@@ -854,6 +857,14 @@ public class ZkController implements Closeable {
throws KeeperException, InterruptedException, IOException {
ZkCmdExecutor cmdExecutor = new
ZkCmdExecutor(zkClient.getZkClientTimeout());
cmdExecutor.ensureExists(ZkStateReader.LIVE_NODES_ZKNODE, zkClient);
+ cmdExecutor.ensureExists(ZkStateReader.NODE_ROLES, zkClient);
+ for (NodeRoles.Role role: NodeRoles.Role.values()) {
+ cmdExecutor.ensureExists(NodeRoles.getZNodeForRole(role), zkClient);
+ for (String mode: role.supportedModes()) {
+ cmdExecutor.ensureExists(NodeRoles.getZNodeForRoleMode(role, mode),
zkClient);
+ }
+ }
+
cmdExecutor.ensureExists(ZkStateReader.COLLECTIONS_ZKNODE, zkClient);
cmdExecutor.ensureExists(ZkStateReader.ALIASES, zkClient);
byte[] emptyJson = "{}".getBytes(StandardCharsets.UTF_8);
@@ -910,10 +921,11 @@ public class ZkController implements Closeable {
overseerElector = new LeaderElector(zkClient);
this.overseer = new Overseer((HttpShardHandler)
cc.getShardHandlerFactory().getShardHandler(), cc.getUpdateShardHandler(),
CommonParams.CORES_HANDLER_PATH, zkStateReader, this, cloudConfig);
- ElectionContext context = new OverseerElectionContext(zkClient,
- overseer, getNodeName());
+ ElectionContext context = new OverseerElectionContext(zkClient,
overseer, getNodeName());
overseerElector.setup(context);
- overseerElector.joinElection(context, false);
+ if (cc.nodeRoles.isOverseerAllowedOrPreferred()) {
+ overseerElector.joinElection(context, false);
+ }
}
Stat stat = zkClient.exists(ZkStateReader.LIVE_NODES_ZKNODE, null, true);
@@ -1086,6 +1098,11 @@ public class ZkController implements Closeable {
log.info("Register node as live in ZooKeeper:{}", nodePath);
List<Op> ops = new ArrayList<>(2);
ops.add(Op.create(nodePath, null,
zkClient.getZkACLProvider().getACLsToAdd(nodePath), CreateMode.EPHEMERAL));
+
+ // Create the roles node as well
+ cc.nodeRoles.getRoles().forEach((role, mode) ->
ops.add(Op.create(NodeRoles.getZNodeForRoleMode(role, mode) +"/" + nodeName,
+ null, zkClient.getZkACLProvider().getACLsToAdd(nodePath),
CreateMode.EPHEMERAL)));
+
zkClient.multi(ops, true);
}
@@ -2198,11 +2215,7 @@ public class ZkController implements Closeable {
List<?> nodeList = (List<?>) roles.get("overseer");
if (nodeList == null) return;
if (nodeList.contains(getNodeName())) {
- ZkNodeProps props = new ZkNodeProps(Overseer.QUEUE_OPERATION,
CollectionParams.CollectionAction.ADDROLE.toString().toLowerCase(Locale.ROOT),
- "node", getNodeName(),
- "role", "overseer");
- log.info("Going to add role {} ", props);
- getOverseerCollectionQueue().offer(Utils.toJSON(props));
+ setPreferredOverseer();
}
} catch (NoNodeException nne) {
return;
@@ -2211,6 +2224,15 @@ public class ZkController implements Closeable {
}
}
+ public void setPreferredOverseer() throws KeeperException,
InterruptedException {
+ ZkNodeProps props = new ZkNodeProps(Overseer.QUEUE_OPERATION,
ADDROLE.toString().toLowerCase(Locale.ROOT),
+ "node", getNodeName(),
+ "role", "overseer",
+ "persist", "false");
+ log.warn("Going to add role {}. It is deprecated to use ADDROLE and
consider using Node Roles instead.", props);
+ getOverseerCollectionQueue().offer(Utils.toJSON(props));
+ }
+
public CoreContainer getCoreContainer() {
return cc;
}
diff --git
a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
index 4dd2dbb..664bc50 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
@@ -21,20 +21,7 @@ import static
org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Random;
-import java.util.Set;
+import java.util.*;
import java.util.stream.Collectors;
import org.apache.solr.client.solrj.cloud.DistribStateManager;
@@ -54,6 +41,8 @@ import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.NodeRoles;
+import org.apache.solr.handler.ClusterAPI;
import org.apache.solr.util.NumberUtils;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@@ -229,7 +218,9 @@ public class Assign {
return false;
}
- public static List<String> getLiveOrLiveAndCreateNodeSetList(final
Set<String> liveNodes, final ZkNodeProps message, final Random random) {
+ public static List<String> getLiveOrLiveAndCreateNodeSetList(final
Set<String> liveNodes, final ZkNodeProps message, final Random random,
+
DistribStateManager zk) {
+
List<String> nodeList;
final String createNodeSetStr = message.getStr(CREATE_NODE_SET);
final List<String> createNodeList = (createNodeSetStr == null) ? null :
@@ -244,13 +235,28 @@ public class Assign {
Collections.shuffle(nodeList, random);
}
} else {
- nodeList = new ArrayList<>(liveNodes);
+ nodeList = new ArrayList<>(filterNonDataNodes(zk, liveNodes));
Collections.shuffle(nodeList, random);
}
return nodeList;
}
+ public static Collection<String> filterNonDataNodes(DistribStateManager zk,
Collection<String> liveNodes) {
+ try {
+ List<String> noData = ClusterAPI.getNodesByRole(NodeRoles.Role.DATA,
NodeRoles.MODE_OFF, zk);
+ if (noData.isEmpty()) {
+ return liveNodes;
+ } else {
+ liveNodes = new HashSet<>(liveNodes);
+ liveNodes.removeAll(noData);
+ return liveNodes;
+ }
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error
fetching roles from Zookeeper", e);
+ }
+ }
+
static class ReplicaCount {
public final String nodeName;
public Map<String, Integer> collectionReplicas;
diff --git
a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
index b7a0c97..60a5bb8 100644
---
a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
+++
b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
@@ -404,7 +404,7 @@ public class CreateCollectionCmd implements
CollApiCmds.CollectionApiCommand {
// but (for now) require that each core goes on a distinct node.
List<ReplicaPosition> replicaPositions;
- List<String> nodeList =
Assign.getLiveOrLiveAndCreateNodeSetList(clusterState.getLiveNodes(), message,
CollectionHandlingUtils.RANDOM);
+ List<String> nodeList =
Assign.getLiveOrLiveAndCreateNodeSetList(clusterState.getLiveNodes(), message,
CollectionHandlingUtils.RANDOM, cloudManager.getDistribStateManager());
if (nodeList.isEmpty()) {
log.warn("It is unusual to create a collection ({}) without cores.",
collectionName);
diff --git
a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java
b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java
index e0766e6..e2e34f6 100644
---
a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java
+++
b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java
@@ -64,6 +64,10 @@ public class OverseerRoleCmd implements
CollApiCmds.CollectionApiCommand {
SolrZkClient zkClient = zkStateReader.getZkClient();
Map<String, List<String>> roles = null;
String node = message.getStr("node");
+ if ("false".equals(message.getStr("persist"))) { // no need to persist to
roles.json
+ runPrioritizer();
+ return;
+ }
String roleName = message.getStr("role");
boolean nodeExists = false;
@@ -89,6 +93,11 @@ public class OverseerRoleCmd implements
CollApiCmds.CollectionApiCommand {
} else {
zkClient.create(ZkStateReader.ROLES, Utils.toJSON(roles),
CreateMode.PERSISTENT, true);
}
+ runPrioritizer();
+
+ }
+
+ private void runPrioritizer() {
//if there are too many nodes this command may time out. And most likely
dedicated
// overseers are created when there are too many nodes . So , do this
operation in a separate thread
new Thread(() -> {
diff --git
a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
index 3ff38ae..5c305f8 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
@@ -182,7 +182,7 @@ public class RestoreCmd implements
CollApiCmds.CollectionApiCommand {
this.shardHandler = ccc.newShardHandler();
this.nodeList = Assign.getLiveOrLiveAndCreateNodeSetList(
- zkStateReader.getClusterState().getLiveNodes(), message,
CollectionHandlingUtils.RANDOM);
+ zkStateReader.getClusterState().getLiveNodes(), message,
CollectionHandlingUtils.RANDOM,
container.getZkController().getSolrCloudManager().getDistribStateManager());
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/cluster/Cluster.java
b/solr/core/src/java/org/apache/solr/cluster/Cluster.java
index 3b7bdd4..b3b210f 100644
--- a/solr/core/src/java/org/apache/solr/cluster/Cluster.java
+++ b/solr/core/src/java/org/apache/solr/cluster/Cluster.java
@@ -32,6 +32,11 @@ public interface Cluster {
Set<Node> getLiveNodes();
/**
+ * @return current set of live nodes that are supposed to host data.
+ */
+ Set<Node> getLiveDataNodes();
+
+ /**
* Returns info about the given collection if one exists.
*
* @return {@code null} if no collection of the given name exists in the
cluster.
diff --git
a/solr/core/src/java/org/apache/solr/cluster/placement/impl/PlacementRequestImpl.java
b/solr/core/src/java/org/apache/solr/cluster/placement/impl/PlacementRequestImpl.java
index ff3f090..285ffba 100644
---
a/solr/core/src/java/org/apache/solr/cluster/placement/impl/PlacementRequestImpl.java
+++
b/solr/core/src/java/org/apache/solr/cluster/placement/impl/PlacementRequestImpl.java
@@ -82,11 +82,17 @@ public class PlacementRequestImpl implements
PlacementRequest {
// If no nodes specified, use all live nodes. If nodes are specified, use
specified list.
if (assignRequest.nodes != null) {
nodes =
SimpleClusterAbstractionsImpl.NodeImpl.getNodes(assignRequest.nodes);
+
+ for (Node n: nodes) {
+ if (!cluster.getLiveDataNodes().contains(n)) {
+ throw new Assign.AssignmentException("Bad assign request: specified
node is a non-data hosting node (" + n.getName() + ") for collection " +
solrCollection.getName());
+ }
+ }
if (nodes.isEmpty()) {
throw new Assign.AssignmentException("Bad assign request: empty list
of nodes for collection " + solrCollection.getName());
}
} else {
- nodes = cluster.getLiveNodes();
+ nodes = cluster.getLiveDataNodes();
if (nodes.isEmpty()) {
throw new Assign.AssignmentException("Impossible assign request: no
live nodes for collection " + solrCollection.getName());
}
diff --git
a/solr/core/src/java/org/apache/solr/cluster/placement/impl/SimpleClusterAbstractionsImpl.java
b/solr/core/src/java/org/apache/solr/cluster/placement/impl/SimpleClusterAbstractionsImpl.java
index 48f3f50..e5d99b2 100644
---
a/solr/core/src/java/org/apache/solr/cluster/placement/impl/SimpleClusterAbstractionsImpl.java
+++
b/solr/core/src/java/org/apache/solr/cluster/placement/impl/SimpleClusterAbstractionsImpl.java
@@ -23,6 +23,7 @@ import java.util.stream.Collectors;
import com.google.common.collect.Maps;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.cloud.api.collections.Assign;
import org.apache.solr.cluster.*;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
@@ -53,10 +54,16 @@ class SimpleClusterAbstractionsImpl {
static class ClusterImpl implements Cluster {
private final Set<Node> liveNodes;
+ private final Set<Node> liveNodesWithData;
private final ClusterState clusterState;
ClusterImpl(SolrCloudManager solrCloudManager) throws IOException {
- liveNodes =
NodeImpl.getNodes(solrCloudManager.getClusterStateProvider().getLiveNodes());
+ Set<String> liveNodes =
solrCloudManager.getClusterStateProvider().getLiveNodes();
+ Collection<String> liveNodesWithData = Assign.filterNonDataNodes(
solrCloudManager.getDistribStateManager(), liveNodes);
+ this.liveNodes = NodeImpl.getNodes(liveNodes);
+ this.liveNodesWithData = liveNodesWithData.size() == liveNodes.size() ?
+ this.liveNodes :
+ NodeImpl.getNodes(liveNodesWithData);
clusterState =
solrCloudManager.getClusterStateProvider().getClusterState();
}
@@ -65,6 +72,10 @@ class SimpleClusterAbstractionsImpl {
return liveNodes;
}
+ public Set<Node> getLiveDataNodes() {
+ return liveNodesWithData;
+ }
+
@Override
public SolrCollection getCollection(String collectionName) {
return SolrCollectionImpl.createCollectionFacade(clusterState,
collectionName);
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index d4d2030..b915330 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -246,6 +246,8 @@ public class CoreContainer {
private final ObjectCache objectCache = new ObjectCache();
+ public final NodeRoles nodeRoles = new
NodeRoles(System.getProperty(NodeRoles.NODE_ROLES_PROP));
+
private final ClusterSingletons clusterSingletons = new ClusterSingletons(
() -> getZkController() != null &&
getZkController().getOverseer() != null &&
@@ -945,6 +947,14 @@ public class CoreContainer {
});
clusterSingletons.setReady();
+ if
(NodeRoles.MODE_PREFERRED.equals(nodeRoles.getRoleMode(NodeRoles.Role.OVERSEER)))
{
+ try {
+ log.info("This node has been started as a preferred overseer");
+ zkSys.getZkController().setPreferredOverseer();
+ } catch (KeeperException | InterruptedException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
+ }
if (!distributedCollectionCommandRunner.isPresent()) {
zkSys.getZkController().checkOverseerDesignate();
}
diff --git a/solr/core/src/java/org/apache/solr/core/NodeRoles.java
b/solr/core/src/java/org/apache/solr/core/NodeRoles.java
new file mode 100644
index 0000000..d13b651
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/NodeRoles.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.StringUtils;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.StrUtils;
+
+import java.util.*;
+
+public class NodeRoles {
+ public static final String NODE_ROLES_PROP = "solr.node.roles";
+
+ /**
+ * Roles to be assumed on nodes that don't have roles specified for them at
startup
+ */
+ public static final String DEFAULT_ROLES_STRING = "data:on,overseer:allowed";
+
+ // Map of roles to mode that are applicable for this node.
+ private Map<Role, String> nodeRoles;
+
+ public NodeRoles(String rolesString) {
+ Map<Role, String> roles = new EnumMap<Role, String>(Role.class);
+ if (StringUtils.isEmpty(rolesString)) {
+ rolesString = DEFAULT_ROLES_STRING;
+ }
+ List<String> rolesList = StrUtils.splitSmart(rolesString, ',');
+ for (String s: rolesList) {
+ List<String> roleMode = StrUtils.splitSmart(s,':');
+ Role r = Role.getRole(roleMode.get(0));
+ String m = roleMode.get(1);
+ if (r.supportedModes().contains(m)) {
+ roles.put(r, m);
+ } else {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Unknown role mode '" + roleMode.get(1) + "' for role '" + r +
"'");
+ }
+ }
+ for(Role r: Role.values()) {
+ if (!roles.containsKey(r)) {
+ roles.put(r, r.modeWhenRoleIsAbsent());
+ }
+ }
+ nodeRoles = Collections.unmodifiableMap(roles);
+ }
+
+ public Map<Role, String> getRoles() {
+ return nodeRoles;
+ }
+
+ public String getRoleMode(Role role) {
+ return nodeRoles.get(role);
+ }
+
+ public boolean isOverseerAllowedOrPreferred() {
+ String roleMode = nodeRoles.get(Role.OVERSEER);
+ return MODE_ALLOWED.equals(roleMode) || MODE_PREFERRED.equals(roleMode);
+ }
+
+ public final static String MODE_ON = "on";
+ public final static String MODE_OFF = "off";
+ public final static String MODE_ALLOWED = "allowed";
+ public final static String MODE_PREFERRED = "preferred";
+ public final static String MODE_DISALLOWED = "disallowed";
+
+ public enum Role {
+ DATA("data") {
+ @Override
+ public Set<String> supportedModes() {
+ return Set.of(MODE_ON, MODE_OFF);
+ }
+ @Override
+ public String modeWhenRoleIsAbsent() {
+ return MODE_OFF;
+ }
+ },
+ OVERSEER("overseer") {
+ @Override
+ public Set<String> supportedModes() {
+ return Set.of(MODE_ALLOWED, MODE_PREFERRED, MODE_DISALLOWED);
+ }
+ @Override
+ public String modeWhenRoleIsAbsent() {
+ return MODE_DISALLOWED;
+ }
+ };
+
+ public final String roleName;
+
+ Role(String name) {
+ this.roleName = name;
+ }
+
+ public static Role getRole(String value) {
+ for (Role role: Role.values()) {
+ if (value.equals(role.roleName)) return role;
+ }
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown
role: " + value);
+ }
+
+ public abstract Set<String> supportedModes();
+
+ /**
+ * Default mode for a role in nodes where this role is not specified.
+ */
+ public abstract String modeWhenRoleIsAbsent();
+
+ @Override
+ public String toString() {
+ return roleName;
+ }
+ }
+
+ public static String getZNodeForRole(Role role) {
+ return ZkStateReader.NODE_ROLES + "/" + role.roleName;
+ }
+
+ public static String getZNodeForRoleMode(Role role, String mode) {
+ return ZkStateReader.NODE_ROLES + "/" + role.roleName + "/" + mode;
+ }
+
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java
b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java
index be5c7de..a40acc0 100644
--- a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java
+++ b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java
@@ -17,13 +17,14 @@
package org.apache.solr.handler;
-import java.util.HashMap;
-import java.util.Map;
+import java.io.IOException;
+import java.util.*;
import com.google.common.collect.Maps;
import org.apache.solr.api.Command;
import org.apache.solr.api.EndPoint;
import org.apache.solr.api.PayloadObj;
+import org.apache.solr.client.solrj.cloud.DistribStateManager;
import org.apache.solr.client.solrj.request.beans.ClusterPropPayload;
import org.apache.solr.client.solrj.request.beans.CreateConfigPayload;
import org.apache.solr.client.solrj.request.beans.RateLimiterPayload;
@@ -31,6 +32,7 @@ import org.apache.solr.cloud.ConfigSetCmds;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.annotation.JsonProperty;
import org.apache.solr.common.cloud.ClusterProperties;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ConfigSetParams;
@@ -39,10 +41,12 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ReflectMapWriter;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.NodeRoles;
import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.handler.admin.ConfigSetsHandler;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.zookeeper.KeeperException;
import static org.apache.solr.client.solrj.SolrRequest.METHOD.DELETE;
import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET;
@@ -57,15 +61,15 @@ import static
org.apache.solr.security.PermissionNameProvider.Name.COLL_READ_PER
import static
org.apache.solr.security.PermissionNameProvider.Name.CONFIG_EDIT_PERM;
import static
org.apache.solr.security.PermissionNameProvider.Name.CONFIG_READ_PERM;
-/** All V2 APIs that have a prefix of /api/cluster/
- *
+/**
+ * All V2 APIs that have a prefix of /api/cluster/
*/
public class ClusterAPI {
private final CollectionsHandler collectionsHandler;
private final ConfigSetsHandler configSetsHandler;
- public final Commands commands = new Commands();
- public final ConfigSetCommands configSetCommands = new ConfigSetCommands();
+ public final Commands commands = new Commands();
+ public final ConfigSetCommands configSetCommands = new ConfigSetCommands();
public ClusterAPI(CollectionsHandler ch, ConfigSetsHandler
configSetsHandler) {
this.collectionsHandler = ch;
@@ -73,8 +77,107 @@ public class ClusterAPI {
}
@EndPoint(method = GET,
- path = "/cluster/aliases",
- permission = COLL_READ_PERM)
+ path = "/cluster/node-roles",
+ permission = COLL_READ_PERM)
+ public void roles(SolrQueryRequest req, SolrQueryResponse rsp) throws
Exception {
+ rsp.add("node-roles", readRecursive(ZkStateReader.NODE_ROLES,
+
collectionsHandler.getCoreContainer().getZkController().getSolrCloudManager().getDistribStateManager(),
3));
+ }
+
+ Object readRecursive(String path, DistribStateManager zk, int depth) {
+ if (depth == 0) return null;
+ Map<String, Object> result;
+ try {
+ List<String> children = zk.listData(path);
+ if (children != null && !children.isEmpty()) {
+ result = new HashMap<>();
+ } else {
+ return Collections.emptySet();
+ }
+ for (String child: children) {
+ Object c = readRecursive(path + "/" + child, zk, depth - 1);
+ result.put(child, c);
+ }
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ if (depth == 1) {
+ return result.keySet();
+ } else {
+ return result;
+ }
+ }
+
+ @EndPoint(method = GET,
+ path = "/cluster/node-roles/role/{role}",
+ permission = COLL_READ_PERM)
+ public void nodesWithRole(SolrQueryRequest req, SolrQueryResponse rsp)
throws Exception {
+ String role = req.getPathTemplateValues().get("role");
+ rsp.add("node-roles", Map.of(role,
+ readRecursive(ZkStateReader.NODE_ROLES + "/" + role,
+
collectionsHandler.getCoreContainer().getZkController().getSolrCloudManager().getDistribStateManager(),
2)));
+ }
+
+ @EndPoint(method = GET,
+ path = "/cluster/node-roles/node/{node}",
+ permission = COLL_READ_PERM)
+ @SuppressWarnings("unchecked")
+ public void rolesForNode(SolrQueryRequest req, SolrQueryResponse rsp) throws
Exception {
+ String node = req.getPathTemplateValues().get("node");
+ Map<String, String> ret = new HashMap<String, String>();
+ Map<String, Map<String, Set<String>>> roles = (Map<String, Map<String,
Set<String>>>) readRecursive(ZkStateReader.NODE_ROLES,
+
collectionsHandler.getCoreContainer().getZkController().getSolrCloudManager().getDistribStateManager(),
3);
+ for (String role: roles.keySet()) {
+ for (String mode: roles.get(role).keySet()) {
+ if (roles.get(role).get(mode).isEmpty()) continue;
+ Set<String> nodes = roles.get(role).get(mode);
+ if (nodes.contains(node)) ret.put(role, mode);
+ }
+ }
+ for (String role: ret.keySet()) {
+ rsp.add(role, ret.get(role));
+ }
+ }
+
+ @EndPoint(method = GET,
+ path = "/cluster/node-roles/supported",
+ permission = COLL_READ_PERM)
+ public void supportedRoles(SolrQueryRequest req, SolrQueryResponse rsp)
throws Exception {
+ Map<String, Object> roleModesSupportedMap = new HashMap<>();
+ for (NodeRoles.Role role: NodeRoles.Role.values()) {
+ roleModesSupportedMap.put(role.toString(),
+ Map.of("modes", role.supportedModes()));
+ }
+ rsp.add("supported-roles", roleModesSupportedMap);
+ }
+
+ @EndPoint(method = GET,
+ path = "/cluster/node-roles/role/{role}/{mode}",
+ permission = COLL_READ_PERM)
+ public void nodesWithRoleMode(SolrQueryRequest req, SolrQueryResponse rsp)
throws Exception {
+ // Here, deal with raw strings instead of Role & Mode types so as to
handle roles and modes
+ // that are not understood by this node (possibly during a rolling upgrade)
+ String roleStr = req.getPathTemplateValues().get("role");
+ String modeStr = req.getPathTemplateValues().get("mode");
+
+ List<String> nodes =
collectionsHandler.getCoreContainer().getZkController().getSolrCloudManager()
+ .getDistribStateManager().listData(ZkStateReader.NODE_ROLES + "/"
+ roleStr + "/" + modeStr);
+ rsp.add( "node-roles", Map.of(roleStr, Collections.singletonMap(modeStr,
nodes)));
+ }
+
+ public static List<String> getNodesByRole(NodeRoles.Role role, String mode,
DistribStateManager zk)
+ throws InterruptedException, IOException, KeeperException {
+ try {
+ return zk.listData(ZkStateReader.NODE_ROLES + "/" + role + "/" + mode);
+ } catch (NoSuchElementException e) {
+ return Collections.emptyList();
+ }
+ }
+
+
+ @EndPoint(method = GET,
+ path = "/cluster/aliases",
+ permission = COLL_READ_PERM)
public void aliases(SolrQueryRequest req, SolrQueryResponse rsp) throws
Exception {
final Map<String, Object> v1Params = Maps.newHashMap();
v1Params.put(ACTION,
CollectionParams.CollectionAction.LISTALIASES.lowerName);
@@ -82,58 +185,58 @@ public class ClusterAPI {
}
@EndPoint(method = GET,
- path = "/cluster/overseer",
- permission = COLL_READ_PERM)
+ path = "/cluster/overseer",
+ permission = COLL_READ_PERM)
public void getOverseerStatus(SolrQueryRequest req, SolrQueryResponse rsp)
throws Exception {
collectionsHandler.handleRequestBody(wrapParams(req, "action",
OVERSEERSTATUS.lowerName), rsp);
}
@EndPoint(method = GET,
- path = "/cluster",
- permission = COLL_READ_PERM)
+ path = "/cluster",
+ permission = COLL_READ_PERM)
public void getCluster(SolrQueryRequest req, SolrQueryResponse rsp) throws
Exception {
collectionsHandler.handleRequestBody(wrapParams(req, "action",
LIST.lowerName), rsp);
}
@EndPoint(method = DELETE,
- path = "/cluster/command-status/{id}",
- permission = COLL_EDIT_PERM)
+ path = "/cluster/command-status/{id}",
+ permission = COLL_EDIT_PERM)
public void deleteCommandStatus(SolrQueryRequest req, SolrQueryResponse rsp)
throws Exception {
final Map<String, Object> v1Params = Maps.newHashMap();
v1Params.put(ACTION, DELETESTATUS.lowerName);
v1Params.put(REQUESTID, req.getPathTemplateValues().get("id"));
collectionsHandler.handleRequestBody(wrapParams(req, v1Params), rsp);
}
-
+
@EndPoint(method = DELETE,
- path = "/cluster/command-status",
- permission = COLL_EDIT_PERM)
+ path = "/cluster/command-status",
+ permission = COLL_EDIT_PERM)
public void flushCommandStatus(SolrQueryRequest req, SolrQueryResponse rsp)
throws Exception {
CollectionsHandler.CollectionOperation.DELETESTATUS_OP.execute(req, rsp,
collectionsHandler);
}
@EndPoint(method = DELETE,
- path = "/cluster/configs/{name}",
- permission = CONFIG_EDIT_PERM
+ path = "/cluster/configs/{name}",
+ permission = CONFIG_EDIT_PERM
)
public void deleteConfigSet(SolrQueryRequest req, SolrQueryResponse rsp)
throws Exception {
req = wrapParams(req,
- "action", ConfigSetParams.ConfigSetAction.DELETE.toString(),
- CommonParams.NAME, req.getPathTemplateValues().get("name"));
+ "action", ConfigSetParams.ConfigSetAction.DELETE.toString(),
+ CommonParams.NAME, req.getPathTemplateValues().get("name"));
configSetsHandler.handleRequestBody(req, rsp);
}
@EndPoint(method = GET,
- path = "/cluster/configs",
- permission = CONFIG_READ_PERM)
+ path = "/cluster/configs",
+ permission = CONFIG_READ_PERM)
public void listConfigSet(SolrQueryRequest req, SolrQueryResponse rsp)
throws Exception {
req = wrapParams(req, "action",
ConfigSetParams.ConfigSetAction.LIST.toString());
configSetsHandler.handleRequestBody(req, rsp);
}
@EndPoint(method = POST,
- path = "/cluster/configs",
- permission = CONFIG_EDIT_PERM
+ path = "/cluster/configs",
+ permission = CONFIG_EDIT_PERM
)
public class ConfigSetCommands {
@@ -141,8 +244,8 @@ public class ClusterAPI {
@SuppressWarnings("unchecked")
public void create(PayloadObj<CreateConfigPayload> obj) throws Exception {
Map<String, Object> mapVals = obj.get().toMap(new HashMap<>());
- Map<String,Object> customProps = obj.get().properties;
- if(customProps!= null) {
+ Map<String, Object> customProps = obj.get().properties;
+ if (customProps != null) {
customProps.forEach((k, o) ->
mapVals.put(ConfigSetCmds.CONFIG_SET_PROPERTY_PREFIX + k, o));
}
mapVals.put("action", ConfigSetParams.ConfigSetAction.CREATE.toString());
@@ -152,8 +255,8 @@ public class ClusterAPI {
}
@EndPoint(method = PUT,
- path = "/cluster/configs/{name}",
- permission = CONFIG_EDIT_PERM
+ path = "/cluster/configs/{name}",
+ permission = CONFIG_EDIT_PERM
)
public void uploadConfigSet(SolrQueryRequest req, SolrQueryResponse rsp)
throws Exception {
req = wrapParams(req,
@@ -165,8 +268,8 @@ public class ClusterAPI {
}
@EndPoint(method = PUT,
- path = "/cluster/configs/{name}/*",
- permission = CONFIG_EDIT_PERM
+ path = "/cluster/configs/{name}/*",
+ permission = CONFIG_EDIT_PERM
)
public void insertIntoConfigSet(SolrQueryRequest req, SolrQueryResponse rsp)
throws Exception {
String path = req.getPathTemplateValues().get("*");
@@ -189,17 +292,17 @@ public class ClusterAPI {
public static SolrQueryRequest wrapParams(SolrQueryRequest req, Map<String,
Object> m) {
ModifiableSolrParams solrParams = new ModifiableSolrParams();
m.forEach((k, v) -> {
- if(v == null) return;
+ if (v == null) return;
solrParams.add(k.toString(), String.valueOf(v));
});
- DefaultSolrParams dsp = new DefaultSolrParams(req.getParams(),solrParams);
+ DefaultSolrParams dsp = new DefaultSolrParams(req.getParams(), solrParams);
req.setParams(dsp);
return req;
}
@EndPoint(method = GET,
- path = "/cluster/command-status/{id}",
- permission = COLL_READ_PERM)
+ path = "/cluster/command-status/{id}",
+ permission = COLL_READ_PERM)
public void getCommandStatus(SolrQueryRequest req, SolrQueryResponse rsp)
throws Exception {
final Map<String, Object> v1Params = Maps.newHashMap();
v1Params.put(ACTION, REQUESTSTATUS.lowerName);
@@ -208,8 +311,8 @@ public class ClusterAPI {
}
@EndPoint(method = GET,
- path = "/cluster/nodes",
- permission = COLL_READ_PERM)
+ path = "/cluster/nodes",
+ permission = COLL_READ_PERM)
public void getNodes(SolrQueryRequest req, SolrQueryResponse rsp) {
rsp.add("nodes",
getCoreContainer().getZkController().getClusterState().getLiveNodes());
}
@@ -219,13 +322,13 @@ public class ClusterAPI {
}
@EndPoint(method = POST,
- path = "/cluster",
- permission = COLL_EDIT_PERM)
+ path = "/cluster",
+ permission = COLL_EDIT_PERM)
public class Commands {
@Command(name = "add-role")
public void addRole(PayloadObj<RoleInfo> obj) throws Exception {
RoleInfo info = obj.get();
- Map<String,Object> m = info.toMap(new HashMap<>());
+ Map<String, Object> m = info.toMap(new HashMap<>());
m.put("action", ADDROLE.toString());
collectionsHandler.handleRequestBody(wrapParams(obj.getRequest(), m),
obj.getResponse());
}
@@ -233,7 +336,7 @@ public class ClusterAPI {
@Command(name = "remove-role")
public void removeRole(PayloadObj<RoleInfo> obj) throws Exception {
RoleInfo info = obj.get();
- Map<String,Object> m = info.toMap(new HashMap<>());
+ Map<String, Object> m = info.toMap(new HashMap<>());
m.put("action", REMOVEROLE.toString());
collectionsHandler.handleRequestBody(wrapParams(obj.getRequest(), m),
obj.getResponse());
}
@@ -241,7 +344,7 @@ public class ClusterAPI {
@Command(name = "set-obj-property")
public void setObjProperty(PayloadObj<ClusterPropPayload> obj) {
//Not using the object directly here because the API differentiate
between {name:null} and {}
- Map<String,Object> m = obj.getDataMap();
+ Map<String, Object> m = obj.getDataMap();
ClusterProperties clusterProperties = new
ClusterProperties(getCoreContainer().getZkController().getZkClient());
try {
clusterProperties.setClusterProperties(m);
@@ -251,8 +354,8 @@ public class ClusterAPI {
}
@Command(name = "set-property")
- public void setProperty(PayloadObj<Map<String,String>> obj) throws
Exception {
- Map<String,Object> m = obj.getDataMap();
+ public void setProperty(PayloadObj<Map<String, String>> obj) throws
Exception {
+ Map<String, Object> m = obj.getDataMap();
m.put("action", CLUSTERPROP.toString());
collectionsHandler.handleRequestBody(wrapParams(obj.getRequest(), m),
obj.getResponse());
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/NodeRolesTest.java
b/solr/core/src/test/org/apache/solr/cloud/NodeRolesTest.java
new file mode 100644
index 0000000..1a573d6
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/NodeRolesTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Collection;
+import java.util.Collections;
+
+import java.util.Map;
+
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.V2Request;
+import org.apache.solr.client.solrj.response.V2Response;
+import org.apache.solr.core.NodeRoles;
+import org.junit.After;
+import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class NodeRolesTest extends SolrCloudTestCase {
+ private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @Before
+ public void setupCluster() throws Exception {
+ configureCluster(1)
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ }
+
+ @After
+ public void tearDownCluster() throws Exception {
+ shutdownCluster();
+ }
+
+ public void testRoleIntegration() throws Exception {
+ JettySolrRunner j0 = cluster.getJettySolrRunner(0);
+ testSupportedRolesAPI();
+
+ // Start a dedicated overseer node
+ JettySolrRunner j1 = startNodeWithRoles("overseer:preferred,data:off");
+ validateNodeRoles(j1.getNodeName(), "node-roles/overseer/preferred",
j1.getNodeName(), "node-roles/data/off");
+
+ V2Response rsp;
+ OverseerRolesTest.waitForNewOverseer(20, j1.getNodeName(), true);
+
+ // Start another node that is allowed or preferred overseer but has data
+ String overseerModeOnDataNode = random().nextBoolean() ? "preferred" :
"allowed";
+ JettySolrRunner j2 = startNodeWithRoles("overseer:" +
overseerModeOnDataNode + ",data:on");
+ validateNodeRoles(j2.getNodeName(), "node-roles/overseer/" +
overseerModeOnDataNode, j2.getNodeName(), "node-roles/data/on");
+
+ // validate the preferred overseers
+ validateNodeRoles(j2.getNodeName(), "node-roles/overseer/" +
overseerModeOnDataNode, j1.getNodeName(), "node-roles/overseer/preferred");
+
+ String COLLECTION_NAME = "TEST_ROLES";
+ CollectionAdminRequest
+ .createCollection(COLLECTION_NAME, "conf", 3, 1)
+ .process(cluster.getSolrClient());
+ cluster.waitForActiveCollection(COLLECTION_NAME, 3, 3);
+
+ // Assert that no replica was placed on the dedicated overseer node
+ String dedicatedOverseer = j1.getNodeName();
+
cluster.getSolrClient().getClusterStateProvider().getCollection(COLLECTION_NAME)
+ .forEachReplica((s, replica) -> assertNotEquals(replica.node,
dedicatedOverseer));
+
+ // Shutdown the dedicated overseer, make sure that node disappears from
the roles output
+ j1.stop();
+
+ // Wait and make sure that another node picks up overseer responsibilities
+ OverseerRolesTest.waitForNewOverseer(20, it ->
!dedicatedOverseer.equals(it), false);
+
+ // Make sure the stopped node no longer has the role assigned
+ rsp = new V2Request.Builder("/cluster/node-roles/role/overseer/" +
overseerModeOnDataNode).GET().build().process(cluster.getSolrClient());
+ assertFalse(((Collection) rsp._get("node-roles/overseer/" +
overseerModeOnDataNode, null)).contains(j1.getNodeName()));
+ }
+
+ @SuppressWarnings("rawtypes")
+ private void validateNodeRoles(String... nodenamePaths) throws
org.apache.solr.client.solrj.SolrServerException, java.io.IOException {
+ V2Response rsp = new
V2Request.Builder("/cluster/node-roles").GET().build().process(cluster.getSolrClient());
+ for (int i = 0; i < nodenamePaths.length; i += 2) {
+ String nodename = nodenamePaths[i];
+ String path = nodenamePaths[i + 1];
+ assertTrue("Didn't find " + nodename + " at " + path + ". Full response:
" + rsp.jsonStr(),
+ ((Collection) rsp._get(path,
Collections.emptyList())).contains(nodename));
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void testSupportedRolesAPI() throws Exception {
+ V2Response rsp = new
V2Request.Builder("/cluster/node-roles/supported").GET().build().process(cluster.getSolrClient());
+ Map<String, Object> l = (Map<String, Object>) rsp._get("supported-roles",
Collections.emptyMap());
+ assertTrue(l.containsKey("data"));
+ assertTrue(l.containsKey("overseer"));
+ }
+
+ private JettySolrRunner startNodeWithRoles(String roles) throws Exception {
+ JettySolrRunner jetty;
+ System.setProperty(NodeRoles.NODE_ROLES_PROP, roles);
+ try {
+ jetty = cluster.startJettySolrRunner();
+ } finally {
+ System.clearProperty(NodeRoles.NODE_ROLES_PROP);
+ }
+ return jetty;
+ }
+
+}
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
index c437a30..6e59f09 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
@@ -54,7 +54,7 @@ public class OverseerRolesTest extends SolrCloudTestCase {
shutdownCluster();
}
- private void waitForNewOverseer(int seconds, Predicate<String> state,
boolean failOnIntermediateTransition) throws Exception {
+ public static void waitForNewOverseer(int seconds, Predicate<String> state,
boolean failOnIntermediateTransition) throws Exception {
TimeOut timeout = new TimeOut(seconds, TimeUnit.SECONDS,
TimeSource.NANO_TIME);
String current = null;
while (timeout.hasTimedOut() == false) {
@@ -72,7 +72,7 @@ public class OverseerRolesTest extends SolrCloudTestCase {
fail("Timed out waiting for overseer state change. The current overseer
is: "+current);
}
- private void waitForNewOverseer(int seconds, String expected, boolean
failOnIntermediateTransition) throws Exception {
+ public static void waitForNewOverseer(int seconds, String expected, boolean
failOnIntermediateTransition) throws Exception {
log.info("Expecting node: {}", expected);
waitForNewOverseer(seconds, s -> Objects.equals(s, expected),
failOnIntermediateTransition);
}
diff --git
a/solr/core/src/test/org/apache/solr/cluster/placement/ClusterAbstractionsForTest.java
b/solr/core/src/test/org/apache/solr/cluster/placement/ClusterAbstractionsForTest.java
index 1260a10..adf4396 100644
---
a/solr/core/src/test/org/apache/solr/cluster/placement/ClusterAbstractionsForTest.java
+++
b/solr/core/src/test/org/apache/solr/cluster/placement/ClusterAbstractionsForTest.java
@@ -43,6 +43,11 @@ class ClusterAbstractionsForTest {
}
@Override
+ public Set<Node> getLiveDataNodes() {
+ return liveNodes;
+ }
+
+ @Override
public SolrCollection getCollection(String collectionName) {
return collections.get(collectionName);
}
diff --git a/solr/solr-ref-guide/src/node-roles.adoc
b/solr/solr-ref-guide/src/node-roles.adoc
new file mode 100644
index 0000000..71d9c6c
--- /dev/null
+++ b/solr/solr-ref-guide/src/node-roles.adoc
@@ -0,0 +1,196 @@
+= Node Roles
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+A node in Solr is usually capable of performing various types of operations,
e.g. hosting replicas, performing indexing and querying, collection management
tasks, etc. To set up a cluster where these functions are isolated to certain
dedicated nodes, we can use the concept of node roles.
+
+== Definitions
+
+=== Node role
+
+A role is a designation for a node that indicates that the node may perform a
certain function.
+
+=== Mode
+Every role has a list of modes under which a node can be. It can be simple
(e.g. `["on", "off"]`) or more granular (e.g. `["allowed", "preferred",
"disallowed"]`).
+
+== Roles
+
+In order to specify role(s) for a node, one needs to start a Solr node with
the following parameter.
+
+.Startup Parameter
+[cols="1,2,1,1"]
+|===
+|Parameter |Value |Required? | Default
+
+|solr.node.roles
+|Comma separated list of roles (in the format: `<role>:<mode>`) for this node.
+Examples: `-Dsolr.node.roles=data:on,overseer:allowed` or
`-Dsolr.node.roles=overseer:preferred`
+|No
+|`data:on,overseer:allowed`
+|===
+
+[TIP]
+====
+If a node has been started with no `solr.node.roles` parameter, it will be
assumed to have the data role turned on and overseer role allowed on it. If
you've never used roles before, you likely won't need to change anything in
your startup parameters to accommodate the functionality associated with these
roles.
+====
+
+.Supported roles
+[cols="1,1"]
+|===
+|Role |Modes
+
+|`data`
+|on, off
+
+|`overseer`
+|allowed, preferred, disallowed
+|===
+
+=== `data` role
+A node with this role (in mode "on") can host shards and replicas for
collections.
+
+=== `overseer` role
+A node with this role can perform duties of an overseer node (unless mode is
`disallowed`). When one or more nodes have the overseer role in `preferred`
mode, the overseer leader will be elected from one of these nodes. In case no
node is designated as a preferred overseer or no such node is live, the
overseer leader will be elected from one of the nodes that have the overseer
role in `allowed` mode. If all nodes that are designated with overseer role
(allowed or preferred) are down, the [...]
+
+== Example usage
+
+Sometimes, when the nodes in a cluster are under heavy querying or indexing
load, the overseer leader node might be unable to perform collection management
duties efficiently. It might be reasonable to have dedicated nodes to act as
the overseer. Such an effect can be achieved as follows:
+
+* Most nodes (data nodes) in the cluster start with
`-Dsolr.node.roles=data:on,overseer:allowed` (or with no parameter, since the
default value for `solr.node.roles` is the same).
+* One or more nodes (dedicated overseer nodes) can start with
`-Dsolr.node.roles=overseer:preferred` (or
`-Dsolr.node.roles=overseer:preferred,data:off`)
+
+In this arrangement, such dedicated nodes can be provisioned on hardware with
lesser resources like CPU, memory or disk space than other data nodes (since
these are stateless nodes) and yet the cluster will behave optimally. In case
the dedicated overseer nodes go down for some reason, the overseer leader will
be elected from one of the data nodes (since they have overseer in "allowed"
mode), and once one of the dedicated overseer nodes are back up again, it will
be re-elected for the ov [...]
+
+== Roles API
+
+=== GET /api/cluster/node-roles/supported
+
+Fetches the list of supported roles and their supported modes for this cluster.
+
+*Input*
+[source,text]
+----
+curl http://localhost:8983/api/cluster/node-roles/supported
+----
+
+*Output*
+[source,text]
+----
+{
+ "supported-roles":{
+ "data":{
+ "modes":["off",
+ "on"]
+ },
+ "overseer":{
+ "modes":["disallowed",
+ "allowed",
+ "preferred"]
+ }
+ }
+}
+----
+
+=== GET /api/cluster/node-roles
+
+Fetches the current node roles assignment for all the nodes in the cluster.
+
+*Input*
+[source,text]
+----
+curl http://localhost:8983/api/cluster/node-roles
+----
+
+*Output*
+[source,text]
+----
+{
+ "node-roles":{
+ "data":{
+ "off":["solr2:8983_solr"],
+ "on":["solr1:8983_solr"]
+ },
+ "overseer":{
+ "allowed":["solr1:8983_solr"],
+ "disallowed":[],
+ "preferred":["solr2:8983_solr"]
+ }
+ }
+}
+----
+
+=== GET /api/cluster/node-roles/role/{role}
+
+Fetches the current node roles assignment for a specified role.
+
+*Input*
+[source,text]
+----
+http://localhost:8983/api/node-roles/role/data
+----
+
+*Output*
+[source,text]
+----
+{
+ "node-roles":{
+ "data":{
+ "off":["solr2:8983_solr"],
+ "on":["solr1:8983_solr"]
+ }
+ }
+}
+----
+
+*Input*
+[source,text]
+----
+http://localhost:8983/api/node-roles/role/data/off
+----
+
+*Output*
+[source,text]
+----
+{
+ "node-roles":{
+ "data":{
+ "off":["solr2:8983_solr"]
+ }
+ }
+}
+----
+
+
+
+=== GET /api/cluster/node-roles/node/{node}
+
+Fetches the current node roles assignment for the specified node.
+
+*Input*
+[source,text]
+----
+curl http://localhost:8983/api/cluster/node-roles/node/solr1:8983_solr
+----
+
+*Output*
+[source,text]
+----
+{
+ "data":"on",
+ "overseer":"allowed"
+}
+----
diff --git a/solr/solr-ref-guide/src/solrcloud-clusters.adoc
b/solr/solr-ref-guide/src/solrcloud-clusters.adoc
index 83b3fde..03723f5 100644
--- a/solr/solr-ref-guide/src/solrcloud-clusters.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-clusters.adoc
@@ -3,6 +3,7 @@
solrcloud-recoveries-and-write-tolerance, \
solrcloud-distributed-requests, \
aliases, \
+ node-roles, \
cluster-node-management, \
shard-management, \
replica-management, \
@@ -47,6 +48,9 @@ ZooKeeper is a critical component of SolrCloud, used to
manage shard and replica
| <<solrcloud-recoveries-and-write-tolerance.adoc#,SolrCloud Recoveries and
Write Tolerance>>: Recovery in a SolrCloud cluster.
| <<solrcloud-distributed-requests.adoc#,SolrCloud Distributed Requests>>:
Query routing in a SolrCloud cluster.
| <<aliases.adoc#,Aliases>>: Alternative names for collections.
+2+^h| Cluster Management
+| <<node-roles.adoc#,Node Roles>>: Functional roles for nodes for multi-tiered
clusters.
+|
2+^h| Collections API
|<<cluster-node-management.adoc#,Cluster and Node Management>>: Cluster
management commands of the Collections API.
| <<shard-management.adoc#,Shard Management>>: Shard management commands of
the Collections API.
diff --git
a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 461dbaf..848c69e 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -103,6 +103,16 @@ public class ZkStateReader implements SolrCloseable {
public static final String STATE_TIMESTAMP_PROP = "stateTimestamp";
public static final String COLLECTIONS_ZKNODE = "/collections";
public static final String LIVE_NODES_ZKNODE = "/live_nodes";
+
+ // TODO: Deprecate and remove support for roles.json in an upcoming release.
+ /**
+ * The following, node_roles and roles.json are for assigning roles to
+ * nodes. The node_roles is the preferred way (using -Dsolr.node.roles
param),
+ * and roles.json is used by legacy ADDROLE API command.
+ */
+ public static final String NODE_ROLES = "/node_roles";
+ public static final String ROLES = "/roles.json";
+
public static final String ALIASES = "/aliases.json";
/**
* This ZooKeeper file is no longer used starting with Solr 9 but keeping
the name around to check if it
@@ -124,7 +134,6 @@ public class ZkStateReader implements SolrCloseable {
public static final String TLOG_REPLICAS = "tlogReplicas";
public static final String READ_ONLY = "readOnly";
- public static final String ROLES = "/roles.json";
public static final String CONFIGS_ZKNODE = "/configs";
public final static String CONFIGNAME_PROP = "configName";
