This is an automated email from the ASF dual-hosted git repository. ishan pushed a commit to branch jira/SOLR15694 in repository https://gitbox.apache.org/repos/asf/solr.git
commit 2716b06007ad7ff541a9c60b17965251a9420fa4 Author: Noble Paul <[email protected]> AuthorDate: Fri Dec 10 16:51:14 2021 +1100 new format --- .../apache/solr/cloud/OverseerNodePrioritizer.java | 3 +- .../java/org/apache/solr/cloud/ZkController.java | 18 ++- .../apache/solr/cloud/api/collections/Assign.java | 17 +-- .../java/org/apache/solr/core/CoreContainer.java | 6 +- .../src/java/org/apache/solr/core/NodeRole.java | 77 ---------- .../src/java/org/apache/solr/core/NodeRoles.java | 107 +++++++++++++ .../java/org/apache/solr/handler/ClusterAPI.java | 166 +++++++++++++-------- .../test/org/apache/solr/cloud/NodeRolesTest.java | 40 +++-- .../client/solrj/cloud/DistribStateManager.java | 23 --- 9 files changed, 258 insertions(+), 199 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java b/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java index f88c686..4bd21dd 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java @@ -28,6 +28,7 @@ import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.Utils; +import org.apache.solr.core.NodeRoles; import org.apache.solr.handler.ClusterAPI; import org.apache.solr.handler.component.ShardHandler; import org.apache.solr.handler.component.ShardHandlerFactory; @@ -72,7 +73,7 @@ public class OverseerNodePrioritizer { } } - overseerDesignates.addAll(ClusterAPI.getNodesByRole("overseer", new ZkDistribStateManager(zkStateReader.getZkClient()))); + overseerDesignates.addAll(ClusterAPI.getNodesByRole(NodeRoles.Role.OVERSEER, NodeRoles.PREFERRED, new ZkDistribStateManager(zkStateReader.getZkClient()))); if (overseerDesignates.isEmpty()) return; String ldr = OverseerTaskProcessor.getLeaderNode(zk); if(overseerDesignates.contains(ldr)) return; diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 05c6c5b..e7b15bf 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -73,12 +73,7 @@ import org.apache.solr.common.util.SolrNamedThreadFactory; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.URLUtil; import org.apache.solr.common.util.Utils; -import org.apache.solr.core.CloseHook; -import org.apache.solr.core.CloudConfig; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.CoreDescriptor; -import org.apache.solr.core.SolrCore; -import org.apache.solr.core.SolrCoreInitializationException; +import org.apache.solr.core.*; import org.apache.solr.handler.component.HttpShardHandler; import org.apache.solr.logging.MDCLoggingContext; import org.apache.solr.search.SolrIndexSearcher; @@ -855,6 +850,13 @@ public class ZkController implements Closeable { ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(zkClient.getZkClientTimeout()); cmdExecutor.ensureExists(ZkStateReader.LIVE_NODES_ZKNODE, zkClient); cmdExecutor.ensureExists(ZkStateReader.NODE_ROLES, zkClient); + for (NodeRoles.Role role : NodeRoles.Role.values()) { + cmdExecutor.ensureExists(ZkStateReader.NODE_ROLES + "/" + role.roleName, zkClient); + for (String v : role.supportedVals()) { + cmdExecutor.ensureExists(ZkStateReader.NODE_ROLES + "/" + role.roleName + "/"+v, zkClient); + } + } + cmdExecutor.ensureExists(ZkStateReader.COLLECTIONS_ZKNODE, zkClient); cmdExecutor.ensureExists(ZkStateReader.ALIASES, zkClient); byte[] emptyJson = "{}".getBytes(StandardCharsets.UTF_8); @@ -1089,8 +1091,8 @@ public class ZkController implements Closeable { ops.add(Op.create(nodePath, null, zkClient.getZkACLProvider().getACLsToAdd(nodePath), CreateMode.EPHEMERAL)); // Create the roles node as well - ops.add(Op.create(ZkStateReader.NODE_ROLES + "/" + nodeName, - Utils.toJSON(cc.nodeRoles), zkClient.getZkACLProvider().getACLsToAdd(nodePath), CreateMode.EPHEMERAL)); + cc.nodeRoles.ROLES.forEach((role, val) -> ops.add(Op.create(ZkStateReader.NODE_ROLES + "/" + role.roleName+ "/"+val +"/"+ nodeName, + null, zkClient.getZkACLProvider().getACLsToAdd(nodePath), CreateMode.EPHEMERAL))); zkClient.multi(ops, true); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java index 5a74d78..185acb3 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java @@ -51,9 +51,9 @@ import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.StrUtils; -import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.NodeRole; +import org.apache.solr.core.NodeRoles; +import org.apache.solr.handler.ClusterAPI; import org.apache.solr.util.NumberUtils; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -256,15 +256,10 @@ public class Assign { public static void filterNonDataNodes(DistribStateManager zk, List<String> liveNodes) { try { - zk.forEachChild(ZkStateReader.NODE_ROLES, (name, data) -> { - if (data != null && data.getData() != null && data.getData().length > 0) { - @SuppressWarnings("unchecked") - List<String> roles = (List<String>) Utils.fromJSON(data.getData()); - if (!roles.contains(NodeRole.Role.DATA.toString())) { - liveNodes.remove(name); - } - } - }); + List<String> noData = ClusterAPI.getNodesByRole(NodeRoles.Role.DATA, NodeRoles.OFF, zk); + if(!noData.isEmpty()){ + liveNodes.removeAll(noData); + } } catch (Exception e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Problem fetching roles from Zookeeper", e); } diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 44c63db..f7e9a1f 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -246,7 +246,7 @@ public class CoreContainer { private final ObjectCache objectCache = new ObjectCache(); - public final NodeRole nodeRoles = new NodeRole(System.getProperty(NodeRole.NODE_ROLES_PROP)); + public final NodeRoles nodeRoles = new NodeRoles(System.getProperty(NodeRoles.NODE_ROLES_PROP)); private final ClusterSingletons clusterSingletons = new ClusterSingletons( () -> getZkController() != null && @@ -947,9 +947,9 @@ public class CoreContainer { }); clusterSingletons.setReady(); - if (nodeRoles.getRoles().contains(NodeRole.Role.OVERSEER)) { + if (NodeRoles.PREFERRED.equals(nodeRoles.getRoleVal(NodeRoles.Role.OVERSEER))) { try { - log.info("This node is started as an overseer"); + log.info("This node is started as a preferred overseer"); zkSys.getZkController().setPreferredOverseer(); } catch (KeeperException | InterruptedException e) { throw new SolrException(ErrorCode.SERVER_ERROR, e); diff --git a/solr/core/src/java/org/apache/solr/core/NodeRole.java b/solr/core/src/java/org/apache/solr/core/NodeRole.java deleted file mode 100644 index a4cf757..0000000 --- a/solr/core/src/java/org/apache/solr/core/NodeRole.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.core; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.*; - -import org.apache.solr.common.IteratorWriter; -import org.apache.solr.common.StringUtils; -import org.apache.solr.common.util.StrUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class NodeRole implements IteratorWriter { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final String NODE_ROLES_PROP = "solr.node.roles"; - - private final Set<Role> roles; - - public NodeRole(String role) { - if (StringUtils.isEmpty(role)) { - // if no roles were specified, assume "data" role for backcompat reasons - roles = Set.of(Role.DATA); - return; - } - Set<String> rolesSet = new HashSet<>(StrUtils.split(role, ',')); - if (rolesSet.isEmpty()) { - // if no roles were specified, assume "data" role for backcompat reasons - roles = Set.of(Role.DATA); - return; - } - roles = new TreeSet<>(); - for (String r: rolesSet) { - roles.add(Role.valueOfCaseInsensitive(r)); - } - } - - public Set<Role> getRoles() { - return roles; - } - - @Override - public void writeIter(ItemWriter iw) throws IOException { - for (Role role: roles) iw.add(role.toString()); - } - - public enum Role { - DATA, OVERSEER; - - public static Role valueOfCaseInsensitive(String value) { - // Given a user string "overseer", convert to OVERSEER and return the enum value - String canonicalValue = value.toUpperCase().replace(' ', '_'); - return Role.valueOf(canonicalValue); - } - - @Override - public String toString() { - return super.toString().toLowerCase(); - } - } -} diff --git a/solr/core/src/java/org/apache/solr/core/NodeRoles.java b/solr/core/src/java/org/apache/solr/core/NodeRoles.java new file mode 100644 index 0000000..fb3f011 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/NodeRoles.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.core; + +import java.lang.invoke.MethodHandles; +import java.util.*; +import org.apache.solr.common.MapWriter; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.StringUtils; +import org.apache.solr.common.util.StrUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class NodeRoles implements MapWriter { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static final String NODE_ROLES_PROP = "solr.node.roles"; + + public Map<Role, String> ROLES ; + + public NodeRoles(String role) { + ROLES = new EnumMap<>(Role.class); + if (StringUtils.isEmpty(role)) { + role = DEFAULT_ROLE; + } + List<String> rolesList = StrUtils.splitSmart(role, ','); + for (String s : rolesList) { + List<String> roleVal = StrUtils.splitSmart(s,':'); + Role r = Role.getRole(roleVal.get(0)); + if(r.supportedVals().contains(roleVal.get(1))) { + ROLES.put(r, roleVal.get(1)); + } else { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "UNKNOWN role value :"+roleVal.get(0)); + } + } + ROLES = Collections.unmodifiableMap(ROLES); + } + + public String getRoleVal(Role role) { + return ROLES.get(role); + } + + + + @Override + public void writeMap(EntryWriter ew) { + ROLES.forEach((role, s) -> ew.putNoEx(role.roleName, s)); + } + + public enum Role { + DATA("data") , + OVERSEER("overseer") { + @Override + public Set<String> supportedVals() { + return OVERSEER_VALS; + } + }, + COORDINATOR("coordinator"); + + public final String roleName; + + Role(String name) { + this.roleName = name; + } + + public static Role getRole(String value) { + // Given a user string "overseer", convert to OVERSEER and return the enum value + for (Role role : Role.values()) { + if(value.equals(role.roleName)) return role; + } + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown role : "+value); + } + + public Set<String> supportedVals() { + return ON_OFF; + + } + + @Override + public String toString() { + return super.toString().toLowerCase(); + } + } + + public static final String ON = "on"; + public static final String OFF = "off"; + public static final Set<String> ON_OFF = Set.of(ON,OFF); + public static final String ALLOWED = "allowed"; + public static final String DISALLOWED = "disallowed"; + public static final String PREFERRED = "preferred"; + public static final Set<String> OVERSEER_VALS = Set.of(ALLOWED, DISALLOWED, PREFERRED); + public static final String DEFAULT_ROLE = "data:on,overseer:allowed"; +} diff --git a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java index 5f512d2..796ac49 100644 --- a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java @@ -42,6 +42,7 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.ReflectMapWriter; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.NodeRoles; import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.handler.admin.ConfigSetsHandler; import org.apache.solr.request.SolrQueryRequest; @@ -63,20 +64,21 @@ import static org.apache.solr.security.PermissionNameProvider.Name.COLL_READ_PER import static org.apache.solr.security.PermissionNameProvider.Name.CONFIG_EDIT_PERM; import static org.apache.solr.security.PermissionNameProvider.Name.CONFIG_READ_PERM; -/** All V2 APIs that have a prefix of /api/cluster/ - * +/** + * All V2 APIs that have a prefix of /api/cluster/ */ public class ClusterAPI { private final CollectionsHandler collectionsHandler; private final ConfigSetsHandler configSetsHandler; - public final Commands commands = new Commands(); - public final ConfigSetCommands configSetCommands = new ConfigSetCommands(); + public final Commands commands = new Commands(); + public final ConfigSetCommands configSetCommands = new ConfigSetCommands(); public ClusterAPI(CollectionsHandler ch, ConfigSetsHandler configSetsHandler) { this.collectionsHandler = ch; this.configSetsHandler = configSetsHandler; } + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @EndPoint(method = GET, @@ -85,14 +87,45 @@ public class ClusterAPI { @SuppressWarnings("unchecked") // nocommit: it must also output all data nodes that didn't start with -Dsolr.node.roles public void roles(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { - Map <String, List<String>> result = new LinkedHashMap<>(); - collectionsHandler.getCoreContainer().getZkController().getSolrCloudManager().getDistribStateManager(). - forEachChild(ZkStateReader.NODE_ROLES, (node, data) -> { - if (data != null && data.getData() != null) { - result.put(node, ((List<String>) Utils.fromJSON(data.getData()))); + Map<String, Object> result = new LinkedHashMap<>(); + + rsp.add("node-roles", readRecursive(ZkStateReader.NODE_ROLES, collectionsHandler.getCoreContainer().getZkController().getSolrCloudManager().getDistribStateManager(), 3)); + } + + Object readRecursive(String path, DistribStateManager zk, int depth) { + if (depth == 0) return null; + Map<String, Object> result = null; + boolean hasSubValues = false; + try { + List<String> children = zk.listData(path); + if (children != null && !children.isEmpty()) { + result = new HashMap<>(); + } else { + return depth >=1 ? Collections.emptyList(): null; } - }); - rsp.add("node-roles", result); + for (String child : children) { + Object c = readRecursive(path + "/" + child, zk, depth - 1); + if (c != null) { + hasSubValues = true; + } + result.put(child, c); + } + + + } catch (Exception e) { + } + + + + if (result == null) { + return null; + } + if (depth == 1) { + return result.keySet(); + } else { + return result; + } + } @EndPoint(method = GET, @@ -100,28 +133,35 @@ public class ClusterAPI { permission = COLL_READ_PERM) public void nodesWithRole(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { String role = req.getPathTemplateValues().get("role"); - rsp.add("nodes", - getNodesByRole(role, collectionsHandler.getCoreContainer().getZkController().getSolrCloudManager().getDistribStateManager())); + rsp.add(role, + readRecursive(ZkStateReader.NODE_ROLES + "/"+ role, + collectionsHandler.getCoreContainer().getZkController().getSolrCloudManager().getDistribStateManager(), 2)); + } + @EndPoint(method = GET, + path = "/cluster/node-roles/{role}/{role-val}", + permission = COLL_READ_PERM) + public void nodesWithRoleVal(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { + String role = req.getPathTemplateValues().get("role"); + String roleVal = req.getPathTemplateValues().get("role-val"); + List<String> nodes = collectionsHandler.getCoreContainer().getZkController().getSolrCloudManager() + .getDistribStateManager().listData(ZkStateReader.NODE_ROLES + "/"+ role+"/"+roleVal); + rsp.add( role, Collections.singletonMap(roleVal, nodes)); } - public static List<String> getNodesByRole(String role, DistribStateManager zk) throws InterruptedException, IOException, KeeperException { - List<String> result = new ArrayList<>(); - zk.forEachChild(ZkStateReader.NODE_ROLES, (node, data) -> { - if (data != null && data.getData() != null) { - @SuppressWarnings("unchecked") - List<String> rolesData = (List<String>) Utils.fromJSON(data.getData()); - if (rolesData.contains(role)) { - result.add(node); - } - } - }); - return result; + public static List<String> getNodesByRole(NodeRoles.Role role, String val, DistribStateManager zk) + throws InterruptedException, IOException, KeeperException { + try { + return zk.listData(ZkStateReader.NODE_ROLES+"/"+ role+ "/"+val); + } catch (NoSuchElementException e) { + return Collections.emptyList(); + } + } @EndPoint(method = GET, - path = "/cluster/aliases", - permission = COLL_READ_PERM) + path = "/cluster/aliases", + permission = COLL_READ_PERM) public void aliases(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { final Map<String, Object> v1Params = Maps.newHashMap(); v1Params.put(ACTION, CollectionParams.CollectionAction.LISTALIASES.lowerName); @@ -129,58 +169,58 @@ public class ClusterAPI { } @EndPoint(method = GET, - path = "/cluster/overseer", - permission = COLL_READ_PERM) + path = "/cluster/overseer", + permission = COLL_READ_PERM) public void getOverseerStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { collectionsHandler.handleRequestBody(wrapParams(req, "action", OVERSEERSTATUS.lowerName), rsp); } @EndPoint(method = GET, - path = "/cluster", - permission = COLL_READ_PERM) + path = "/cluster", + permission = COLL_READ_PERM) public void getCluster(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { collectionsHandler.handleRequestBody(wrapParams(req, "action", LIST.lowerName), rsp); } @EndPoint(method = DELETE, - path = "/cluster/command-status/{id}", - permission = COLL_EDIT_PERM) + path = "/cluster/command-status/{id}", + permission = COLL_EDIT_PERM) public void deleteCommandStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { final Map<String, Object> v1Params = Maps.newHashMap(); v1Params.put(ACTION, DELETESTATUS.lowerName); v1Params.put(REQUESTID, req.getPathTemplateValues().get("id")); collectionsHandler.handleRequestBody(wrapParams(req, v1Params), rsp); } - + @EndPoint(method = DELETE, - path = "/cluster/command-status", - permission = COLL_EDIT_PERM) + path = "/cluster/command-status", + permission = COLL_EDIT_PERM) public void flushCommandStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { CollectionsHandler.CollectionOperation.DELETESTATUS_OP.execute(req, rsp, collectionsHandler); } @EndPoint(method = DELETE, - path = "/cluster/configs/{name}", - permission = CONFIG_EDIT_PERM + path = "/cluster/configs/{name}", + permission = CONFIG_EDIT_PERM ) public void deleteConfigSet(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { req = wrapParams(req, - "action", ConfigSetParams.ConfigSetAction.DELETE.toString(), - CommonParams.NAME, req.getPathTemplateValues().get("name")); + "action", ConfigSetParams.ConfigSetAction.DELETE.toString(), + CommonParams.NAME, req.getPathTemplateValues().get("name")); configSetsHandler.handleRequestBody(req, rsp); } @EndPoint(method = GET, - path = "/cluster/configs", - permission = CONFIG_READ_PERM) + path = "/cluster/configs", + permission = CONFIG_READ_PERM) public void listConfigSet(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { req = wrapParams(req, "action", ConfigSetParams.ConfigSetAction.LIST.toString()); configSetsHandler.handleRequestBody(req, rsp); } @EndPoint(method = POST, - path = "/cluster/configs", - permission = CONFIG_EDIT_PERM + path = "/cluster/configs", + permission = CONFIG_EDIT_PERM ) public class ConfigSetCommands { @@ -188,8 +228,8 @@ public class ClusterAPI { @SuppressWarnings("unchecked") public void create(PayloadObj<CreateConfigPayload> obj) throws Exception { Map<String, Object> mapVals = obj.get().toMap(new HashMap<>()); - Map<String,Object> customProps = obj.get().properties; - if(customProps!= null) { + Map<String, Object> customProps = obj.get().properties; + if (customProps != null) { customProps.forEach((k, o) -> mapVals.put(ConfigSetCmds.CONFIG_SET_PROPERTY_PREFIX + k, o)); } mapVals.put("action", ConfigSetParams.ConfigSetAction.CREATE.toString()); @@ -199,8 +239,8 @@ public class ClusterAPI { } @EndPoint(method = PUT, - path = "/cluster/configs/{name}", - permission = CONFIG_EDIT_PERM + path = "/cluster/configs/{name}", + permission = CONFIG_EDIT_PERM ) public void uploadConfigSet(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { req = wrapParams(req, @@ -212,8 +252,8 @@ public class ClusterAPI { } @EndPoint(method = PUT, - path = "/cluster/configs/{name}/*", - permission = CONFIG_EDIT_PERM + path = "/cluster/configs/{name}/*", + permission = CONFIG_EDIT_PERM ) public void insertIntoConfigSet(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { String path = req.getPathTemplateValues().get("*"); @@ -236,17 +276,17 @@ public class ClusterAPI { public static SolrQueryRequest wrapParams(SolrQueryRequest req, Map<String, Object> m) { ModifiableSolrParams solrParams = new ModifiableSolrParams(); m.forEach((k, v) -> { - if(v == null) return; + if (v == null) return; solrParams.add(k.toString(), String.valueOf(v)); }); - DefaultSolrParams dsp = new DefaultSolrParams(req.getParams(),solrParams); + DefaultSolrParams dsp = new DefaultSolrParams(req.getParams(), solrParams); req.setParams(dsp); return req; } @EndPoint(method = GET, - path = "/cluster/command-status/{id}", - permission = COLL_READ_PERM) + path = "/cluster/command-status/{id}", + permission = COLL_READ_PERM) public void getCommandStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { final Map<String, Object> v1Params = Maps.newHashMap(); v1Params.put(ACTION, REQUESTSTATUS.lowerName); @@ -255,8 +295,8 @@ public class ClusterAPI { } @EndPoint(method = GET, - path = "/cluster/nodes", - permission = COLL_READ_PERM) + path = "/cluster/nodes", + permission = COLL_READ_PERM) public void getNodes(SolrQueryRequest req, SolrQueryResponse rsp) { rsp.add("nodes", getCoreContainer().getZkController().getClusterState().getLiveNodes()); } @@ -266,13 +306,13 @@ public class ClusterAPI { } @EndPoint(method = POST, - path = "/cluster", - permission = COLL_EDIT_PERM) + path = "/cluster", + permission = COLL_EDIT_PERM) public class Commands { @Command(name = "add-role") public void addRole(PayloadObj<RoleInfo> obj) throws Exception { RoleInfo info = obj.get(); - Map<String,Object> m = info.toMap(new HashMap<>()); + Map<String, Object> m = info.toMap(new HashMap<>()); m.put("action", ADDROLE.toString()); collectionsHandler.handleRequestBody(wrapParams(obj.getRequest(), m), obj.getResponse()); } @@ -280,7 +320,7 @@ public class ClusterAPI { @Command(name = "remove-role") public void removeRole(PayloadObj<RoleInfo> obj) throws Exception { RoleInfo info = obj.get(); - Map<String,Object> m = info.toMap(new HashMap<>()); + Map<String, Object> m = info.toMap(new HashMap<>()); m.put("action", REMOVEROLE.toString()); collectionsHandler.handleRequestBody(wrapParams(obj.getRequest(), m), obj.getResponse()); } @@ -288,7 +328,7 @@ public class ClusterAPI { @Command(name = "set-obj-property") public void setObjProperty(PayloadObj<ClusterPropPayload> obj) { //Not using the object directly here because the API differentiate between {name:null} and {} - Map<String,Object> m = obj.getDataMap(); + Map<String, Object> m = obj.getDataMap(); ClusterProperties clusterProperties = new ClusterProperties(getCoreContainer().getZkController().getZkClient()); try { clusterProperties.setClusterProperties(m); @@ -298,8 +338,8 @@ public class ClusterAPI { } @Command(name = "set-property") - public void setProperty(PayloadObj<Map<String,String>> obj) throws Exception { - Map<String,Object> m = obj.getDataMap(); + public void setProperty(PayloadObj<Map<String, String>> obj) throws Exception { + Map<String, Object> m = obj.getDataMap(); m.put("action", CLUSTERPROP.toString()); collectionsHandler.handleRequestBody(wrapParams(obj.getRequest(), m), obj.getResponse()); } diff --git a/solr/core/src/test/org/apache/solr/cloud/NodeRolesTest.java b/solr/core/src/test/org/apache/solr/cloud/NodeRolesTest.java index e5f2ee1..18a16fe 100644 --- a/solr/core/src/test/org/apache/solr/cloud/NodeRolesTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/NodeRolesTest.java @@ -18,14 +18,14 @@ package org.apache.solr.cloud; import java.lang.invoke.MethodHandles; -import java.util.List; -import java.util.TreeSet; +import java.util.Collection; +import java.util.Collections; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.response.V2Response; -import org.apache.solr.core.NodeRole; +import org.apache.solr.core.NodeRoles; import org.junit.After; import org.junit.Before; import org.slf4j.Logger; @@ -51,20 +51,34 @@ public class NodeRolesTest extends SolrCloudTestCase { JettySolrRunner j0 = cluster.getJettySolrRunner(0); JettySolrRunner j1 = null, j2 = null; - j1 = startNodeWithRoles("overseer"); + j1 = startNodeWithRoles("overseer:preferred,data:off"); V2Response rsp = new V2Request.Builder("/cluster/node-roles").GET().build().process(cluster.getSolrClient()); - assertEquals(List.of("overseer"), rsp._get(List.of("node-roles", j1.getNodeName()), null)); - + assertEquals(j1.getNodeName(), rsp._getStr("node-roles/overseer/preferred[0]", null)); + assertEquals(j1.getNodeName(), rsp._getStr("node-roles/data/off[0]", null)); OverseerRolesTest.waitForNewOverseer(20, j1.getNodeName(), false); //start another node that is overseer but has data - j2 = startNodeWithRoles("overseer,data"); + j2 = startNodeWithRoles("overseer:preferred,data:on"); rsp = new V2Request.Builder("/cluster/node-roles").GET().build().process(cluster.getSolrClient()); - assertEquals(List.of("data", "overseer"), rsp._get(List.of("node-roles", j2.getNodeName()), null)); + + + assertTrue( ((Collection)rsp._get("node-roles/overseer/preferred", Collections.emptyList())).contains(j2.getNodeName())); + assertTrue( ((Collection)rsp._get("node-roles/data/on", Collections.emptyList())).contains(j2.getNodeName())); + + rsp = new V2Request.Builder("/cluster/node-roles/overseer").GET().build().process(cluster.getSolrClient()); - assertEquals(new TreeSet<String>(List.of(j1.getNodeName(), j2.getNodeName())), new TreeSet<String>((List<String>) rsp._get("nodes", null))); + + + assertTrue( ((Collection)rsp._get("overseer/preferred", Collections.emptyList())).contains(j2.getNodeName())); + assertTrue( ((Collection)rsp._get("overseer/preferred", Collections.emptyList())).contains(j1.getNodeName())); + + rsp = new V2Request.Builder("/cluster/node-roles/overseer/preferred").GET().build().process(cluster.getSolrClient()); + assertTrue( ((Collection)rsp._get("overseer/preferred", Collections.emptyList())).contains(j2.getNodeName())); + assertTrue( ((Collection)rsp._get("overseer/preferred", Collections.emptyList())).contains(j1.getNodeName())); + + String COLLECTION_NAME = "TEST_ROLES"; CollectionAdminRequest @@ -79,17 +93,17 @@ public class NodeRolesTest extends SolrCloudTestCase { // Shutdown the dedicated overseer, make sure that node disappears from the roles output j1.stop(); - rsp = new V2Request.Builder("/cluster/node-roles").GET().build().process(cluster.getSolrClient()); - assertNull(rsp._get(List.of("node-roles", dedicatedOverseer), null)); + rsp = new V2Request.Builder("/cluster/node-roles/overseer/preferred").GET().build().process(cluster.getSolrClient()); + assertFalse (((Collection) rsp._get("overseer/preferred" , null)). contains(j1.getNodeName())); } private JettySolrRunner startNodeWithRoles(String roles) throws Exception { JettySolrRunner jetty; - System.setProperty(NodeRole.NODE_ROLES_PROP, roles); + System.setProperty(NodeRoles.NODE_ROLES_PROP, roles); try { jetty = cluster.startJettySolrRunner(); } finally { - System.clearProperty(NodeRole.NODE_ROLES_PROP); + System.clearProperty(NodeRoles.NODE_ROLES_PROP); } return jetty; } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DistribStateManager.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DistribStateManager.java index 84010c8..feae27f 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DistribStateManager.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DistribStateManager.java @@ -24,10 +24,8 @@ import java.util.LinkedList; import java.util.List; import java.util.NoSuchElementException; -import java.util.function.BiConsumer; import org.apache.solr.common.SolrCloseable; import org.apache.solr.common.cloud.PerReplicaStates; -import org.apache.solr.common.cloud.ZkStateReader; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.Op; @@ -143,25 +141,4 @@ public interface DistribStateManager extends SolrCloseable { } } } - default void forEachChild(String path, BiConsumer<String, VersionedData> fun) - throws InterruptedException, IOException, KeeperException { - List<String> children = null; - try { - children = listData(path); - } catch (NoSuchElementException e) { - return; - } - if (!children.isEmpty()) { - for (String node: children) { - VersionedData data = null; - try { - data = getData(path + "/" + node); - } catch (NoSuchElementException e) { - // this node must've been deleted - continue; - } - fun.accept(node, data); - } - } - } }
