This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new dae9a938241 SOLR-17582: Stream CLUSTERSTATUS API for SolrJ version >=
9.9 (#3156)
dae9a938241 is described below
commit dae9a93824124805b6b96700dab6037a737f40eb
Author: Matthew Biscocho <54160956+mlbis...@users.noreply.github.com>
AuthorDate: Tue Feb 4 21:18:07 2025 -0500
SOLR-17582: Stream CLUSTERSTATUS API for SolrJ version >= 9.9 (#3156)
Or if unknown (e.g. some misc. HTTP/JSON).
Co-authored-by: Matthew Biscocho <mbisco...@bloomberg.net>
(cherry picked from commit 5e328ee77e5eeee858f4112e02652ddd4c8e14b6)
---
.../src/java/org/apache/solr/cli/StatusTool.java | 2 +-
.../apache/solr/handler/admin/ClusterStatus.java | 35 +++++------
.../solr/handler/admin/CollectionsHandler.java | 2 +-
.../cloud/api/collections/TestCollectionAPI.java | 32 +++++-----
.../solrj/impl/BaseHttpClusterStateProvider.java | 15 ++---
.../solrj/impl/ClusterStateProviderTest.java | 70 +++++++++++++++++++++-
6 files changed, 110 insertions(+), 46 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cli/StatusTool.java
b/solr/core/src/java/org/apache/solr/cli/StatusTool.java
index 3e2368a8d6a..582143f713b 100644
--- a/solr/core/src/java/org/apache/solr/cli/StatusTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/StatusTool.java
@@ -372,7 +372,7 @@ public class StatusTool extends ToolBase {
cloudStatus.put("liveNodes", String.valueOf(liveNodes.size()));
// TODO get this as a metric from the metrics API instead, or something
else.
- var collections = (NamedList<Object>) json.findRecursive("cluster",
"collections");
+ var collections = (Map<String, Object>) json.findRecursive("cluster",
"collections");
cloudStatus.put("collections", String.valueOf(collections.size()));
return cloudStatus;
diff --git
a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java
b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java
index 7a8ecf9c850..6ef549698d3 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Stream;
+import org.apache.solr.client.api.util.SolrVersion;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Aliases;
@@ -34,7 +35,6 @@ import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.PerReplicaStates;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
@@ -101,7 +101,7 @@ public class ClusterStatus {
collection = params.get(ZkStateReader.COLLECTION_PROP);
}
- public void getClusterStatus(NamedList<Object> results)
+ public void getClusterStatus(NamedList<Object> results, SolrVersion
solrVersion)
throws KeeperException, InterruptedException {
NamedList<Object> clusterStatus = new SimpleOrderedMap<>();
@@ -127,7 +127,7 @@ public class ClusterStatus {
if (withCollection) {
assert liveNodes != null;
- fetchClusterStatusForCollOrAlias(clusterStatus, liveNodes, aliases);
+ fetchClusterStatusForCollOrAlias(clusterStatus, liveNodes, aliases,
solrVersion);
}
if (withAliases) {
@@ -158,7 +158,10 @@ public class ClusterStatus {
}
private void fetchClusterStatusForCollOrAlias(
- NamedList<Object> clusterStatus, List<String> liveNodes, Aliases
aliases) {
+ NamedList<Object> clusterStatus,
+ List<String> liveNodes,
+ Aliases aliases,
+ SolrVersion solrVersion) {
// read aliases
Map<String, List<String>> collectionVsAliases = new HashMap<>();
@@ -206,19 +209,7 @@ public class ClusterStatus {
}
}
- // Because of back-compat for SolrJ, create the whole response into a
NamedList
- // Otherwise stream with MapWriter to save memory
- if (CommonParams.JAVABIN.equals(solrParams.get(CommonParams.WT))) {
- NamedList<Object> collectionProps = new SimpleOrderedMap<>();
- collectionStream.forEach(
- collectionState -> {
- collectionProps.add(
- collectionState.getName(),
- buildResponseForCollection(
- collectionState, collectionVsAliases, routeKey, liveNodes,
requestedShards));
- });
- clusterStatus.add("collections", collectionProps);
- } else {
+ if (solrVersion == null ||
solrVersion.greaterThanOrEqualTo(SolrVersion.valueOf("9.9.0"))) {
MapWriter collectionPropsWriter =
ew -> {
collectionStream.forEach(
@@ -234,6 +225,16 @@ public class ClusterStatus {
});
};
clusterStatus.add("collections", collectionPropsWriter);
+ } else {
+ NamedList<Object> collectionProps = new SimpleOrderedMap<>();
+ collectionStream.forEach(
+ collectionState -> {
+ collectionProps.add(
+ collectionState.getName(),
+ buildResponseForCollection(
+ collectionState, collectionVsAliases, routeKey, liveNodes,
requestedShards));
+ });
+ clusterStatus.add("collections", collectionProps);
}
}
diff --git
a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index 330cd7c729b..c61951c76ac 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -979,7 +979,7 @@ public class CollectionsHandler extends RequestHandlerBase
implements Permission
CLUSTERSTATUS,
(req, rsp, h) -> {
new
ClusterStatus(h.coreContainer.getZkController().getZkStateReader(),
req.getParams())
- .getClusterStatus(rsp.getValues());
+ .getClusterStatus(rsp.getValues(),
req.getHttpSolrCall().getUserAgentSolrVersion());
return null;
}),
ADDREPLICAPROP_OP(
diff --git
a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java
b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java
index c286c15c016..6a161de5182 100644
---
a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java
+++
b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java
@@ -135,10 +135,11 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
.getResponse();
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- NamedList<?> collections = (NamedList<?>) cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertEquals(1, collections.size());
- assertEquals("25", collections._getStr(List.of(COLLECTION_NAME,
"replicationFactor"), null));
+ Map<?, ?> collectionProperties = (Map<?, ?>)
collections.get(COLLECTION_NAME);
+ assertEquals("25", collectionProperties.get("replicationFactor"));
params = new ModifiableSolrParams();
params.set("action",
CollectionParams.CollectionAction.MODIFYCOLLECTION.toString());
@@ -157,10 +158,11 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
System.out.println(rsp);
cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- collections = (NamedList<?>) cluster.get("collections");
+ collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertEquals(1, collections.size());
- assertNull(collections._getStr(List.of(COLLECTION_NAME,
"replicationFactor"), null));
+ collectionProperties = (Map<?, ?>) collections.get(COLLECTION_NAME);
+ assertNull(collectionProperties.get("replicationFactor"));
params = new ModifiableSolrParams();
params.set("action",
CollectionParams.CollectionAction.MODIFYCOLLECTION.toString());
@@ -259,7 +261,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
NamedList<?> rsp = client.request(req);
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- NamedList<?> collections = (NamedList<?>) cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertNotNull(
"Testing to insure collections are returned",
collections.get(COLLECTION_NAME1));
@@ -286,7 +288,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
NamedList<Object> rsp = client.request(request);
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- NamedList<?> collections = (NamedList<?>) cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertEquals(1, collections.size());
@SuppressWarnings({"unchecked"})
@@ -308,7 +310,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
NamedList<Object> rsp = client.request(request);
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- NamedList<?> collections = (NamedList<?>) cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertNotNull(collections.get(COLLECTION_NAME));
assertEquals(1, collections.size());
@@ -334,7 +336,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
NamedList<Object> rsp = request.process(client).getResponse();
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- NamedList<?> collections = (NamedList<?>) cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertNotNull(collections.get(COLLECTION_NAME));
assertEquals(1, collections.size());
@@ -469,7 +471,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
NamedList<Object> rsp = client.request(request);
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- NamedList<?> collections = (NamedList<?>) cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertNotNull(collections.get(COLLECTION_NAME1));
assertEquals(4, collections.size());
@@ -491,7 +493,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
NamedList<Object> rsp = client.request(request);
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- NamedList<?> collections = (NamedList<?>) cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertEquals(1, collections.size());
@SuppressWarnings({"unchecked"})
@@ -521,7 +523,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
NamedList<Object> rsp = client.request(request);
NamedList<Object> cluster = (NamedList<Object>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- NamedList<Object> collections = (NamedList<Object>)
cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertEquals(1, collections.size());
Map<String, Object> collection = (Map<String, Object>)
collections.get(cname);
@@ -537,7 +539,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
rsp = client.request(request);
cluster = (NamedList<Object>) rsp.get("cluster");
- collections = (NamedList<Object>) cluster.get("collections");
+ collections = (Map<?, ?>) cluster.get("collections");
collection = (Map<String, Object>) collections.get(cname);
Integer newVersion = (Integer) collection.get("znodeVersion");
assertNotNull(newVersion);
@@ -581,7 +583,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
NamedList<Object> cluster = (NamedList<Object>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
@SuppressWarnings({"unchecked"})
- NamedList<Object> collections = (NamedList<Object>)
cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertNotNull(collections.get(DEFAULT_COLLECTION));
assertEquals(1, collections.size());
@@ -628,7 +630,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
DEFAULT_COLLECTION + "," + COLLECTION_NAME,
aliases.get("myalias"));
- NamedList<Object> collections = (NamedList<Object>)
cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertNotNull(collections.get(DEFAULT_COLLECTION));
Map<String, Object> collection = (Map<String, Object>)
collections.get(DEFAULT_COLLECTION);
@@ -648,7 +650,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
cluster = (NamedList<Object>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- collections = (NamedList<Object>) cluster.get("collections");
+ collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertNotNull(collections.get(DEFAULT_COLLECTION));
assertNotNull(collections.get(COLLECTION_NAME));
diff --git
a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java
index a0075010862..8d956ca7a94 100644
---
a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java
+++
b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java
@@ -29,7 +29,6 @@ import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -46,7 +45,6 @@ import org.apache.solr.common.cloud.PerReplicaStates;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.CollectionUtil;
import org.apache.solr.common.util.EnvUtils;
-import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.URLUtil;
import org.apache.solr.common.util.Utils;
@@ -160,15 +158,12 @@ public abstract class BaseHttpClusterStateProvider
implements ClusterStateProvid
liveNodesTimestamp = System.nanoTime();
}
- var collectionsNl = (NamedList<Map<String, Object>>)
cluster.get("collections");
+ var collectionsMap = (Map<String, Map<String, Object>>)
cluster.get("collections");
Map<String, DocCollection> collStateByName =
- CollectionUtil.newLinkedHashMap(collectionsNl.size());
- for (Entry<String, Map<String, Object>> entry : collectionsNl) {
- collStateByName.put(
- entry.getKey(), getDocCollectionFromObjects(entry.getKey(),
entry.getValue()));
- }
-
+ CollectionUtil.newLinkedHashMap(collectionsMap.size());
+ collectionsMap.forEach(
+ (key, value) -> collStateByName.put(key,
getDocCollectionFromObjects(key, value)));
return new ClusterState(this.liveNodes, collStateByName);
}
@@ -205,7 +200,7 @@ public abstract class BaseHttpClusterStateProvider
implements ClusterStateProvid
SimpleOrderedMap<?> cluster =
submitClusterStateRequest(client, collection,
ClusterStateRequestType.FETCH_COLLECTION);
- var collStateMap = (Map<String, Object>)
cluster.findRecursive("collections", collection);
+ var collStateMap = (Map<String, Object>)
cluster._get(List.of("collections", collection), null);
if (collStateMap == null) {
throw new NotACollectionException(); // probably an alias
}
diff --git
a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.java
b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.java
index 41aa52e4817..9198956261e 100644
---
a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.java
+++
b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.java
@@ -23,6 +23,7 @@ import static org.hamcrest.Matchers.equalTo;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.lang.reflect.InvocationTargetException;
import java.time.Instant;
import java.util.List;
@@ -37,6 +38,8 @@ import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.util.NamedList;
+import org.eclipse.jetty.http.HttpField;
+import org.eclipse.jetty.http.HttpHeader;
import org.hamcrest.Matchers;
import org.junit.After;
import org.junit.BeforeClass;
@@ -137,7 +140,6 @@ public class ClusterStateProviderTest extends
SolrCloudTestCase {
try (ClusterStateProvider provider = createClusterStateProvider()) {
ClusterState.CollectionRef collectionRef =
provider.getState("testGetState");
-
DocCollection docCollection = collectionRef.get();
assertNotNull(docCollection);
assertEquals(
@@ -161,7 +163,7 @@ public class ClusterStateProviderTest extends
SolrCloudTestCase {
NamedList<Object> response = clusterStatusResponse.getResponse();
NamedList<Object> cluster = (NamedList<Object>) response.get("cluster");
- NamedList<Object> collections = (NamedList<Object>)
cluster.get("collections");
+ Map<String, Object> collections = (Map<String, Object>)
cluster.get("collections");
Map<String, Object> collection = (Map<String, Object>)
collections.get(collectionName);
return Instant.ofEpochMilli((long) collection.get("creationTimeMillis"));
}
@@ -200,6 +202,70 @@ public class ClusterStateProviderTest extends
SolrCloudTestCase {
}
}
+ @Test
+ public void testClusterStateProviderOldVersion() throws SolrServerException,
IOException {
+ CollectionAdminRequest.setClusterProperty("ext.foo",
"bar").process(cluster.getSolrClient());
+ createCollection("col1");
+ createCollection("col2");
+
+ try (var cspZk = zkClientClusterStateProvider();
+ var cspHttp = http2ClusterStateProvider()) {
+ // SolrJ < version 9.9.0 for non streamed response
+ cspHttp
+ .getHttpClient()
+ .getHttpClient()
+ .setUserAgentField(
+ new HttpField(
+ HttpHeader.USER_AGENT,
+ "Solr[" + MethodHandles.lookup().lookupClass().getName() +
"] " + "9.8.0"));
+
+ assertThat(cspHttp.getCollection("col1"),
equalTo(cspZk.getCollection("col1")));
+
+ final var clusterStateZk = cspZk.getClusterState();
+ final var clusterStateHttp = cspHttp.getClusterState();
+ assertThat(
+ clusterStateHttp.getLiveNodes(),
+ containsInAnyOrder(clusterStateHttp.getLiveNodes().toArray()));
+ assertEquals(2, clusterStateZk.size());
+ assertEquals(clusterStateZk.size(), clusterStateHttp.size());
+ assertThat(
+ clusterStateHttp.collectionStream().collect(Collectors.toList()),
+ containsInAnyOrder(clusterStateHttp.collectionStream().toArray()));
+
+ assertThat(
+ clusterStateZk.getCollection("col2"),
equalTo(clusterStateHttp.getCollection("col2")));
+ }
+ }
+
+ @Test
+ public void testClusterStateProviderEmptySolrVersion() throws
SolrServerException, IOException {
+ CollectionAdminRequest.setClusterProperty("ext.foo",
"bar").process(cluster.getSolrClient());
+ createCollection("col1");
+ createCollection("col2");
+
+ try (var cspZk = zkClientClusterStateProvider();
+ var cspHttp = http2ClusterStateProvider()) {
+
+ cspHttp.getHttpClient().getHttpClient().setUserAgentField(null);
+
+ assertThat(cspHttp.getCollection("col1"),
equalTo(cspZk.getCollection("col1")));
+
+ final var clusterStateZk = cspZk.getClusterState();
+ final var clusterStateHttp = cspHttp.getClusterState();
+ assertThat(
+ clusterStateHttp.getLiveNodes(),
+ containsInAnyOrder(clusterStateHttp.getLiveNodes().toArray()));
+ assertEquals(2, clusterStateZk.size());
+ assertEquals(clusterStateZk.size(), clusterStateHttp.size());
+ assertThat(
+ clusterStateHttp.collectionStream().collect(Collectors.toList()),
+ containsInAnyOrder(clusterStateHttp.collectionStream().toArray()));
+
+ assertThat(
+ clusterStateZk.getCollection("col2"),
equalTo(clusterStateHttp.getCollection("col2")));
+ }
+ }
+
@Test
public void testClusterStateProviderDownedInitialLiveNodes() throws
Exception {
try (var cspHttp = http2ClusterStateProvider()) {
