This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 5e328ee77e5 SOLR-17582: Stream CLUSTERSTATUS API for SolrJ version >=
9.9 (#3156)
5e328ee77e5 is described below
commit 5e328ee77e5eeee858f4112e02652ddd4c8e14b6
Author: Matthew Biscocho <[email protected]>
AuthorDate: Tue Feb 4 21:18:07 2025 -0500
SOLR-17582: Stream CLUSTERSTATUS API for SolrJ version >= 9.9 (#3156)
Or if unknown (e.g. some misc. HTTP/JSON).
Co-authored-by: Matthew Biscocho <[email protected]>
---
.../src/java/org/apache/solr/cli/StatusTool.java | 2 +-
.../apache/solr/handler/admin/ClusterStatus.java | 35 +++++------
.../solr/handler/admin/CollectionsHandler.java | 2 +-
.../cloud/api/collections/TestCollectionAPI.java | 32 +++++-----
.../solrj/impl/BaseHttpClusterStateProvider.java | 15 ++---
.../solrj/impl/ClusterStateProviderTest.java | 70 +++++++++++++++++++++-
6 files changed, 110 insertions(+), 46 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cli/StatusTool.java
b/solr/core/src/java/org/apache/solr/cli/StatusTool.java
index fa6c2cbf2bd..a935a7b364f 100644
--- a/solr/core/src/java/org/apache/solr/cli/StatusTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/StatusTool.java
@@ -353,7 +353,7 @@ public class StatusTool extends ToolBase {
cloudStatus.put("liveNodes", String.valueOf(liveNodes.size()));
// TODO get this as a metric from the metrics API instead, or something
else.
- var collections = (NamedList<Object>) json.findRecursive("cluster",
"collections");
+ var collections = (Map<String, Object>) json.findRecursive("cluster",
"collections");
cloudStatus.put("collections", String.valueOf(collections.size()));
return cloudStatus;
diff --git
a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java
b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java
index 7a8ecf9c850..6ef549698d3 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Stream;
+import org.apache.solr.client.api.util.SolrVersion;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Aliases;
@@ -34,7 +35,6 @@ import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.PerReplicaStates;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
@@ -101,7 +101,7 @@ public class ClusterStatus {
collection = params.get(ZkStateReader.COLLECTION_PROP);
}
- public void getClusterStatus(NamedList<Object> results)
+ public void getClusterStatus(NamedList<Object> results, SolrVersion
solrVersion)
throws KeeperException, InterruptedException {
NamedList<Object> clusterStatus = new SimpleOrderedMap<>();
@@ -127,7 +127,7 @@ public class ClusterStatus {
if (withCollection) {
assert liveNodes != null;
- fetchClusterStatusForCollOrAlias(clusterStatus, liveNodes, aliases);
+ fetchClusterStatusForCollOrAlias(clusterStatus, liveNodes, aliases,
solrVersion);
}
if (withAliases) {
@@ -158,7 +158,10 @@ public class ClusterStatus {
}
private void fetchClusterStatusForCollOrAlias(
- NamedList<Object> clusterStatus, List<String> liveNodes, Aliases
aliases) {
+ NamedList<Object> clusterStatus,
+ List<String> liveNodes,
+ Aliases aliases,
+ SolrVersion solrVersion) {
// read aliases
Map<String, List<String>> collectionVsAliases = new HashMap<>();
@@ -206,19 +209,7 @@ public class ClusterStatus {
}
}
- // Because of back-compat for SolrJ, create the whole response into a
NamedList
- // Otherwise stream with MapWriter to save memory
- if (CommonParams.JAVABIN.equals(solrParams.get(CommonParams.WT))) {
- NamedList<Object> collectionProps = new SimpleOrderedMap<>();
- collectionStream.forEach(
- collectionState -> {
- collectionProps.add(
- collectionState.getName(),
- buildResponseForCollection(
- collectionState, collectionVsAliases, routeKey, liveNodes,
requestedShards));
- });
- clusterStatus.add("collections", collectionProps);
- } else {
+ if (solrVersion == null ||
solrVersion.greaterThanOrEqualTo(SolrVersion.valueOf("9.9.0"))) {
MapWriter collectionPropsWriter =
ew -> {
collectionStream.forEach(
@@ -234,6 +225,16 @@ public class ClusterStatus {
});
};
clusterStatus.add("collections", collectionPropsWriter);
+ } else {
+ NamedList<Object> collectionProps = new SimpleOrderedMap<>();
+ collectionStream.forEach(
+ collectionState -> {
+ collectionProps.add(
+ collectionState.getName(),
+ buildResponseForCollection(
+ collectionState, collectionVsAliases, routeKey, liveNodes,
requestedShards));
+ });
+ clusterStatus.add("collections", collectionProps);
}
}
diff --git
a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index aefc1033d5e..3b22b3072e5 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -975,7 +975,7 @@ public class CollectionsHandler extends RequestHandlerBase
implements Permission
CLUSTERSTATUS,
(req, rsp, h) -> {
new
ClusterStatus(h.coreContainer.getZkController().getZkStateReader(),
req.getParams())
- .getClusterStatus(rsp.getValues());
+ .getClusterStatus(rsp.getValues(),
req.getHttpSolrCall().getUserAgentSolrVersion());
return null;
}),
ADDREPLICAPROP_OP(
diff --git
a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java
b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java
index 0aa5d4ae2d0..ce605c70271 100644
---
a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java
+++
b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java
@@ -131,10 +131,11 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
.getResponse();
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- NamedList<?> collections = (NamedList<?>) cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertEquals(1, collections.size());
- assertEquals("25", collections._getStr(List.of(COLLECTION_NAME,
"replicationFactor"), null));
+ Map<?, ?> collectionProperties = (Map<?, ?>)
collections.get(COLLECTION_NAME);
+ assertEquals("25", collectionProperties.get("replicationFactor"));
params = new ModifiableSolrParams();
params.set("action",
CollectionParams.CollectionAction.MODIFYCOLLECTION.toString());
@@ -153,10 +154,11 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
System.out.println(rsp);
cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- collections = (NamedList<?>) cluster.get("collections");
+ collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertEquals(1, collections.size());
- assertNull(collections._getStr(List.of(COLLECTION_NAME,
"replicationFactor"), null));
+ collectionProperties = (Map<?, ?>) collections.get(COLLECTION_NAME);
+ assertNull(collectionProperties.get("replicationFactor"));
params = new ModifiableSolrParams();
params.set("action",
CollectionParams.CollectionAction.MODIFYCOLLECTION.toString());
@@ -255,7 +257,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
NamedList<?> rsp = client.request(req);
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- NamedList<?> collections = (NamedList<?>) cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertNotNull(
"Testing to insure collections are returned",
collections.get(COLLECTION_NAME1));
@@ -282,7 +284,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
NamedList<Object> rsp = client.request(request);
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- NamedList<?> collections = (NamedList<?>) cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertEquals(1, collections.size());
@SuppressWarnings({"unchecked"})
@@ -304,7 +306,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
NamedList<Object> rsp = client.request(request);
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- NamedList<?> collections = (NamedList<?>) cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertNotNull(collections.get(COLLECTION_NAME));
assertEquals(1, collections.size());
@@ -330,7 +332,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
NamedList<Object> rsp = request.process(client).getResponse();
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- NamedList<?> collections = (NamedList<?>) cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertNotNull(collections.get(COLLECTION_NAME));
assertEquals(1, collections.size());
@@ -465,7 +467,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
NamedList<Object> rsp = client.request(request);
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- NamedList<?> collections = (NamedList<?>) cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertNotNull(collections.get(COLLECTION_NAME1));
assertEquals(4, collections.size());
@@ -487,7 +489,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
NamedList<Object> rsp = client.request(request);
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- NamedList<?> collections = (NamedList<?>) cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertEquals(1, collections.size());
@SuppressWarnings({"unchecked"})
@@ -517,7 +519,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
NamedList<Object> rsp = client.request(request);
NamedList<Object> cluster = (NamedList<Object>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- NamedList<Object> collections = (NamedList<Object>)
cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertEquals(1, collections.size());
Map<String, Object> collection = (Map<String, Object>)
collections.get(cname);
@@ -533,7 +535,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
rsp = client.request(request);
cluster = (NamedList<Object>) rsp.get("cluster");
- collections = (NamedList<Object>) cluster.get("collections");
+ collections = (Map<?, ?>) cluster.get("collections");
collection = (Map<String, Object>) collections.get(cname);
Integer newVersion = (Integer) collection.get("znodeVersion");
assertNotNull(newVersion);
@@ -560,7 +562,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
NamedList<Object> cluster = (NamedList<Object>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
@SuppressWarnings({"unchecked"})
- NamedList<Object> collections = (NamedList<Object>)
cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertNotNull(collections.get(DEFAULT_COLLECTION));
assertEquals(1, collections.size());
@@ -607,7 +609,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
DEFAULT_COLLECTION + "," + COLLECTION_NAME,
aliases.get("myalias"));
- NamedList<Object> collections = (NamedList<Object>)
cluster.get("collections");
+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertNotNull(collections.get(DEFAULT_COLLECTION));
Map<String, Object> collection = (Map<String, Object>)
collections.get(DEFAULT_COLLECTION);
@@ -627,7 +629,7 @@ public class TestCollectionAPI extends
ReplicaPropertiesBase {
cluster = (NamedList<Object>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
- collections = (NamedList<Object>) cluster.get("collections");
+ collections = (Map<?, ?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state",
collections);
assertNotNull(collections.get(DEFAULT_COLLECTION));
assertNotNull(collections.get(COLLECTION_NAME));
diff --git
a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java
index a09d936582e..150b4fe0de2 100644
---
a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java
+++
b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java
@@ -29,7 +29,6 @@ import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -46,7 +45,6 @@ import org.apache.solr.common.cloud.PerReplicaStates;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.CollectionUtil;
import org.apache.solr.common.util.EnvUtils;
-import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.URLUtil;
import org.apache.solr.common.util.Utils;
@@ -160,15 +158,12 @@ public abstract class BaseHttpClusterStateProvider
implements ClusterStateProvid
liveNodesTimestamp = System.nanoTime();
}
- var collectionsNl = (NamedList<Map<String, Object>>)
cluster.get("collections");
+ var collectionsMap = (Map<String, Map<String, Object>>)
cluster.get("collections");
Map<String, DocCollection> collStateByName =
- CollectionUtil.newLinkedHashMap(collectionsNl.size());
- for (Entry<String, Map<String, Object>> entry : collectionsNl) {
- collStateByName.put(
- entry.getKey(), getDocCollectionFromObjects(entry.getKey(),
entry.getValue()));
- }
-
+ CollectionUtil.newLinkedHashMap(collectionsMap.size());
+ collectionsMap.forEach(
+ (key, value) -> collStateByName.put(key,
getDocCollectionFromObjects(key, value)));
return new ClusterState(this.liveNodes, collStateByName);
}
@@ -205,7 +200,7 @@ public abstract class BaseHttpClusterStateProvider
implements ClusterStateProvid
SimpleOrderedMap<?> cluster =
submitClusterStateRequest(client, collection,
ClusterStateRequestType.FETCH_COLLECTION);
- var collStateMap = (Map<String, Object>)
cluster.findRecursive("collections", collection);
+ var collStateMap = (Map<String, Object>)
cluster._get(List.of("collections", collection), null);
if (collStateMap == null) {
throw new NotACollectionException(); // probably an alias
}
diff --git
a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.java
b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.java
index f3ee722ee58..5c6ce5fbdb9 100644
---
a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.java
+++
b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.java
@@ -23,6 +23,7 @@ import static org.hamcrest.Matchers.equalTo;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.lang.reflect.InvocationTargetException;
import java.time.Instant;
import java.util.List;
@@ -36,6 +37,8 @@ import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.util.NamedList;
+import org.eclipse.jetty.http.HttpField;
+import org.eclipse.jetty.http.HttpHeader;
import org.hamcrest.Matchers;
import org.junit.After;
import org.junit.BeforeClass;
@@ -135,7 +138,6 @@ public class ClusterStateProviderTest extends
SolrCloudTestCase {
try (ClusterStateProvider provider = createClusterStateProvider()) {
ClusterState.CollectionRef collectionRef =
provider.getState("testGetState");
-
DocCollection docCollection = collectionRef.get();
assertNotNull(docCollection);
assertEquals(
@@ -159,7 +161,7 @@ public class ClusterStateProviderTest extends
SolrCloudTestCase {
NamedList<Object> response = clusterStatusResponse.getResponse();
NamedList<Object> cluster = (NamedList<Object>) response.get("cluster");
- NamedList<Object> collections = (NamedList<Object>)
cluster.get("collections");
+ Map<String, Object> collections = (Map<String, Object>)
cluster.get("collections");
Map<String, Object> collection = (Map<String, Object>)
collections.get(collectionName);
return Instant.ofEpochMilli((long) collection.get("creationTimeMillis"));
}
@@ -198,6 +200,70 @@ public class ClusterStateProviderTest extends
SolrCloudTestCase {
}
}
+ @Test
+ public void testClusterStateProviderOldVersion() throws SolrServerException,
IOException {
+ CollectionAdminRequest.setClusterProperty("ext.foo",
"bar").process(cluster.getSolrClient());
+ createCollection("col1");
+ createCollection("col2");
+
+ try (var cspZk = zkClientClusterStateProvider();
+ var cspHttp = http2ClusterStateProvider()) {
+ // SolrJ < version 9.9.0 for non streamed response
+ cspHttp
+ .getHttpClient()
+ .getHttpClient()
+ .setUserAgentField(
+ new HttpField(
+ HttpHeader.USER_AGENT,
+ "Solr[" + MethodHandles.lookup().lookupClass().getName() +
"] " + "9.8.0"));
+
+ assertThat(cspHttp.getCollection("col1"),
equalTo(cspZk.getCollection("col1")));
+
+ final var clusterStateZk = cspZk.getClusterState();
+ final var clusterStateHttp = cspHttp.getClusterState();
+ assertThat(
+ clusterStateHttp.getLiveNodes(),
+ containsInAnyOrder(clusterStateHttp.getLiveNodes().toArray()));
+ assertEquals(2, clusterStateZk.size());
+ assertEquals(clusterStateZk.size(), clusterStateHttp.size());
+ assertThat(
+ clusterStateHttp.collectionStream().toList(),
+ containsInAnyOrder(clusterStateHttp.collectionStream().toArray()));
+
+ assertThat(
+ clusterStateZk.getCollection("col2"),
equalTo(clusterStateHttp.getCollection("col2")));
+ }
+ }
+
+ @Test
+ public void testClusterStateProviderEmptySolrVersion() throws
SolrServerException, IOException {
+ CollectionAdminRequest.setClusterProperty("ext.foo",
"bar").process(cluster.getSolrClient());
+ createCollection("col1");
+ createCollection("col2");
+
+ try (var cspZk = zkClientClusterStateProvider();
+ var cspHttp = http2ClusterStateProvider()) {
+
+ cspHttp.getHttpClient().getHttpClient().setUserAgentField(null);
+
+ assertThat(cspHttp.getCollection("col1"),
equalTo(cspZk.getCollection("col1")));
+
+ final var clusterStateZk = cspZk.getClusterState();
+ final var clusterStateHttp = cspHttp.getClusterState();
+ assertThat(
+ clusterStateHttp.getLiveNodes(),
+ containsInAnyOrder(clusterStateHttp.getLiveNodes().toArray()));
+ assertEquals(2, clusterStateZk.size());
+ assertEquals(clusterStateZk.size(), clusterStateHttp.size());
+ assertThat(
+ clusterStateHttp.collectionStream().toList(),
+ containsInAnyOrder(clusterStateHttp.collectionStream().toArray()));
+
+ assertThat(
+ clusterStateZk.getCollection("col2"),
equalTo(clusterStateHttp.getCollection("col2")));
+ }
+ }
+
@Test
public void testClusterStateProviderDownedInitialLiveNodes() throws
Exception {
try (var cspHttp = http2ClusterStateProvider()) {
