This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new cec23b880c5 SOLR-17006: Collections/AddReplica: Persist user-defined
properties to state.json (#1973)
cec23b880c5 is described below
commit cec23b880c59b7edbf379025081b6301e843b631
Author: Vincent P <[email protected]>
AuthorDate: Fri Oct 20 14:54:15 2023 +0200
SOLR-17006: Collections/AddReplica: Persist user-defined properties to
state.json (#1973)
User-defined properties are persisted to state.json so that any new replica
will use them. Adding a replica can also specify properties and they are saved
to state.json and override that of the collection.
---------
Co-authored-by: Vincent Primault <[email protected]>
---
solr/CHANGES.txt | 3 +
.../solr/cloud/api/collections/AddReplicaCmd.java | 14 +++--
.../solr/cloud/overseer/ClusterStateMutator.java | 7 +++
.../apache/solr/cloud/overseer/SliceMutator.java | 41 +++++++------
.../test/org/apache/solr/cloud/AddReplicaTest.java | 34 +++++++++++
.../cloud/CreateCollectionWithPropertiesTest.java | 68 ++++++++++++++++++++++
.../pages/collection-management.adoc | 1 +
7 files changed, 146 insertions(+), 22 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index ac0a9254310..dc86f0b22fd 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -10,6 +10,9 @@ New Features
---------------------
(No changes)
+* SOLR-17006: Collection creation & adding replicas: User-defined properties
are persisted to state.json and
+ applied to new replicas, available for use as property substitution in
configuration files. (Vincent Primault)
+
Improvements
---------------------
* SOLR-16924: RESTORECORE now sets the UpdateLog to ACTIVE state instead of
requiring a separate
diff --git
a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
index f306fc3726c..42e4409f7f7 100644
---
a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
+++
b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
@@ -255,10 +255,10 @@ public class AddReplicaCmd implements
CollApiCmds.CollectionApiCommand {
CreateReplica createReplica)
throws InterruptedException, KeeperException {
if (!skipCreateReplicaInClusterState) {
- ZkNodeProps props =
- new ZkNodeProps(
+ Map<String, Object> replicaProps =
+ Utils.makeMap(
Overseer.QUEUE_OPERATION,
- ADDREPLICA.toLower(),
+ (Object) ADDREPLICA.toLower(),
ZkStateReader.COLLECTION_PROP,
collectionName,
ZkStateReader.SHARD_ID_PROP,
@@ -274,8 +274,11 @@ public class AddReplicaCmd implements
CollApiCmds.CollectionApiCommand {
ZkStateReader.REPLICA_TYPE,
createReplica.replicaType.name());
if (createReplica.coreNodeName != null) {
- props = props.plus(ZkStateReader.CORE_NODE_NAME_PROP,
createReplica.coreNodeName);
+ replicaProps.put(ZkStateReader.CORE_NODE_NAME_PROP,
createReplica.coreNodeName);
}
+ CollectionHandlingUtils.addPropertyParams(message, replicaProps);
+
+ ZkNodeProps props = new ZkNodeProps(replicaProps);
if (ccc.getDistributedClusterStateUpdater().isDistributedStateUpdate()) {
ccc.getDistributedClusterStateUpdater()
.doSingleStateUpdate(
@@ -342,6 +345,9 @@ public class AddReplicaCmd implements
CollApiCmds.CollectionApiCommand {
if (createReplica.coreNodeName != null) {
params.set(CoreAdminParams.CORE_NODE_NAME, createReplica.coreNodeName);
}
+ // Inherit user-defined properties from collection.
+ CollectionHandlingUtils.addPropertyParams(coll, params);
+ // Inherit user-defined properties from replica.
CollectionHandlingUtils.addPropertyParams(message, params);
return params;
diff --git
a/solr/core/src/java/org/apache/solr/cloud/overseer/ClusterStateMutator.java
b/solr/core/src/java/org/apache/solr/cloud/overseer/ClusterStateMutator.java
index 2c625b2c5c4..11b3795745b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ClusterStateMutator.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ClusterStateMutator.java
@@ -125,6 +125,13 @@ public class ClusterStateMutator {
collectionProps.put(ZkStateReader.CONFIGNAME_PROP, configName);
}
+ // add user-defined properties
+ for (String prop : message.keySet()) {
+ if (prop.startsWith(CollectionAdminParams.PROPERTY_PREFIX)) {
+ collectionProps.put(prop, message.get(prop));
+ }
+ }
+
assert !collectionProps.containsKey(CollectionAdminParams.COLL_CONF);
DocCollection newCollection =
DocCollection.create(
diff --git
a/solr/core/src/java/org/apache/solr/cloud/overseer/SliceMutator.java
b/solr/core/src/java/org/apache/solr/cloud/overseer/SliceMutator.java
index c83a965d5a1..c016fa5489b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/SliceMutator.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/SliceMutator.java
@@ -94,25 +94,30 @@ public class SliceMutator {
cloudManager
.getClusterStateProvider()
.getClusterProperty(ZkStateReader.URL_SCHEME, "http"));
- Replica replica =
- new Replica(
- coreNodeName,
- Utils.makeMap(
- ZkStateReader.CORE_NAME_PROP,
- message.getStr(ZkStateReader.CORE_NAME_PROP),
- ZkStateReader.STATE_PROP,
- message.getStr(ZkStateReader.STATE_PROP),
- ZkStateReader.NODE_NAME_PROP,
- nodeName,
- ZkStateReader.BASE_URL_PROP,
- baseUrl,
- ZkStateReader.FORCE_SET_STATE_PROP,
- "false",
- ZkStateReader.REPLICA_TYPE,
- message.get(ZkStateReader.REPLICA_TYPE)),
- coll,
- slice);
+ Map<String, Object> replicaProps =
+ Utils.makeMap(
+ ZkStateReader.CORE_NAME_PROP,
+ message.getStr(ZkStateReader.CORE_NAME_PROP),
+ ZkStateReader.STATE_PROP,
+ message.getStr(ZkStateReader.STATE_PROP),
+ ZkStateReader.NODE_NAME_PROP,
+ nodeName,
+ ZkStateReader.BASE_URL_PROP,
+ baseUrl,
+ ZkStateReader.FORCE_SET_STATE_PROP,
+ "false",
+ ZkStateReader.REPLICA_TYPE,
+ message.get(ZkStateReader.REPLICA_TYPE));
+
+ // add user-defined properties
+ for (String prop : message.keySet()) {
+ if (prop.startsWith(CollectionAdminParams.PROPERTY_PREFIX)) {
+ replicaProps.put(prop, message.get(prop));
+ }
+ }
+
+ Replica replica = new Replica(coreNodeName, replicaProps, coll, slice);
return new ZkWriteCommand(coll, updateReplica(collection, sl,
replica.getName(), replica));
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/AddReplicaTest.java
b/solr/core/src/test/org/apache/solr/cloud/AddReplicaTest.java
index 7702012713d..fa2bdf84695 100644
--- a/solr/core/src/test/org/apache/solr/cloud/AddReplicaTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/AddReplicaTest.java
@@ -27,6 +27,7 @@ import
org.apache.solr.client.solrj.response.RequestStatusState;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.core.CoreDescriptor;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -187,4 +188,37 @@ public class AddReplicaTest extends SolrCloudTestCase {
assertSame(coll + "\n" + replica, replica.getState(),
Replica.State.ACTIVE);
}
}
+
+ @Test
+ public void testAddReplicaWithUserDefinedProperties() throws Exception {
+ // When creating a collection with user-defined properties
+ CloudSolrClient cloudClient = cluster.getSolrClient();
+ String collectionName = "testAddReplicaWithUserDefinedProperties";
+ CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1)
+ .withProperty("customProp1", "val1")
+ .withProperty("customProp2", "val2")
+ .process(cloudClient);
+ cluster.waitForActiveCollection(collectionName, 1, 1);
+
+ // When adding a replica to the collection with user-defined properties
+ CollectionAdminRequest.AddReplica addReplica =
+ CollectionAdminRequest.addReplicaToShard(collectionName, "shard1");
+ addReplica.withProperty("customProp2", "val2.1");
+ addReplica.withProperty("customProp3", "val3");
+ addReplica.setWaitForFinalState(true);
+ addReplica.process(cloudClient);
+
+ // Verify that the new core was created with user-defined properties
coming from the request
+ // and inherited from the collection (the former taking precedence over
the latter).
+ Replica replica =
+
cloudClient.getClusterState().getCollection(collectionName).getReplicas().get(1);
+ CoreDescriptor coreDescriptor =
+ cluster
+ .getReplicaJetty(replica)
+ .getCoreContainer()
+ .getCoreDescriptor(replica.getCoreName());
+ assertEquals("val1", coreDescriptor.getCoreProperty("customProp1", ""));
+ assertEquals("val2.1", coreDescriptor.getCoreProperty("customProp2", ""));
+ assertEquals("val3", coreDescriptor.getCoreProperty("customProp3", ""));
+ }
}
diff --git
a/solr/core/src/test/org/apache/solr/cloud/CreateCollectionWithPropertiesTest.java
b/solr/core/src/test/org/apache/solr/cloud/CreateCollectionWithPropertiesTest.java
new file mode 100644
index 00000000000..c90e9d21190
--- /dev/null
+++
b/solr/core/src/test/org/apache/solr/cloud/CreateCollectionWithPropertiesTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cloud;
+
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.core.CoreDescriptor;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class CreateCollectionWithPropertiesTest extends SolrCloudTestCase {
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(3)
+ .addConfig(
+ "conf1",
TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+ .configure();
+ }
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ cluster.deleteAllCollections();
+ }
+
+ @Test
+ public void testCreateCollectionWithUserDefinedProperties() throws Exception
{
+ // When creating a collection with user-defined properties
+ CloudSolrClient cloudClient = cluster.getSolrClient();
+ String collectionName = "testCreateCollectionWithUserDefinedProperties";
+ CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1)
+ .withProperty("customProp1", "val1")
+ .process(cloudClient);
+ cluster.waitForActiveCollection(collectionName, 1, 1);
+
+ // Verify that user-defined properties are stored in cluster state
+ DocCollection collection =
cloudClient.getClusterState().getCollection(collectionName);
+ assertEquals("val1",
collection.getProperties().get("property.customProp1"));
+
+ // Verify that the core was created with user-defined properties
+ Replica replica =
+
cloudClient.getClusterState().getCollection(collectionName).getReplicas().get(0);
+ CoreDescriptor coreDescriptor =
+ cluster
+ .getReplicaJetty(replica)
+ .getCoreContainer()
+ .getCoreDescriptor(replica.getCoreName());
+ assertEquals("val1", coreDescriptor.getCoreProperty("customProp1", ""));
+ }
+}
diff --git
a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc
b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc
index 107b4849209..63da9c86e80 100644
---
a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc
+++
b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc
@@ -250,6 +250,7 @@ If `true` the states of individual replicas will be
maintained as individual chi
+
Set core property _name_ to _value_.
See the section xref:configuration-guide:core-discovery.adoc[] for details on
supported properties and values.
+Those properties are also applied to every new core that will be created when
adding replicas to the collection later on.
+
[WARNING]
====
