This is an automated email from the ASF dual-hosted git repository.
gerlowskija pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 2b1fbc13767 SOLR-17187: make replica poll interval configurable (#3144)
2b1fbc13767 is described below
commit 2b1fbc137670655aef2b7eef82bb79642eafea31
Author: Christine Poerschke <[email protected]>
AuthorDate: Fri May 16 15:57:20 2025 +0100
SOLR-17187: make replica poll interval configurable (#3144)
A supplied `commitPollInterval` in `updateHandler` now takes
precedence over the default (which is calculated based on
hard/soft commit settings).
---------
Co-authored-by: Torsten Bøgh Köster <[email protected]>
Co-authored-by: Torsten Bøgh Köster <[email protected]>
Co-authored-by: Christine Poerschke <[email protected]>
Co-authored-by: Jason Gerlowski <[email protected]>
---
solr/CHANGES.txt | 3 ++
.../org/apache/solr/cloud/ReplicateFromLeader.java | 29 ++++++------
.../src/java/org/apache/solr/core/SolrConfig.java | 7 ++-
.../apache/solr/handler/ReplicationHandler.java | 2 +-
.../apache/solr/cloud/ReplicateFromLeaderTest.java | 53 ++++++++++++++++++----
.../pages/commits-transaction-logs.adoc | 4 ++
6 files changed, 73 insertions(+), 25 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 51e38d4dfd7..7b969f90990 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -238,6 +238,9 @@ Improvements
* SOLR-17744: Solr now enables Jetty's Graceful Shutdown features to prevent
client connections from being abruptly terminated on orderly shutdown (hossman)
+* SOLR-17187: The polling interval for PULL and TLOG replicas can now be
overridden using the `commitPollInterval` setting, which takes a String
+ formatted as "HH:mm:ss" (Torsten Koster via Christine Poerschke, Jason
Gerlowski)
+
Optimizations
---------------------
* SOLR-17578: Remove ZkController internal core supplier, for slightly faster
reconnection after Zookeeper session loss. (Pierre Salagnac)
diff --git a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java
b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java
index 06bbbefedbe..7cc932365e1 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java
@@ -75,16 +75,9 @@ public class ReplicateFromLeader {
"SolrCore not found:" + coreName + " in " +
CloudUtil.getLoadedCoreNamesAsString(cc));
}
}
- SolrConfig.UpdateHandlerInfo uinfo =
core.getSolrConfig().getUpdateHandlerInfo();
- String pollIntervalStr = "00:00:03";
- if (System.getProperty("jetty.testMode") != null) {
- pollIntervalStr = "00:00:01";
- }
- String calculatedPollIntervalString = determinePollInterval(uinfo);
- if (calculatedPollIntervalString != null) {
- pollIntervalStr = calculatedPollIntervalString;
- }
+ final SolrConfig.UpdateHandlerInfo uinfo =
core.getSolrConfig().getUpdateHandlerInfo();
+ final String pollIntervalStr = determinePollInterval(uinfo);
log.info("Will start replication from leader with poll interval: {}",
pollIntervalStr);
NamedList<Object> followerConfig = new NamedList<>();
@@ -134,18 +127,25 @@ public class ReplicateFromLeader {
}
/**
- * Determine the poll interval for replicas based on the auto soft/hard
commit schedule
+ * Determine the poll interval for replicas based on the auto soft/hard
commit schedule or
+ * configured commit poll interval
*
* @param uinfo the update handler info containing soft/hard commit
configuration
* @return a poll interval string representing a cadence of polling
frequency in the form of
- * hh:mm:ss
+ * hh:mm:ss, never <code>null</code>
*/
public static String determinePollInterval(SolrConfig.UpdateHandlerInfo
uinfo) {
int hardCommitMaxTime = uinfo.autoCommmitMaxTime;
int softCommitMaxTime = uinfo.autoSoftCommmitMaxTime;
boolean hardCommitNewSearcher = uinfo.openSearcher;
- String pollIntervalStr = null;
- if (hardCommitMaxTime != -1) {
+ String customCommitPollInterval = uinfo.commitPollInterval;
+ String pollIntervalStr = "00:00:03";
+
+ if (System.getProperty("jetty.testMode") != null) {
+ pollIntervalStr = "00:00:01";
+ } else if (customCommitPollInterval != null) {
+ pollIntervalStr = customCommitPollInterval;
+ } else if (hardCommitMaxTime != -1) {
// configured hardCommit places a ceiling on the interval at which new
segments will be
// available to replicate
if (softCommitMaxTime != -1
@@ -168,6 +168,9 @@ public class ReplicateFromLeader {
pollIntervalStr = toPollIntervalStr(softCommitMaxTime / 2);
}
+ // validate poll interval and fail early
+ ReplicationHandler.readIntervalNs(pollIntervalStr);
+ assert pollIntervalStr != null;
return pollIntervalStr;
}
diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java
b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
index 4034fb42e2a..4d7962a5ea5 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
@@ -800,6 +800,7 @@ public class SolrConfig implements MapSerializable {
public final long autoCommitMaxSizeBytes;
public final boolean openSearcher; // is opening a new searcher part of
hard autocommit?
public final boolean commitWithinSoftCommit;
+ public final String commitPollInterval;
public final boolean aggregateNodeLevelMetricsEnabled;
/**
@@ -815,7 +816,8 @@ public class SolrConfig implements MapSerializable {
boolean openSearcher,
int autoSoftCommmitMaxDocs,
int autoSoftCommmitMaxTime,
- boolean commitWithinSoftCommit) {
+ boolean commitWithinSoftCommit,
+ String commitPollInterval) {
this.className = className;
this.autoCommmitMaxDocs = autoCommmitMaxDocs;
this.autoCommmitMaxTime = autoCommmitMaxTime;
@@ -826,6 +828,7 @@ public class SolrConfig implements MapSerializable {
this.autoSoftCommmitMaxTime = autoSoftCommmitMaxTime;
this.commitWithinSoftCommit = commitWithinSoftCommit;
+ this.commitPollInterval = commitPollInterval;
this.aggregateNodeLevelMetricsEnabled = false;
}
@@ -841,6 +844,7 @@ public class SolrConfig implements MapSerializable {
this.autoSoftCommmitMaxTime =
updateHandler.get("autoSoftCommit").get("maxTime").intVal(-1);
this.commitWithinSoftCommit =
updateHandler.get("commitWithin").get("softCommit").boolVal(true);
+ this.commitPollInterval = updateHandler.get("commitPollInterval").txt();
this.aggregateNodeLevelMetricsEnabled =
updateHandler.boolAttr("aggregateNodeLevelMetricsEnabled", false);
}
@@ -857,6 +861,7 @@ public class SolrConfig implements MapSerializable {
map.put(
"autoSoftCommit",
Map.of("maxDocs", autoSoftCommmitMaxDocs, "maxTime",
autoSoftCommmitMaxTime));
+ map.put("commitPollInterval", commitPollInterval);
return map;
}
}
diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
index c506a875294..8367e9166f9 100644
--- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
@@ -1490,7 +1490,7 @@ public class ReplicationHandler extends RequestHandlerBase
return TimeUnit.MILLISECONDS.convert(readIntervalNs(interval),
TimeUnit.NANOSECONDS);
}
- private Long readIntervalNs(String interval) {
+ public static Long readIntervalNs(String interval) {
if (interval == null) return null;
int result = 0;
Matcher m = INTERVAL_PATTERN.matcher(interval.trim());
diff --git
a/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java
b/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java
index 723eda5874d..aa4ae37e1c5 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java
@@ -17,55 +17,88 @@
package org.apache.solr.cloud;
-import static org.junit.Assert.assertEquals;
-
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrException;
import org.apache.solr.core.SolrConfig;
import org.junit.Test;
-public class ReplicateFromLeaderTest {
+public class ReplicateFromLeaderTest extends SolrTestCaseJ4 {
+
+ @Test
+ public void determineTestPollIntervalString() {
+ SolrConfig.UpdateHandlerInfo updateHandlerInfo =
+ new SolrConfig.UpdateHandlerInfo(
+ "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false,
"0:0:56");
+ String pollInterval =
ReplicateFromLeader.determinePollInterval(updateHandlerInfo);
+ assertEquals("00:00:01", pollInterval);
+ }
@Test
public void determinePollIntervalString() {
+ // disable jetty test mode
+ System.clearProperty("jetty.testMode");
+
SolrConfig.UpdateHandlerInfo updateHandlerInfo =
new SolrConfig.UpdateHandlerInfo(
- "solr.DirectUpdateHandler2", -1, 15000, -1, true, -1, 60000,
false);
+ "solr.DirectUpdateHandler2", -1, 15000, -1, true, -1, 60000,
false, null);
String pollInterval =
ReplicateFromLeader.determinePollInterval(updateHandlerInfo);
assertEquals("0:0:7", pollInterval);
updateHandlerInfo =
new SolrConfig.UpdateHandlerInfo(
- "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, 15000,
false);
+ "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, 15000,
false, null);
pollInterval =
ReplicateFromLeader.determinePollInterval(updateHandlerInfo);
assertEquals("0:0:30", pollInterval);
updateHandlerInfo =
new SolrConfig.UpdateHandlerInfo(
- "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, 60000,
false);
+ "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, 60000,
false, null);
pollInterval =
ReplicateFromLeader.determinePollInterval(updateHandlerInfo);
assertEquals("0:0:30", pollInterval);
updateHandlerInfo =
new SolrConfig.UpdateHandlerInfo(
- "solr.DirectUpdateHandler2", -1, 60000, -1, false, -1, 15000,
false);
+ "solr.DirectUpdateHandler2", -1, 60000, -1, false, -1, 15000,
false, null);
pollInterval =
ReplicateFromLeader.determinePollInterval(updateHandlerInfo);
assertEquals("0:0:30", pollInterval);
updateHandlerInfo =
new SolrConfig.UpdateHandlerInfo(
- "solr.DirectUpdateHandler2", -1, -1, -1, false, -1, 60000, false);
+ "solr.DirectUpdateHandler2", -1, -1, -1, false, -1, 60000, false,
null);
pollInterval =
ReplicateFromLeader.determinePollInterval(updateHandlerInfo);
assertEquals("0:0:30", pollInterval);
updateHandlerInfo =
new SolrConfig.UpdateHandlerInfo(
- "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, -1, false);
+ "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, -1, false,
null);
pollInterval =
ReplicateFromLeader.determinePollInterval(updateHandlerInfo);
assertEquals("0:0:7", pollInterval);
updateHandlerInfo =
new SolrConfig.UpdateHandlerInfo(
- "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false);
+ "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false,
null);
pollInterval =
ReplicateFromLeader.determinePollInterval(updateHandlerInfo);
assertEquals("0:0:30", pollInterval);
+
+ updateHandlerInfo =
+ new SolrConfig.UpdateHandlerInfo(
+ "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false,
"0:0:56");
+ pollInterval =
ReplicateFromLeader.determinePollInterval(updateHandlerInfo);
+ assertEquals("0:0:56", pollInterval);
+
+ final SolrConfig.UpdateHandlerInfo illegalUpdateHandlerInfo =
+ new SolrConfig.UpdateHandlerInfo(
+ "solr.DirectUpdateHandler2",
+ -1,
+ 60000,
+ -1,
+ true,
+ -1,
+ -1,
+ false,
+ "garbage-unfortunately");
+ assertThrows(
+ SolrException.class,
+ () ->
ReplicateFromLeader.determinePollInterval(illegalUpdateHandlerInfo));
}
}
diff --git
a/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc
b/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc
index 812bbcbc3a8..c13790c9da3 100644
---
a/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc
+++
b/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc
@@ -62,6 +62,10 @@ It is recommended that this be set for as long as is
reasonable given the applic
A hard commit means that, if a server crashes, Solr will know exactly where
your data was stored; a soft commit means that the data is stored, but the
location information isn't yet stored.
The tradeoff is that a soft commit gives you faster visibility because it's
not waiting for background merges to finish.
+In a TLOG/PULL replica setup, the commit configuration also influences the
interval at which the replica is polling the shard leader.
+Users wishing to use a different polling interval in their TLOG/PULL replicas
can do so by specifying a `commitPollInterval` value of the form "hh:mm:ss".
+"01:00:00" to poll every hour, "00:15:00" to poll every fifteen minutes, etc.
+
=== Explicit Commits
When a client includes a `commit=true` parameter with an update request, this
ensures that all index segments affected by the adds and deletes on an update
are written to disk as soon as index updates are completed.
