This is an automated email from the ASF dual-hosted git repository.
broustant pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 7561f5f SOLR-15986: CommitUpdateCommand and SplitIndexCommand can
write user commit metadata.
7561f5f is described below
commit 7561f5ff99f1d187665de84d3d3ec825a2e82bce
Author: Bruno Roustant <[email protected]>
AuthorDate: Tue Feb 8 09:50:49 2022 +0100
SOLR-15986: CommitUpdateCommand and SplitIndexCommand can write user commit
metadata.
---
solr/CHANGES.txt | 2 +-
.../apache/solr/update/CommitUpdateCommand.java | 24 +++++---
.../apache/solr/update/DirectUpdateHandler2.java | 6 +-
.../org/apache/solr/update/SolrIndexSplitter.java | 67 +++++++++++-----------
.../org/apache/solr/update/SolrIndexWriter.java | 12 ++--
.../org/apache/solr/update/SplitIndexCommand.java | 9 +++
6 files changed, 67 insertions(+), 53 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 87b94dc..b519bad 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -35,7 +35,7 @@ New Features
Improvements
---------------------
-(No changes)
+* SOLR-15986: CommitUpdateCommand and SplitIndexCommand can write user commit
metadata. (Bruno Roustant)
Optimizations
---------------------
diff --git a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java
b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java
index 8edc6d2..6ba14ec 100644
--- a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java
+++ b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java
@@ -18,8 +18,10 @@ package org.apache.solr.update;
import org.apache.solr.request.SolrQueryRequest;
+import java.util.Map;
+
/**
- *
+ * A commit index command encapsulated in an object.
*/
public class CommitUpdateCommand extends UpdateCommand {
public boolean optimize;
@@ -28,6 +30,8 @@ public class CommitUpdateCommand extends UpdateCommand {
public boolean expungeDeletes = false;
public boolean softCommit = false;
public boolean prepareCommit = false;
+ /** User provided commit data. Can be let to null if there is none. */
+ public Map<String, String> commitData;
/**
* During optimize, optimize down to <= this many segments. Must be
>= 1
@@ -48,12 +52,16 @@ public class CommitUpdateCommand extends UpdateCommand {
@Override
public String toString() {
- return super.toString() + ",optimize="+optimize
- +",openSearcher="+openSearcher
- +",waitSearcher="+waitSearcher
- +",expungeDeletes="+expungeDeletes
- +",softCommit="+softCommit
- +",prepareCommit="+prepareCommit
- +'}';
+ StringBuilder sb = new StringBuilder(super.toString())
+ .append(",optimize=").append(optimize)
+ .append(",openSearcher=").append(openSearcher)
+ .append(",expungeDeletes=").append(expungeDeletes)
+ .append(",softCommit=").append(softCommit)
+ .append(",prepareCommit=").append(prepareCommit);
+ if (commitData != null) {
+ sb.append(",commitData=").append(commitData);
+ }
+ sb.append('}');
+ return sb.toString();
}
}
diff --git
a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
index 7b1dd17..b68b626 100644
--- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
+++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
@@ -580,7 +580,7 @@ public class DirectUpdateHandler2 extends UpdateHandler
implements SolrCoreState
log.debug("start {}", cmd);
RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
try {
- SolrIndexWriter.setCommitData(iw.get(), cmd.getVersion());
+ SolrIndexWriter.setCommitData(iw.get(), cmd.getVersion(),
cmd.commitData);
iw.get().prepareCommit();
} finally {
iw.decref();
@@ -657,7 +657,7 @@ public class DirectUpdateHandler2 extends UpdateHandler
implements SolrCoreState
// SolrCore.verbose("writer.commit() start writer=",writer);
if (writer.hasUncommittedChanges()) {
- SolrIndexWriter.setCommitData(writer, cmd.getVersion());
+ SolrIndexWriter.setCommitData(writer, cmd.getVersion(),
cmd.commitData);
writer.commit();
} else {
log.debug("No uncommitted changes. Skipping IW.commit.");
@@ -859,7 +859,7 @@ public class DirectUpdateHandler2 extends UpdateHandler
implements SolrCoreState
}
// todo: refactor this shared code (or figure out why a real
CommitUpdateCommand can't be used)
- SolrIndexWriter.setCommitData(writer, cmd.getVersion());
+ SolrIndexWriter.setCommitData(writer, cmd.getVersion(), null);
writer.commit();
synchronized (solrCoreState.getUpdateLock()) {
diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
index ea8bc1c..e1ed501 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
@@ -103,42 +103,39 @@ public class SolrIndexSplitter {
}
}
- SolrIndexSearcher searcher;
- SchemaField field;
- List<DocRouter.Range> ranges;
- DocRouter.Range[] rangesArr; // same as ranges list, but an array for extra
speed in inner loops
- List<String> paths;
- List<SolrCore> cores;
- HashBasedRouter hashRouter;
- int numPieces;
- String routeFieldName;
- String splitKey;
+ final SplitIndexCommand cmd;
+ final SolrIndexSearcher searcher;
+ final SchemaField field;
+ final DocRouter.Range[] rangesArr; // same as ranges list, but an array for
extra speed in inner loops
+ final HashBasedRouter hashRouter;
+ final int numPieces;
+ final String splitKey;
SplitMethod splitMethod;
- RTimerTree timings = new RTimerTree();
+ final RTimerTree timings = new RTimerTree();
public SolrIndexSplitter(SplitIndexCommand cmd) {
+ this.cmd = cmd;
searcher = cmd.getReq().getSearcher();
- ranges = cmd.ranges;
- paths = cmd.paths;
- cores = cmd.cores;
hashRouter = cmd.router instanceof HashBasedRouter ?
(HashBasedRouter)cmd.router : null;
- if (ranges == null) {
- numPieces = paths != null ? paths.size() : cores.size();
+ if (cmd.ranges == null) {
+ numPieces = cmd.paths != null ? cmd.paths.size() : cmd.cores.size();
+ rangesArr = null;
} else {
- numPieces = ranges.size();
- rangesArr = ranges.toArray(new DocRouter.Range[ranges.size()]);
+ numPieces = cmd.ranges.size();
+ rangesArr = cmd.ranges.toArray(new DocRouter.Range[0]);
}
- routeFieldName = cmd.routeFieldName;
- if (routeFieldName == null) {
+ if (cmd.routeFieldName == null) {
field = searcher.getSchema().getUniqueKeyField();
} else {
- field = searcher.getSchema().getField(routeFieldName);
+ field = searcher.getSchema().getField(cmd.routeFieldName);
}
- if (cmd.splitKey != null) {
+ if (cmd.splitKey == null) {
+ splitKey = null;
+ } else {
splitKey = getRouteKey(cmd.splitKey);
}
- if (cores == null) {
+ if (cmd.cores == null) {
this.splitMethod = SplitMethod.REWRITE;
} else {
this.splitMethod = cmd.splitMethod;
@@ -239,20 +236,20 @@ public class SolrIndexSplitter {
// - need to worry about if IW.addIndexes does a sync or not...
// - would be more efficient on the read side, but prob less efficient
merging
for (int partitionNumber=0; partitionNumber<numPieces; partitionNumber++) {
- String partitionName = "SolrIndexSplitter:partition=" + partitionNumber
+ ",partitionCount=" + numPieces + (ranges != null ? ",range=" +
ranges.get(partitionNumber) : "");
+ String partitionName = "SolrIndexSplitter:partition=" + partitionNumber
+ ",partitionCount=" + numPieces + (cmd.ranges != null ? ",range=" +
cmd.ranges.get(partitionNumber) : "");
log.info(partitionName);
boolean success = false;
RefCounted<IndexWriter> iwRef = null;
IndexWriter iw;
- if (cores != null && splitMethod != SplitMethod.LINK) {
- SolrCore subCore = cores.get(partitionNumber);
+ if (cmd.cores != null && splitMethod != SplitMethod.LINK) {
+ SolrCore subCore = cmd.cores.get(partitionNumber);
iwRef =
subCore.getUpdateHandler().getSolrCoreState().getIndexWriter(subCore);
iw = iwRef.get();
} else {
if (splitMethod == SplitMethod.LINK) {
- SolrCore subCore = cores.get(partitionNumber);
+ SolrCore subCore = cmd.cores.get(partitionNumber);
String path = subCore.getDataDir() + INDEX_PREFIX + timestamp;
t = timings.sub("hardLinkCopy");
t.resume();
@@ -288,7 +285,7 @@ public class SolrIndexSplitter {
t.pause();
} else {
SolrCore core = searcher.getCore();
- String path = paths.get(partitionNumber);
+ String path = cmd.paths.get(partitionNumber);
t = timings.sub("createSubIW");
t.resume();
iw = SolrIndexWriter.create(core, partitionName, path,
@@ -313,7 +310,7 @@ public class SolrIndexSplitter {
for (int segmentNumber = 0; segmentNumber<leaves.size();
segmentNumber++) {
if (log.isInfoEnabled()) {
log.info("SolrIndexSplitter: partition # {} partitionCount={} {}
segment #={} segmentCount={}", partitionNumber, numPieces
- , (ranges != null ? " range=" + ranges.get(partitionNumber)
: ""), segmentNumber, leaves.size()); // nowarn
+ , (cmd.ranges != null ? " range=" +
cmd.ranges.get(partitionNumber) : ""), segmentNumber, leaves.size()); // nowarn
}
CodecReader subReader =
SlowCodecReaderWrapper.wrap(leaves.get(segmentNumber).reader());
iw.addIndexes(new LiveDocsReader(subReader,
segmentDocSets.get(segmentNumber)[partitionNumber]));
@@ -324,7 +321,7 @@ public class SolrIndexSplitter {
// because the sub-shard cores will just ignore such a commit because
the update log is not
// in active state at this time.
//TODO no commitUpdateCommand
- SolrIndexWriter.setCommitData(iw, -1);
+ SolrIndexWriter.setCommitData(iw, -1, cmd.commitData);
t = timings.sub("subIWCommit");
t.resume();
iw.commit();
@@ -343,7 +340,7 @@ public class SolrIndexSplitter {
IOUtils.closeWhileHandlingException(iw);
}
if (splitMethod == SplitMethod.LINK) {
- SolrCore subCore = cores.get(partitionNumber);
+ SolrCore subCore = cmd.cores.get(partitionNumber);
subCore.getDirectoryFactory().release(iw.getDirectory());
}
}
@@ -351,11 +348,11 @@ public class SolrIndexSplitter {
}
// all sub-indexes created ok
// when using hard-linking switch directories & refresh cores
- if (splitMethod == SplitMethod.LINK && cores != null) {
+ if (splitMethod == SplitMethod.LINK && cmd.cores != null) {
boolean switchOk = true;
t = timings.sub("switchSubIndexes");
for (int partitionNumber = 0; partitionNumber < numPieces;
partitionNumber++) {
- SolrCore subCore = cores.get(partitionNumber);
+ SolrCore subCore = cmd.cores.get(partitionNumber);
String indexDirPath = subCore.getIndexDir();
log.debug("Switching directories");
@@ -375,7 +372,7 @@ public class SolrIndexSplitter {
t = timings.sub("rollbackSubIndexes");
// rollback the switch
for (int partitionNumber = 0; partitionNumber < numPieces;
partitionNumber++) {
- SolrCore subCore = cores.get(partitionNumber);
+ SolrCore subCore = cmd.cores.get(partitionNumber);
Directory dir = null;
try {
dir = subCore.getDirectoryFactory().get(subCore.getDataDir(),
DirectoryFactory.DirContext.META_DATA,
@@ -411,7 +408,7 @@ public class SolrIndexSplitter {
// complete the switch - remove original index
t = timings.sub("cleanSubIndex");
for (int partitionNumber = 0; partitionNumber < numPieces;
partitionNumber++) {
- SolrCore subCore = cores.get(partitionNumber);
+ SolrCore subCore = cmd.cores.get(partitionNumber);
String oldIndexPath = subCore.getDataDir() + "index";
Directory indexDir = null;
try {
diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
index aa841f3..b7bd35e 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
@@ -178,12 +178,12 @@ public class SolrIndexWriter extends IndexWriter {
@SuppressForbidden(reason = "Need currentTimeMillis, commit time should be
used only for debugging purposes, " +
" but currently suspiciously used for replication as well")
- public static void setCommitData(IndexWriter iw, long commitCommandVersion) {
- log.debug("Calling setCommitData with IW:{} commitCommandVersion:{}", iw,
commitCommandVersion);
- final Map<String,String> commitData = new HashMap<>();
- commitData.put(COMMIT_TIME_MSEC_KEY,
String.valueOf(System.currentTimeMillis()));
- commitData.put(COMMIT_COMMAND_VERSION,
String.valueOf(commitCommandVersion));
- iw.setLiveCommitData(commitData.entrySet());
+ public static void setCommitData(IndexWriter iw, long commitCommandVersion,
Map<String, String> commitData) {
+ log.debug("Calling setCommitData with IW:{} commitCommandVersion:{}
commitData:{}", iw, commitCommandVersion, commitData);
+ Map<String,String> finalCommitData = commitData == null ? new HashMap<>(4)
: new HashMap<>(commitData);
+ finalCommitData.put(COMMIT_TIME_MSEC_KEY,
String.valueOf(System.currentTimeMillis()));
+ finalCommitData.put(COMMIT_COMMAND_VERSION,
String.valueOf(commitCommandVersion));
+ iw.setLiveCommitData(finalCommitData.entrySet());
}
private void setDirectoryFactory(DirectoryFactory factory) {
diff --git a/solr/core/src/java/org/apache/solr/update/SplitIndexCommand.java
b/solr/core/src/java/org/apache/solr/update/SplitIndexCommand.java
index 56208a8..a18b12b 100644
--- a/solr/core/src/java/org/apache/solr/update/SplitIndexCommand.java
+++ b/solr/core/src/java/org/apache/solr/update/SplitIndexCommand.java
@@ -22,6 +22,7 @@ import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import java.util.List;
+import java.util.Map;
/**
* A split index command encapsulated in an object.
@@ -38,6 +39,11 @@ public class SplitIndexCommand extends UpdateCommand {
public final String routeFieldName;
public final String splitKey;
public final SolrIndexSplitter.SplitMethod splitMethod;
+ /**
+ * User provided commit data. Can be let to null if there is none.
+ * This commit data is used when the split command commits.
+ */
+ public Map<String, String> commitData;
public SplitIndexCommand(SolrQueryRequest req, SolrQueryResponse rsp,
List<String> paths, List<SolrCore> cores, List<DocRouter.Range> ranges,
DocRouter router, String routeFieldName, String
splitKey, SolrIndexSplitter.SplitMethod splitMethod) {
@@ -71,6 +77,9 @@ public class SplitIndexCommand extends UpdateCommand {
sb.append(",split.key=").append(splitKey);
}
sb.append(",method=").append(splitMethod.toLower());
+ if (commitData != null) {
+ sb.append(",commitData=").append(commitData);
+ }
sb.append('}');
return sb.toString();
}
