This is an automated email from the ASF dual-hosted git repository.
houston pushed a commit to branch branch_9_0
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9_0 by this push:
new dc3b998 SOLR-15842: Fix async backup response (#479)
dc3b998 is described below
commit dc3b9987ec4a2ebee0ba8f8810eb49bbee39235b
Author: ijioio <[email protected]>
AuthorDate: Mon Jan 24 18:24:07 2022 +0300
SOLR-15842: Fix async backup response (#479)
- Add response information for non-incremental backups
- Correctly add in indexSize and indexFiles information into the response
- Add an endTime to the BackupProperties
Co-authored-by: Houston Putman <[email protected]>
---
solr/CHANGES.txt | 2 +
.../solr/cloud/api/collections/BackupCmd.java | 68 ++++++++++++++++++----
.../org/apache/solr/core/backup/BackupManager.java | 1 +
.../apache/solr/core/backup/BackupProperties.java | 5 ++
.../apache/solr/handler/admin/BackupCoreOp.java | 2 +-
.../solr/handler/admin/CoreAdminHandler.java | 12 +++-
.../solr/handler/admin/CoreAdminOperation.java | 7 ++-
.../src/major-changes-in-solr-9.adoc | 10 +++-
8 files changed, 87 insertions(+), 20 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index cec7920..a70368b 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -510,6 +510,8 @@ Bug Fixes
* SOLR-15919: Replace File with Path for many ZK operations (Mike Drob)
+* SOLR-15842: Async response for backups now correctly aggregates and returns
information (Houston Putman, Artem Abeleshev, Christine Poerschke)
+
================== 8.11.2 ==================
Bug Fixes
diff --git
a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java
b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java
index 3a42a12..6df02b2 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java
@@ -45,11 +45,14 @@ import org.apache.solr.handler.component.ShardHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.URI;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import java.util.Optional;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
@@ -255,30 +258,66 @@ public class BackupCmd implements
CollApiCmds.CollectionApiCommand {
}
//Aggregating result from different shards
- NamedList<Object> aggRsp = aggregateResults(results, collectionName,
backupManager, backupProperties, slices);
+ NamedList<Object> aggRsp = aggregateResults(results, collectionName,
slices, backupManager, backupProperties);
results.add("response", aggRsp);
}
- private NamedList<Object> aggregateResults(NamedList<Object> results, String
collectionName,
- BackupManager backupManager,
- BackupProperties backupProps,
- Collection<Slice> slices) {
+ private NamedList<Object> aggregateResults(NamedList<Object> results,
+ String collectionName,
+ Collection<Slice> slices,
+ @Nullable BackupManager
backupManager,
+ @Nullable BackupProperties
backupProps) {
NamedList<Object> aggRsp = new SimpleOrderedMap<>();
aggRsp.add("collection", collectionName);
aggRsp.add("numShards", slices.size());
- aggRsp.add("backupId", backupManager.getBackupId().id);
- aggRsp.add("indexVersion", backupProps.getIndexVersion());
- aggRsp.add("startTime", backupProps.getStartTime());
+ if (backupManager != null) {
+ aggRsp.add("backupId", backupManager.getBackupId().id);
+ }
+ if (backupProps != null) {
+ aggRsp.add("indexVersion", backupProps.getIndexVersion());
+ aggRsp.add("startTime", backupProps.getStartTime());
+ }
- double indexSizeMB = 0;
+ // Optional options for incremental backups
+ Optional<Integer> indexFileCount = Optional.empty();
+ Optional<Integer> uploadedIndexFileCount = Optional.empty();
+ Optional<Double> indexSizeMB = Optional.empty();
+ Optional<Double> uploadedIndexFileMB = Optional.empty();
NamedList<?> shards = (NamedList<?>) results.get("success");
+ List<String> shardBackupIds = new ArrayList<>(shards.size());
for (int i = 0; i < shards.size(); i++) {
NamedList<?> shardResp =
(NamedList<?>)((NamedList<?>)shards.getVal(i)).get("response");
if (shardResp == null)
continue;
- indexSizeMB += (double) shardResp.get("indexSizeMB");
+ Integer shardIndexFileCount = (Integer) shardResp.get("indexFileCount");
+ if (shardIndexFileCount != null) {
+ indexFileCount = Optional.of(indexFileCount.orElse(0) +
shardIndexFileCount);
+ }
+ Integer shardUploadedIndexFileCount = (Integer)
shardResp.get("uploadedIndexFileCount");
+ if (shardUploadedIndexFileCount != null) {
+ uploadedIndexFileCount = Optional.of(uploadedIndexFileCount.orElse(0)
+ shardUploadedIndexFileCount);
+ }
+ Double shardIndexSizeMB = (Double) shardResp.get("indexSizeMB");
+ if (shardUploadedIndexFileCount != null) {
+ indexSizeMB = Optional.of(indexSizeMB.orElse(0.0) + shardIndexSizeMB);
+ }
+ Double shardUploadedIndexFileMB = (Double)
shardResp.get("uploadedIndexFileMB");
+ if (shardUploadedIndexFileMB != null) {
+ uploadedIndexFileMB = Optional.of(uploadedIndexFileMB.orElse(0.0) +
shardUploadedIndexFileMB);
+ }
+ Optional.ofNullable((String)
shardResp.get("shardBackupId")).ifPresent(shardBackupIds::add);
+ }
+ if (backupProps != null) {
+ backupProps.countIndexFiles(indexFileCount.orElse(0),
indexSizeMB.orElse(0.0));
}
- aggRsp.add("indexSizeMB", indexSizeMB);
+ indexFileCount.ifPresent(val -> aggRsp.add("indexFileCount", val));
+ uploadedIndexFileCount.ifPresent(val ->
aggRsp.add("uploadedIndexFileCount", val));
+ indexSizeMB.ifPresent(val -> aggRsp.add("indexSizeMB", val));
+ uploadedIndexFileMB.ifPresent(val -> aggRsp.add("uploadedIndexFileMB",
val));
+ if (!shardBackupIds.isEmpty()) {
+ aggRsp.add("shardBackupIds", shardBackupIds);
+ }
+
return aggRsp;
}
@@ -323,7 +362,8 @@ public class BackupCmd implements
CollApiCmds.CollectionApiCommand {
}
final ShardRequestTracker shardRequestTracker =
CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc);
- for (Slice slice :
ccc.getZkStateReader().getClusterState().getCollection(collectionName).getActiveSlices())
{
+ Collection<Slice> slices =
ccc.getZkStateReader().getClusterState().getCollection(collectionName).getActiveSlices();
+ for (Slice slice : slices) {
Replica replica = null;
if (snapshotMeta.isPresent()) {
@@ -354,5 +394,9 @@ public class BackupCmd implements
CollApiCmds.CollectionApiCommand {
log.debug("Sent backup requests to all shard leaders for backupName={}",
backupName);
shardRequestTracker.processResponses(results, shardHandler, true, "Could
not backup all shards");
+
+ //Aggregating result from different shards
+ NamedList<Object> aggRsp = aggregateResults(results, collectionName,
slices, null, null);
+ results.add("response", aggRsp);
}
}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
index 6f919d0..7b095f8 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
@@ -62,6 +62,7 @@ public class BackupManager {
public static final String BACKUP_NAME_PROP = "backupName";
public static final String INDEX_VERSION_PROP = "indexVersion";
public static final String START_TIME_PROP = "startTime";
+ public static final String END_TIME_PROP = "endTime";
protected final ZkStateReader zkStateReader;
protected final BackupRepository repository;
diff --git
a/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java
b/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java
index 9ea2c7e..3a4fa6f 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java
@@ -125,6 +125,7 @@ public class BackupProperties {
public void store(Writer propsWriter) throws IOException {
properties.put("indexSizeMB", String.valueOf(indexSizeMB));
properties.put("indexFileCount", String.valueOf(indexFileCount));
+ properties.put(BackupManager.END_TIME_PROP, Instant.now().toString());
properties.store(propsWriter, "Backup properties file");
}
@@ -145,6 +146,10 @@ public class BackupProperties {
return properties.getProperty(BackupManager.START_TIME_PROP);
}
+ public String getEndTime() {
+ return properties.getProperty(BackupManager.END_TIME_PROP);
+ }
+
public String getIndexVersion() {
return properties.getProperty(BackupManager.INDEX_VERSION_PROP);
}
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java
b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java
index 0dfc6e1..3129b60 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java
@@ -82,7 +82,7 @@ class BackupCoreOp implements CoreAdminHandler.CoreAdminOp {
"requires a shared file system mounted at the same
path on all nodes!");
}
snapShooter.validateCreateSnapshot();
- snapShooter.createSnapshot();
+ it.rsp.addResponse(snapShooter.createSnapshot());
}
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
diff --git
a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
index f7438dd..52b2944 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
@@ -89,9 +89,9 @@ public class CoreAdminHandler extends RequestHandlerBase
implements PermissionNa
public static String RUNNING = "running";
public static String COMPLETED = "completed";
public static String FAILED = "failed";
- public static String RESPONSE = "Response";
public static String RESPONSE_STATUS = "STATUS";
public static String RESPONSE_MESSAGE = "msg";
+ public static String OPERATION_RESPONSE = "response";
public CoreAdminHandler() {
super();
@@ -198,6 +198,7 @@ public class CoreAdminHandler extends RequestHandlerBase
implements PermissionNa
try {
callInfo.call();
taskObject.setRspObject(callInfo.rsp);
+ taskObject.setOperationRspObject(callInfo.rsp);
} catch (Exception e) {
exceptionCaught = true;
taskObject.setRspObjectFromException(e);
@@ -325,6 +326,7 @@ public class CoreAdminHandler extends RequestHandlerBase
implements PermissionNa
static class TaskObject {
String taskId;
String rspInfo;
+ Object operationRspInfo;
public TaskObject(String taskId) {
this.taskId = taskId;
@@ -341,6 +343,14 @@ public class CoreAdminHandler extends RequestHandlerBase
implements PermissionNa
public void setRspObjectFromException(Exception e) {
this.rspInfo = e.getMessage();
}
+
+ public Object getOperationRspObject() {
+ return operationRspInfo;
+ }
+
+ public void setOperationRspObject(SolrQueryResponse rspObject) {
+ this.operationRspInfo = rspObject.getResponse();
+ }
}
/**
diff --git
a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
index 7ac25a2..c0b260a 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
@@ -56,7 +56,7 @@ import static
org.apache.solr.common.params.CoreAdminParams.SHARD;
import static org.apache.solr.handler.admin.CoreAdminHandler.COMPLETED;
import static org.apache.solr.handler.admin.CoreAdminHandler.CallInfo;
import static org.apache.solr.handler.admin.CoreAdminHandler.FAILED;
-import static org.apache.solr.handler.admin.CoreAdminHandler.RESPONSE;
+import static
org.apache.solr.handler.admin.CoreAdminHandler.OPERATION_RESPONSE;
import static org.apache.solr.handler.admin.CoreAdminHandler.RESPONSE_MESSAGE;
import static org.apache.solr.handler.admin.CoreAdminHandler.RESPONSE_STATUS;
import static org.apache.solr.handler.admin.CoreAdminHandler.RUNNING;
@@ -185,10 +185,11 @@ public enum CoreAdminOperation implements CoreAdminOp {
it.rsp.add(RESPONSE_STATUS, RUNNING);
} else if
(it.handler.getRequestStatusMap(COMPLETED).containsKey(requestId)) {
it.rsp.add(RESPONSE_STATUS, COMPLETED);
- it.rsp.add(RESPONSE,
it.handler.getRequestStatusMap(COMPLETED).get(requestId).getRspObject());
+ it.rsp.add(RESPONSE_MESSAGE,
it.handler.getRequestStatusMap(COMPLETED).get(requestId).getRspObject());
+ it.rsp.add(OPERATION_RESPONSE,
it.handler.getRequestStatusMap(COMPLETED).get(requestId).getOperationRspObject());
} else if (it.handler.getRequestStatusMap(FAILED).containsKey(requestId)) {
it.rsp.add(RESPONSE_STATUS, FAILED);
- it.rsp.add(RESPONSE,
it.handler.getRequestStatusMap(FAILED).get(requestId).getRspObject());
+ it.rsp.add(RESPONSE_MESSAGE,
it.handler.getRequestStatusMap(FAILED).get(requestId).getRspObject());
} else {
it.rsp.add(RESPONSE_STATUS, "notfound");
it.rsp.add(RESPONSE_MESSAGE, "No task found in running, completed or
failed tasks");
diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
index e9cf158..23d3063 100644
--- a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
+++ b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
@@ -98,6 +98,13 @@ All the usages are replaced by
BaseHttpSolrClient.RemoteSolrException and BaseHt
* SOLR-14142: Jetty low level request-logging in NCSA format is now enabled by
default, with a retention of 3 days worth of logs.
This may require some more disk space for logs than was the case in 8.x. See
Reference Guide chapter "Configuring Logging" for how to change this.
+* SOLR-15842: Async responses for backups now correctly aggregate and return
information.
+In previous versions there was a field returned in async backup status
responses, `Response`. This has now been renamed to `msg`, to better fit other
collections API responses.
+The `response` field is now a map, containing information about the backup
(`startTime`, `indexSizeMB`, `indexFileCount`, etc.).
+
+* SOLR-15884: In Backup request responses, the `response` key now uses a map
to return information instead of a list.
+This is only applicable for users returning information in JSON format, which
is the default behavior.
+
== New Features & Enhancements
* Replica placement plugins
@@ -143,9 +150,6 @@ Currently this change should only effect compatibility of
custom code overriding
* SOLR-14510: The `writeStartDocumentList` in `TextResponseWriter` now
receives an extra boolean parameter representing the "exactness" of the
`numFound` value (exact vs approximation).
Any custom response writer extending `TextResponseWriter` will need to
implement this abstract method now (instead previous with the same name but
without the new boolean parameter).
-* SOLR-15884: In Backup request responses, the `response` key now uses a map
to return information instead of a list.
-This is only applicable for users returning information in JSON format, which
is the default behavior.
-
=== solr.xml maxBooleanClauses now enforced recursively
Lucene 9.0 has additional safety checks over previous versions that impact how
the `solr.xml` global
`<<configuring-solr-xml#global-maxbooleanclauses,maxBooleanClauses>>` option is
enforced.
