This is an automated email from the ASF dual-hosted git repository.
epugh pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 66a16d1994a SOLR-17180: Deprecate snapshotscli.sh in favour of
bin/solr sub commands. (#2381)
66a16d1994a is described below
commit 66a16d1994a5d728e2650b1014611104f657859b
Author: Eric Pugh <[email protected]>
AuthorDate: Thu Aug 22 12:28:14 2024 -0500
SOLR-17180: Deprecate snapshotscli.sh in favour of bin/solr sub commands.
(#2381)
* Migrating some tools for managing snapshots and backups into bin/solr
* Migrate into the modules/hdfs/bin a hdfs specific script for preparing an
export that uses hdfs specific tooling
* Introduce Ref Guide documentation and unit tests
(cherry picked from commit 5374d3428248444efcb36d1771dea38f26c71d33)
---
solr/CHANGES.txt | 1 +
.../org/apache/solr/cli/SnapshotCreateTool.java | 103 +++++++++++++++
.../org/apache/solr/cli/SnapshotDeleteTool.java | 103 +++++++++++++++
.../org/apache/solr/cli/SnapshotDescribeTool.java | 142 +++++++++++++++++++++
.../org/apache/solr/cli/SnapshotExportTool.java | 132 +++++++++++++++++++
.../java/org/apache/solr/cli/SnapshotListTool.java | 96 ++++++++++++++
.../core/src/java/org/apache/solr/cli/SolrCLI.java | 11 +-
.../hdfs/bin/prepare-snapshot-export.sh} | 86 +++++++------
solr/modules/hdfs/build.gradle | 7 +
.../solr/hdfs/snapshots/SolrSnapshotsTool.java | 27 ++++
solr/packaging/test/test_snapshots.bats | 77 +++++++++++
solr/server/README.md | 13 +-
solr/server/scripts/cloud-scripts/snapshotscli.sh | 3 +-
.../pages/solr-control-script-reference.adoc | 66 ++++++++++
14 files changed, 821 insertions(+), 46 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index d74bac4c0f9..e5e088b07bd 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -12,6 +12,7 @@ New Features
Improvements
---------------------
* SOLR-17397: SkipExistingDocumentsProcessor now functions correctly with
child documents. (Tim Owens via Eric Pugh)
+* SOLR-17180: Deprecate snapshotscli.sh in favour of bin/solr snapshot sub
commands. Now able to manage Snapshots from the CLI. HDFS module specific
snapshot script now ships as part of that module in the modules/hdfs/bin
directory. (Eric Pugh)
Optimizations
---------------------
diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotCreateTool.java
b/solr/core/src/java/org/apache/solr/cli/SnapshotCreateTool.java
new file mode 100644
index 00000000000..41a5cf13a6b
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cli/SnapshotCreateTool.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cli;
+
+import java.io.PrintStream;
+import java.util.List;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+
+/** Supports snapshot-create command in the bin/solr script. */
+public class SnapshotCreateTool extends ToolBase {
+
+ public SnapshotCreateTool() {
+ this(CLIO.getOutStream());
+ }
+
+ public SnapshotCreateTool(PrintStream stdout) {
+ super(stdout);
+ }
+
+ @Override
+ public String getName() {
+ return "snapshot-create";
+ }
+
+ @Override
+ public List<Option> getOptions() {
+ return List.of(
+ SolrCLI.OPTION_ZKHOST,
+ SolrCLI.OPTION_SOLRURL,
+ Option.builder("c")
+ .longOpt("name")
+ .argName("NAME")
+ .hasArg()
+ .required(true)
+ .desc("Name of collection to be snapshot.")
+ .build(),
+ Option.builder()
+ .longOpt("snapshot-name")
+ .argName("NAME")
+ .hasArg()
+ .required(true)
+ .desc("Name of the snapshot to produce")
+ .build(),
+ SolrCLI.OPTION_CREDENTIALS,
+ SolrCLI.OPTION_VERBOSE);
+ }
+
+ @Override
+ public void runImpl(CommandLine cli) throws Exception {
+ SolrCLI.raiseLogLevelUnlessVerbose(cli);
+
+ String snapshotName = cli.getOptionValue("snapshot-name");
+ String collectionName = cli.getOptionValue("name");
+ try (var solrClient = SolrCLI.getSolrClient(cli)) {
+ createSnapshot(solrClient, collectionName, snapshotName);
+ }
+ }
+
+ public void createSnapshot(SolrClient solrClient, String collectionName,
String snapshotName) {
+ CollectionAdminRequest.CreateSnapshot createSnapshot =
+ new CollectionAdminRequest.CreateSnapshot(collectionName,
snapshotName);
+ CollectionAdminResponse resp;
+ try {
+ resp = createSnapshot.process(solrClient);
+ if (resp.getStatus() != 0) {
+ throw new IllegalStateException(
+ "The CREATESNAPSHOT request failed. The status code is " +
resp.getStatus());
+ }
+ echo(
+ "Successfully created snapshot with name "
+ + snapshotName
+ + " for collection "
+ + collectionName);
+
+ } catch (Exception e) {
+ echo(
+ "Failed to create a snapshot with name "
+ + snapshotName
+ + " for collection "
+ + collectionName
+ + " due to following error : "
+ + e.getLocalizedMessage());
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotDeleteTool.java
b/solr/core/src/java/org/apache/solr/cli/SnapshotDeleteTool.java
new file mode 100644
index 00000000000..ee8a0506492
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cli/SnapshotDeleteTool.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cli;
+
+import java.io.PrintStream;
+import java.util.List;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+
+/** Supports snapshot-delete command in the bin/solr script. */
+public class SnapshotDeleteTool extends ToolBase {
+
+ public SnapshotDeleteTool() {
+ this(CLIO.getOutStream());
+ }
+
+ public SnapshotDeleteTool(PrintStream stdout) {
+ super(stdout);
+ }
+
+ @Override
+ public String getName() {
+ return "snapshot-delete";
+ }
+
+ @Override
+ public List<Option> getOptions() {
+ return List.of(
+ SolrCLI.OPTION_ZKHOST,
+ SolrCLI.OPTION_SOLRURL,
+ Option.builder("c")
+ .longOpt("name")
+ .argName("NAME")
+ .hasArg()
+ .required(true)
+ .desc("Name of collection to manage.")
+ .build(),
+ Option.builder()
+ .longOpt("snapshot-name")
+ .argName("NAME")
+ .hasArg()
+ .required(true)
+ .desc("Name of the snapshot to delete")
+ .build(),
+ SolrCLI.OPTION_CREDENTIALS,
+ SolrCLI.OPTION_VERBOSE);
+ }
+
+ @Override
+ public void runImpl(CommandLine cli) throws Exception {
+ SolrCLI.raiseLogLevelUnlessVerbose(cli);
+
+ String snapshotName = cli.getOptionValue("snapshot-name");
+ String collectionName = cli.getOptionValue("name");
+ try (var solrClient = SolrCLI.getSolrClient(cli)) {
+ deleteSnapshot(solrClient, collectionName, snapshotName);
+ }
+ }
+
+ public void deleteSnapshot(SolrClient solrClient, String collectionName,
String snapshotName) {
+ CollectionAdminRequest.DeleteSnapshot deleteSnapshot =
+ new CollectionAdminRequest.DeleteSnapshot(collectionName,
snapshotName);
+ CollectionAdminResponse resp;
+ try {
+ resp = deleteSnapshot.process(solrClient);
+ if (resp.getStatus() != 0) {
+ throw new IllegalStateException(
+ "The DELETESNAPSHOT request failed. The status code is " +
resp.getStatus());
+ }
+ echo(
+ "Successfully deleted snapshot with name "
+ + snapshotName
+ + " for collection "
+ + collectionName);
+
+ } catch (Exception e) {
+ echo(
+ "Failed to delete a snapshot with name "
+ + snapshotName
+ + " for collection "
+ + collectionName
+ + " due to following error : "
+ + e.getLocalizedMessage());
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotDescribeTool.java
b/solr/core/src/java/org/apache/solr/cli/SnapshotDescribeTool.java
new file mode 100644
index 00000000000..94ef6d98a9e
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cli/SnapshotDescribeTool.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cli;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.snapshots.CollectionSnapshotMetaData;
+import org.apache.solr.core.snapshots.SolrSnapshotManager;
+
+/** Supports snapshot-describe command in the bin/solr script. */
+public class SnapshotDescribeTool extends ToolBase {
+
+ public SnapshotDescribeTool() {
+ this(CLIO.getOutStream());
+ }
+
+ public SnapshotDescribeTool(PrintStream stdout) {
+ super(stdout);
+ }
+
+ private static final DateFormat dateFormat =
+ new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z", Locale.getDefault());
+
+ @Override
+ public String getName() {
+ return "snapshot-describe";
+ }
+
+ @Override
+ public List<Option> getOptions() {
+ return List.of(
+ SolrCLI.OPTION_ZKHOST,
+ SolrCLI.OPTION_SOLRURL,
+ Option.builder("c")
+ .longOpt("name")
+ .argName("NAME")
+ .hasArg()
+ .required(true)
+ .desc("Name of collection to be snapshot.")
+ .build(),
+ Option.builder()
+ .longOpt("snapshot-name")
+ .argName("NAME")
+ .hasArg()
+ .required(true)
+ .desc("Name of the snapshot to describe")
+ .build(),
+ SolrCLI.OPTION_CREDENTIALS,
+ SolrCLI.OPTION_VERBOSE);
+ }
+
+ @Override
+ public void runImpl(CommandLine cli) throws Exception {
+ SolrCLI.raiseLogLevelUnlessVerbose(cli);
+
+ String snapshotName = cli.getOptionValue("snapshot-name");
+ String collectionName = cli.getOptionValue("name");
+ try (var solrClient = SolrCLI.getSolrClient(cli)) {
+ describeSnapshot(solrClient, collectionName, snapshotName);
+ }
+ }
+
+ public void describeSnapshot(SolrClient solrClient, String collectionName,
String snapshotName) {
+ try {
+ Collection<CollectionSnapshotMetaData> snaps =
+ listCollectionSnapshots(solrClient, collectionName);
+ for (CollectionSnapshotMetaData m : snaps) {
+ if (snapshotName.equals(m.getName())) {
+ echo("Name: " + m.getName());
+ echo("Status: " + m.getStatus());
+ echo("Time of creation: " + dateFormat.format(m.getCreationDate()));
+ echo("Total number of cores with snapshot: " +
m.getReplicaSnapshots().size());
+ echo("-----------------------------------");
+ for (CollectionSnapshotMetaData.CoreSnapshotMetaData n :
m.getReplicaSnapshots()) {
+ String builder =
+ "Core [name="
+ + n.getCoreName()
+ + ", leader="
+ + n.isLeader()
+ + ", generation="
+ + n.getGenerationNumber()
+ + ", indexDirPath="
+ + n.getIndexDirPath()
+ + "]\n";
+ echo(builder);
+ }
+ }
+ }
+ } catch (Exception e) {
+ echo("Failed to fetch snapshot details due to following error : " +
e.getLocalizedMessage());
+ }
+ }
+
+ private Collection<CollectionSnapshotMetaData> listCollectionSnapshots(
+ SolrClient solrClient, String collectionName) throws
SolrServerException, IOException {
+ CollectionAdminRequest.ListSnapshots listSnapshots =
+ new CollectionAdminRequest.ListSnapshots(collectionName);
+ CollectionAdminResponse resp = listSnapshots.process(solrClient);
+
+ if (resp.getStatus() != 0) {
+ throw new IllegalStateException(
+ "The LISTSNAPSHOTS request failed. The status code is " +
resp.getStatus());
+ }
+
+ NamedList<?> apiResult =
+ (NamedList<?>)
resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);
+
+ Collection<CollectionSnapshotMetaData> result = new ArrayList<>();
+ for (int i = 0; i < apiResult.size(); i++) {
+ result.add(new CollectionSnapshotMetaData((NamedList<?>)
apiResult.getVal(i)));
+ }
+
+ return result;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotExportTool.java
b/solr/core/src/java/org/apache/solr/cli/SnapshotExportTool.java
new file mode 100644
index 00000000000..d38a1b19b69
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cli/SnapshotExportTool.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cli;
+
+import java.io.PrintStream;
+import java.util.List;
+import java.util.Optional;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.common.params.CollectionAdminParams;
+
+/** Supports snapshot-export command in the bin/solr script. */
+public class SnapshotExportTool extends ToolBase {
+
+ public SnapshotExportTool() {
+ this(CLIO.getOutStream());
+ }
+
+ public SnapshotExportTool(PrintStream stdout) {
+ super(stdout);
+ }
+
+ @Override
+ public String getName() {
+ return "snapshot-export";
+ }
+
+ @Override
+ public List<Option> getOptions() {
+ return List.of(
+ SolrCLI.OPTION_ZKHOST,
+ SolrCLI.OPTION_SOLRURL,
+ Option.builder("c")
+ .longOpt("name")
+ .argName("NAME")
+ .hasArg()
+ .required(true)
+ .desc("Name of collection to be snapshot.")
+ .build(),
+ Option.builder()
+ .longOpt("snapshot-name")
+ .argName("NAME")
+ .hasArg()
+ .required(true)
+ .desc("Name of the snapshot to be exported.")
+ .build(),
+ Option.builder()
+ .longOpt("dest-dir")
+ .argName("DIR")
+ .hasArg()
+ .required(true)
+ .desc(
+ "Path of a temporary directory on local filesystem during
snapshot export command.")
+ .build(),
+ Option.builder()
+ .longOpt("backup-repo-name")
+ .argName("DIR")
+ .hasArg()
+ .required(false)
+ .desc(
+ "Specifies name of the backup repository to be used during
snapshot export preparation.")
+ .build(),
+ Option.builder("i")
+ .longOpt("async-id")
+ .argName("ID")
+ .hasArg()
+ .required(false)
+ .desc(
+ "Specifies the async request identifier to be used during
snapshot export preparation.")
+ .build(),
+ SolrCLI.OPTION_CREDENTIALS,
+ SolrCLI.OPTION_VERBOSE);
+ }
+
+ @Override
+ public void runImpl(CommandLine cli) throws Exception {
+ SolrCLI.raiseLogLevelUnlessVerbose(cli);
+ //
+ String snapshotName = cli.getOptionValue("snapshot-name");
+ String collectionName = cli.getOptionValue("name");
+ String destDir = cli.getOptionValue("dest-dir");
+ Optional<String> backupRepo =
Optional.ofNullable(cli.getOptionValue("backup-repo-name"));
+ Optional<String> asyncReqId =
Optional.ofNullable(cli.getOptionValue("async-id"));
+
+ try (var solrClient = SolrCLI.getSolrClient(cli)) {
+ exportSnapshot(solrClient, collectionName, snapshotName, destDir,
backupRepo, asyncReqId);
+ }
+ }
+
+ public void exportSnapshot(
+ SolrClient solrClient,
+ String collectionName,
+ String snapshotName,
+ String destPath,
+ Optional<String> backupRepo,
+ Optional<String> asyncReqId) {
+ try {
+ CollectionAdminRequest.Backup backup =
+ new CollectionAdminRequest.Backup(collectionName, snapshotName);
+ backup.setCommitName(snapshotName);
+ backup.setIndexBackupStrategy(CollectionAdminParams.COPY_FILES_STRATEGY);
+ backup.setLocation(destPath);
+ if (backupRepo.isPresent()) {
+ backup.setRepositoryName(backupRepo.get());
+ }
+ // if asyncId is null, processAsync will block and throw an Exception
with any error
+ backup.processAsync(asyncReqId.orElse(null), solrClient);
+ } catch (Exception e) {
+ throw new IllegalStateException(
+ "Failed to backup collection meta-data for collection "
+ + collectionName
+ + " due to following error : "
+ + e.getLocalizedMessage());
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotListTool.java
b/solr/core/src/java/org/apache/solr/cli/SnapshotListTool.java
new file mode 100644
index 00000000000..04cd0738825
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cli/SnapshotListTool.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cli;
+
+import java.io.PrintStream;
+import java.util.List;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.snapshots.SolrSnapshotManager;
+
+/** Supports snapshot-list command in the bin/solr script. */
+public class SnapshotListTool extends ToolBase {
+
+ public SnapshotListTool() {
+ this(CLIO.getOutStream());
+ }
+
+ public SnapshotListTool(PrintStream stdout) {
+ super(stdout);
+ }
+
+ @Override
+ public String getName() {
+ return "snapshot-list";
+ }
+
+ @Override
+ public List<Option> getOptions() {
+ return List.of(
+ SolrCLI.OPTION_ZKHOST,
+ SolrCLI.OPTION_SOLRURL,
+ Option.builder("c")
+ .longOpt("name")
+ .argName("NAME")
+ .hasArg()
+ .required(true)
+ .desc("Name of collection to list snapshots for.")
+ .build(),
+ SolrCLI.OPTION_CREDENTIALS,
+ SolrCLI.OPTION_VERBOSE);
+ }
+
+ @Override
+ public void runImpl(CommandLine cli) throws Exception {
+ SolrCLI.raiseLogLevelUnlessVerbose(cli);
+
+ String collectionName = cli.getOptionValue("name");
+ try (var solrClient = SolrCLI.getSolrClient(cli)) {
+ listSnapshots(solrClient, collectionName);
+ }
+ }
+
+ public void listSnapshots(SolrClient solrClient, String collectionName) {
+ CollectionAdminRequest.ListSnapshots listSnaps =
+ new CollectionAdminRequest.ListSnapshots(collectionName);
+ CollectionAdminResponse resp;
+ try {
+ resp = listSnaps.process(solrClient);
+ if (resp.getStatus() != 0) {
+ throw new IllegalStateException(
+ "The LISTSNAPSHOTS request failed. The status code is " +
resp.getStatus());
+ }
+
+ NamedList<?> apiResult =
+ (NamedList<?>)
resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);
+ for (int i = 0; i < apiResult.size(); i++) {
+ echo(apiResult.getName(i));
+ }
+
+ } catch (Exception e) {
+ echo(
+ "Failed to list snapshots for collection "
+ + collectionName
+ + " due to following error : "
+ + e.getLocalizedMessage());
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
index 99b2b4d12c0..be597c30363 100755
--- a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
+++ b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
@@ -325,6 +325,11 @@ public class SolrCLI implements CLIO {
else if ("postlogs".equals(toolType)) return new PostLogsTool();
else if ("version".equals(toolType)) return new VersionTool();
else if ("post".equals(toolType)) return new PostTool();
+ else if ("snapshot-create".equals(toolType)) return new
SnapshotCreateTool();
+ else if ("snapshot-delete".equals(toolType)) return new
SnapshotDeleteTool();
+ else if ("snapshot-list".equals(toolType)) return new SnapshotListTool();
+ else if ("snapshot-describe".equals(toolType)) return new
SnapshotDescribeTool();
+ else if ("snapshot-export".equals(toolType)) return new
SnapshotExportTool();
// If you add a built-in tool to this class, add it here to avoid
// classpath scanning
@@ -568,7 +573,11 @@ public class SolrCLI implements CLIO {
print("Usage: solr COMMAND OPTIONS");
print(
- " where COMMAND is one of: start, stop, restart, status,
healthcheck, create, delete, version, zk, auth, assert, config, export, api,
package, post");
+ " where COMMAND is one of: start, stop, restart, status,
healthcheck, create, delete, version, auth, assert, config, export, api,
package, post, ");
+ print(
+ " zk ls, zk cp, zk rm , zk mv, zk
mkroot, zk upconfig, zk downconfig,");
+ print(
+ " snapshot-create, snapshot-list,
snapshot-delete, snapshot-export, snapshot-prepare-export");
print("");
print(" Standalone server example (start Solr running in the background
on port 8984):");
print("");
diff --git a/solr/server/scripts/cloud-scripts/snapshotscli.sh
b/solr/modules/hdfs/bin/prepare-snapshot-export.sh
similarity index 64%
copy from solr/server/scripts/cloud-scripts/snapshotscli.sh
copy to solr/modules/hdfs/bin/prepare-snapshot-export.sh
index 5db119efab7..7de91e19197 100755
--- a/solr/server/scripts/cloud-scripts/snapshotscli.sh
+++ b/solr/modules/hdfs/bin/prepare-snapshot-export.sh
@@ -1,21 +1,32 @@
#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
set -e
-run_solr_snapshot_tool() {
- JVM="java"
- scriptDir=$(dirname "$0")
- if [ -n "$LOG4J_PROPS" ]; then
- log4j_config="file:${LOG4J_PROPS}"
- else
- log4j_config="file:${scriptDir}/../../resources/log4j2-console.xml"
- fi
- PATH=${JAVA_HOME}/bin:${PATH} ${JVM} ${ZKCLI_JVM_FLAGS}
-Dlog4j.configurationFile=${log4j_config} \
- -classpath "${solrLibPath}" org.apache.solr.hdfs.snapshots.SolrSnapshotsTool
"$@" 2> /dev/null
-}
-
usage() {
- run_solr_snapshot_tool --help
+ echo "This script is to support the specific use case of using Hadoop
specific libraries for managing Solr snapshot."
+ echo "--prepare-snapshot-export <arg> This command will prepare"
+ echo " copylistings for the specified"
+ echo " snapshot. This command should only"
+ echo " be used only if Solr is deployed"
+ echo " with Hadoop and collection index"
+ echo " files are stored on a shared"
+ echo " file-system e.g. HDFS"
+ echo "--export <arg> This command will create a backup"
+ echo " for the specified snapshot."
}
distcp_warning() {
@@ -23,9 +34,13 @@ distcp_warning() {
Do you want to use hadoop distcp tool for exporting Solr collection
snapshot ?"
}
+use_bin_solr_warning() {
+ echo "Please use the bin/solr script to execute this command."
+}
+
parse_options() {
OPTIND=3
- while getopts ":c:d:s:z:p:r:i:" o ; do
+ while getopts ":c:d:s:" o ; do
case "${o}" in
d)
destPath=${OPTARG}
@@ -36,18 +51,6 @@ parse_options() {
c)
collectionName=${OPTARG}
;;
- z)
- solrZkEnsemble=${OPTARG}
- ;;
- p)
- pathPrefix=${OPTARG}
- ;;
- r)
- backupRepoName=${OPTARG}
- ;;
- i)
- aysncReqId=${OPTARG}
- ;;
*)
echo "Unknown option ${OPTARG}"
usage 1>&2
@@ -57,6 +60,18 @@ parse_options() {
done
}
+run_solr_snapshot_tool() {
+ JVM="java"
+ scriptDir=$(dirname "$0")
+ if [ -n "$LOG4J_PROPS" ]; then
+ log4j_config="file:${LOG4J_PROPS}"
+ else
+
log4j_config="file:${scriptDir}/../../../server/resources/log4j2-console.xml"
+ fi
+ PATH=${JAVA_HOME}/bin:${PATH} ${JVM} ${ZKCLI_JVM_FLAGS}
-Dlog4j.configurationFile=${log4j_config} \
+ -classpath "${solrLibPath}" org.apache.solr.hdfs.snapshots.SolrSnapshotsTool
"$@" 2> /dev/null
+}
+
prepare_snapshot_export() {
#Make sure to cleanup the temporary files.
scratch=$(mktemp -d -t solrsnaps.XXXXXXXXXX)
@@ -95,8 +110,6 @@ copy_snapshot_files() {
}
collectionName=""
-solrZkEnsemble=""
-pathPrefix=""
destPath=""
sourcePath=""
cmd="$1"
@@ -104,20 +117,19 @@ snapshotName="$2"
copyListingDirPath=""
distCpCmd="${SOLR_DISTCP_CMD:-hadoop distcp}"
scriptDir=$(dirname "$0")
-solrLibPath="${SOLR_LIB_PATH:-${scriptDir}/../../solr-webapp/webapp/WEB-INF/lib/*:${scriptDir}/../../lib/ext/*:${scriptDir}/../../../modules/hdfs/lib/*}"
-
+solrLibPath="${SOLR_LIB_PATH:-${scriptDir}/../../../server/solr-webapp/webapp/WEB-INF/lib/*:${scriptDir}/../../../server/lib/ext/*:${scriptDir}/../lib/*}"
case "${cmd}" in
--create)
- run_solr_snapshot_tool "$@"
+ use_bin_solr_warning
;;
--delete)
- run_solr_snapshot_tool "$@"
+ use_bin_solr_warning
;;
--list)
- run_solr_snapshot_tool "$@"
+ use_bin_solr_warning
;;
--describe)
- run_solr_snapshot_tool "$@"
+ use_bin_solr_warning
;;
--prepare-snapshot-export)
: "${SOLR_USE_DISTCP:? $(distcp_warning)}"
@@ -132,7 +144,8 @@ case "${cmd}" in
;;
--export)
if [ -z "${SOLR_USE_DISTCP}" ]; then
- run_solr_snapshot_tool "$@"
+ echo "Not using the distcp command"
+ use_bin_solr_warning
echo "Done. GoodBye!"
exit 0
fi
@@ -165,7 +178,7 @@ case "${cmd}" in
echo "Done. GoodBye!"
;;
- --help|-h|-help)
+ --help|-h)
usage 1>&2
;;
*)
@@ -173,4 +186,3 @@ case "${cmd}" in
usage 1>&2
exit 1
esac
-
diff --git a/solr/modules/hdfs/build.gradle b/solr/modules/hdfs/build.gradle
index ad05e9bcd25..07e222bc3d2 100644
--- a/solr/modules/hdfs/build.gradle
+++ b/solr/modules/hdfs/build.gradle
@@ -94,3 +94,10 @@ tasks.withType(Test).configureEach {
jvmArgs(["--add-opens=java.base/java.lang=ALL-UNNAMED",
"--add-opens=java.base/java.util=ALL-UNNAMED"])
}
+
+assemblePackaging {
+ // Add bin folder to default packaging.
+ from(projectDir, {
+ include "bin/**"
+ })
+}
diff --git
a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrSnapshotsTool.java
b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrSnapshotsTool.java
index 5f903cbd932..ee49025061c 100644
---
a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrSnapshotsTool.java
+++
b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrSnapshotsTool.java
@@ -112,6 +112,11 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
}
}
+ /**
+ * @deprecated use equivalent snapshot related tool in {@link
+ * org.apache.solr.cli.SnapshotCreateTool}
+ */
+ @Deprecated(since = "9.8.0")
public void createSnapshot(String collectionName, String snapshotName) {
CollectionAdminRequest.CreateSnapshot createSnap =
new CollectionAdminRequest.CreateSnapshot(collectionName,
snapshotName);
@@ -144,6 +149,11 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
}
}
+ /**
+ * @deprecated use equivalent snapshot related tool in {@link
+ * org.apache.solr.cli.SnapshotDeleteTool}
+ */
+ @Deprecated(since = "9.8.0")
public void deleteSnapshot(String collectionName, String snapshotName) {
CollectionAdminRequest.DeleteSnapshot deleteSnap =
new CollectionAdminRequest.DeleteSnapshot(collectionName,
snapshotName);
@@ -180,6 +190,11 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
}
}
+ /**
+ * @deprecated use equivalent snapshot related tool in {@link
+ * org.apache.solr.cli.SnapshotListTool}
+ */
+ @Deprecated(since = "9.8.0")
public void listSnapshots(String collectionName) {
CollectionAdminRequest.ListSnapshots listSnaps =
new CollectionAdminRequest.ListSnapshots(collectionName);
@@ -204,6 +219,11 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
}
}
+ /**
+ * @deprecated use equivalent snapshot related tool in {@link
+ * org.apache.solr.cli.SnapshotDescribeTool}
+ */
+ @Deprecated(since = "9.8.0")
public void describeSnapshot(String collectionName, String snapshotName) {
try {
Collection<CollectionSnapshotMetaData> snaps =
listCollectionSnapshots(collectionName);
@@ -332,6 +352,8 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
}
/**
+ * HDFS specific step required before exporting a snapshot
+ *
* @param pathPrefix optional
*/
public void prepareForExport(
@@ -375,6 +397,11 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
}
}
+ /**
+ * @deprecated use equivalent snapshot related tool in {@link
+ * org.apache.solr.cli.SnapshotExportTool}
+ */
+ @Deprecated(since = "9.8.0")
public void exportSnapshot(
String collectionName,
String snapshotName,
diff --git a/solr/packaging/test/test_snapshots.bats
b/solr/packaging/test/test_snapshots.bats
new file mode 100644
index 00000000000..4a6135bbd01
--- /dev/null
+++ b/solr/packaging/test/test_snapshots.bats
@@ -0,0 +1,77 @@
+#!/usr/bin/env bats
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load bats_helper
+
+setup_file() {
+ common_clean_setup
+
+ solr start -c -e films
+
+}
+
+teardown_file() {
+ common_setup
+ solr stop -all
+
+}
+
+setup() {
+ common_setup
+}
+
+teardown() {
+ # save a snapshot of SOLR_HOME for failed tests
+ save_home_on_failure
+
+}
+
+@test "snapshot lifecycle" {
+ # the way snapshots live, if you run create twice you get an error because
snapshot already exists
+ run solr snapshot-create -c films --snapshot-name snapshot1 --solr-url
http://localhost:${SOLR_PORT}
+ assert_output --partial "Successfully created snapshot with name snapshot1
for collection films"
+
+ run solr snapshot-delete -c films --snapshot-name snapshot1 -url
http://localhost:${SOLR_PORT}
+ assert_output --partial "Successfully deleted snapshot with name snapshot1
for collection films"
+
+ # make sure you can create it again!
+ run solr snapshot-create -c films --snapshot-name snapshot1 -z
localhost:${ZK_PORT}
+ assert_output --partial "Successfully created snapshot with name snapshot1
for collection films"
+
+ run solr snapshot-delete -c films --snapshot-name snapshot1
+ assert_output --partial "Successfully deleted snapshot with name snapshot1
for collection films"
+}
+
+@test "snapshot list" {
+ solr snapshot-create -c films --snapshot-name snapshot3 --solr-url
http://localhost:${SOLR_PORT}
+
+ run solr snapshot-list -c films -url http://localhost:${SOLR_PORT}/solr
+ assert_output --partial "snapshot3"
+
+ run solr snapshot-delete -c films --snapshot-name snapshot3 -url
http://localhost:${SOLR_PORT}
+ assert_output --partial "Successfully deleted snapshot with name snapshot3
for collection films"
+}
+
+@test "snapshot describe" {
+ solr snapshot-create -c films --snapshot-name snapshot4 -url
http://localhost:${SOLR_PORT}
+
+ run solr snapshot-describe -c films --snapshot-name snapshot4
+ assert_output --partial "Name: snapshot4"
+
+ run solr snapshot-delete -c films --snapshot-name snapshot4
+ assert_output --partial "Successfully deleted snapshot with name snapshot4
for collection films"
+}
diff --git a/solr/server/README.md b/solr/server/README.md
index 0c26546614f..41f8dec3d2b 100644
--- a/solr/server/README.md
+++ b/solr/server/README.md
@@ -18,7 +18,7 @@
Solr server
------------
-This directory contains an instance of the Jetty Servlet container setup to
+This directory contains an instance of the Jetty Servlet container setup to
run Solr.
To run Solr:
@@ -88,19 +88,19 @@ Notes About Solr Examples
### SolrHome
By default, start.jar starts Solr in Jetty using the default Solr Home
-directory of "./solr/" (relative to the working directory of the servlet
+directory of "./solr/" (relative to the working directory of the servlet
container).
### References to Jar Files Outside This Directory
Various example SolrHome dirs contained in this directory may use "<lib>"
-statements in the solrconfig.xml file to reference plugin jars outside of
+statements in the solrconfig.xml file to reference plugin jars outside of
this directory for loading modules via relative paths.
-If you make a copy of this example server and wish to use the
+If you make a copy of this example server and wish to use the
ExtractingRequestHandler (SolrCell), the clustering component,
or any other modules, you will need to
-copy the required jars or update the paths to those jars in your
+copy the required jars or update the paths to those jars in your
solrconfig.xml.
### Logging
@@ -109,6 +109,5 @@ By default, Jetty & Solr will log to the console and
logs/solr.log. This can
be convenient when first getting started, but eventually you will want to
log just to a file. To configure logging, edit the log4j2.xml file in
"resources".
-
-It is also possible to setup log4j or other popular logging frameworks.
+It is also possible to setup log4j or other popular logging frameworks.
diff --git a/solr/server/scripts/cloud-scripts/snapshotscli.sh
b/solr/server/scripts/cloud-scripts/snapshotscli.sh
index 5db119efab7..7efcf6ba823 100755
--- a/solr/server/scripts/cloud-scripts/snapshotscli.sh
+++ b/solr/server/scripts/cloud-scripts/snapshotscli.sh
@@ -2,6 +2,8 @@
set -e
+echo "This command is deprecated in favour of bin/solr equivalents and will be
removed in Solr 10."
+
run_solr_snapshot_tool() {
JVM="java"
scriptDir=$(dirname "$0")
@@ -173,4 +175,3 @@ case "${cmd}" in
usage 1>&2
exit 1
esac
-
diff --git
a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc
b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc
index 76d31daff8c..4cda6ef1d20 100644
---
a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc
+++
b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc
@@ -1960,3 +1960,69 @@ Here is an example of sending a SQL query to the
techproducts /sql end point (as
$ bin/solr api --solr-url
http://localhost:8983/solr/techproducts/sql?stmt=select+id+from+techproducts+limit+10
Results are streamed to the terminal.
+
+
+== Managing Solr Packages
+
+The `package` command allows you to interact with Solr's
xref:configuration-guide:package-manager.adoc[Package Manager].
+
+
+== Snapshots and Backups
+
+The snapshots capablity of the CLI allows you to:
+
+* Create snapshotted view of your index
+* List all the available snapshots
+* Provide details about individual Snapshots
+* Export snapshots as part of a backup strategy
+* Delete existing snapshots
+
+Learn more about on the xref:backup-restore.adoc[Backup and Restore] and
xref:collection-management.adoc[Collection Management] pages.
+
+=== Create Snapshotted View
+
+To create a snapshot you just give it the collection and the name you want:
+
+[,console]
+----
+$ bin/solr snapshot-create -c <collection-name> --snapshot-name
<snapshot-name>
+----
+
+=== List all available snapshots
+
+To list all the snapshots that have been taken for a collection:
+
+[,console]
+----
+$ bin/solr snapshot-list -c <collection-name>
+----
+
+=== Detailed information about a Snapshot
+
+Use the describe command to gain detailed information about a specific
snapshot:
+
+[,console]
+----
+$ bin/solr snapshot-describe -c <collection-name> --snapshot-name
<snapshot-name>
+----
+
+=== Converting a Snapshot to a Backup
+
+Use the export command to take :
+
+[,console]
+----
+$ bin/solr snapshot-export [--backup-repo-name <DIR>] -c <NAME> --dest-dir
<DIR> [-i <ID>] --snapshot-name <NAME>
+----
+
+The `-i` parameter specifies that this is an asynchronous process.
+
+
+=== Delete a Snapshot
+
+Use the delete command to delete a snapshot :
+
+[,console]
+----
+$ bin/solr snapshot-delete -c <NAME> --snapshot-name <NAME>
+----
