This is an automated email from the ASF dual-hosted git repository.
epugh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 8cba5bf5328 SOLR-18320: Use Configsets V2 API instead of direct
ZooKeeper access in bin/solr create (#4672)
8cba5bf5328 is described below
commit 8cba5bf53288af47e8177dd9221f6095ca6fbe66
Author: Eric Pugh <[email protected]>
AuthorDate: Tue Oct 27 23:45:01 2026 -0400
SOLR-18320: Use Configsets V2 API instead of direct ZooKeeper access in
bin/solr create (#4672)
Co-authored-by: Claude Sonnet 5 <[email protected]>
---
.../SOLR-18320-skip-zk-for-configset-upload.yml | 7 +++
.../org/apache/solr/cli/ConfigSetUploadTool.java | 9 +++-
.../src/java/org/apache/solr/cli/CreateTool.java | 34 +++++++------
.../solr/handler/configsets/DownloadConfigSet.java | 46 +-----------------
.../test/org/apache/solr/cli/CreateToolTest.java | 55 +++++++++++++++++++++
.../java/org/apache/solr/common/util/Utils.java | 56 ++++++++++++++++++++++
.../org/apache/solr/cloud/RemoteSolrBackend.java | 23 +--------
7 files changed, 150 insertions(+), 80 deletions(-)
diff --git a/changelog/unreleased/SOLR-18320-skip-zk-for-configset-upload.yml
b/changelog/unreleased/SOLR-18320-skip-zk-for-configset-upload.yml
new file mode 100644
index 00000000000..fa9d69fa2f7
--- /dev/null
+++ b/changelog/unreleased/SOLR-18320-skip-zk-for-configset-upload.yml
@@ -0,0 +1,7 @@
+title: bin/solr create no longer requires a direct ZooKeeper connection to
upload a configset; it now uses the Configsets V2 API.
+type: changed
+authors:
+ - name: Eric Pugh
+links:
+ - name: SOLR-18320
+ url: https://issues.apache.org/jira/browse/SOLR-18320
diff --git a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
index dae7174605a..20cf3b5820e 100644
--- a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
@@ -31,7 +31,14 @@ import org.apache.solr.util.FileTypeMagicUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/** Supports zk upconfig command in the bin/solr script. */
+/**
+ * Supports zk upconfig command in the bin/solr script.
+ *
+ * <p>Unlike {@link CreateTool}, this uploads directly to ZooKeeper rather
than going through the
+ * Configsets V2 API. That's intentional: this command is meant to work
against a bare ZooKeeper
+ * ensemble with no Solr node running yet (e.g. provisioning configs before
the cluster starts), and
+ * the V2 API requires an HTTP endpoint on an already-running node to PUT the
zip to.
+ */
@SuppressWarnings("UnnecessarilyFullyQualified")
@picocli.CommandLine.Command(
name = "upconfig",
diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java
b/solr/core/src/java/org/apache/solr/cli/CreateTool.java
index 58149b97475..9e58a157ab2 100644
--- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.cli;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -34,12 +35,13 @@ import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.jetty.HttpJettySolrClient;
import org.apache.solr.client.solrj.request.CollectionsApi;
+import org.apache.solr.client.solrj.request.ConfigsetsApi;
import org.apache.solr.client.solrj.request.CoresApi;
import org.apache.solr.client.solrj.request.SystemInfoRequest;
import org.apache.solr.client.solrj.response.SystemInfoResponse;
-import org.apache.solr.cloud.ZkConfigSetService;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.EnvUtils;
+import org.apache.solr.common.util.Utils;
import org.apache.solr.core.ConfigSetService;
/** Supports create command in the bin/solr script. */
@@ -317,24 +319,22 @@ public class CreateTool extends ToolBase {
int replicationFactor = params.replicationFactor;
String confName = params.confName;
- boolean configExistsInZk =
+ boolean configExists =
confName != null
&& !confName.trim().isEmpty()
- &&
ZkStateReader.from(cloudSolrClient).getZkClient().exists("/configs/" +
confName);
+ && new ConfigsetsApi.ListConfigSet()
+ .process(cloudSolrClient)
+ .configSets
+ .contains(confName);
- if (configExistsInZk) {
+ if (configExists) {
echo("Re-using existing configuration directory " + confName);
- } else { // if (confdir != null && !confdir.trim().isEmpty()) {
+ } else {
if (confName == null || confName.trim().isEmpty()) {
confName = params.name;
}
- // TODO: This should be done using the configSet API. This would let us
remove
- // the direct dependency on ZooKeeper APIs. Unlike the bin/solr zk
comamnds that
- // work directly with ZooKeeper.
final Path configsetsDirPath =
CLIUtils.getConfigSetsDir(solrInstallDirPath);
- ConfigSetService configSetService =
- new
ZkConfigSetService(ZkStateReader.from(cloudSolrClient).getZkClient());
Path confPath =
ConfigSetService.getConfigsetPath(params.confDir,
configsetsDirPath.toString());
@@ -343,10 +343,11 @@ public class CreateTool extends ToolBase {
+ confPath.toAbsolutePath()
+ " for config "
+ confName
- + " to ZooKeeper at "
- + cloudSolrClient.getClusterStateProvider().getQuorumHosts());
- // We will trust the config since we have the Zookeeper Address
- configSetService.uploadConfig(confName, confPath);
+ + " using the Configsets V2 API");
+ var uploadReq =
+ new ConfigsetsApi.UploadConfigSet(
+ confName, new ByteArrayInputStream(zipConfigSet(confPath)));
+ uploadReq.process(cloudSolrClient);
}
// since creating a collection is a heavy-weight operation, check for
existence first
@@ -387,6 +388,11 @@ public class CreateTool extends ToolBase {
echo(endMessage);
}
+ /** Zips the contents of a configset directory for upload. */
+ static byte[] zipConfigSet(Path confPath) throws IOException {
+ return Utils.zipDirectory(confPath);
+ }
+
private Path getFullConfDir(Path solrInstallDir, Path confDirName) {
return CLIUtils.getConfigSetsDir(solrInstallDir).resolve(confDirName);
}
diff --git
a/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java
b/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java
index 729aaf00d91..76d12758db0 100644
---
a/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java
+++
b/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java
@@ -21,20 +21,14 @@ import static
org.apache.solr.security.PermissionNameProvider.Name.CONFIG_READ_P
import jakarta.inject.Inject;
import jakarta.ws.rs.core.Response;
import jakarta.ws.rs.core.StreamingOutput;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
import org.apache.commons.io.file.PathUtils;
import org.apache.solr.client.api.endpoint.ConfigsetsApi;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
import org.apache.solr.core.ConfigSetService;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.jersey.PermissionName;
@@ -86,48 +80,12 @@ public class DownloadConfigSet extends ConfigSetAPIBase
implements ConfigsetsApi
*/
public static byte[] zipConfigSet(ConfigSetService configSetService, String
configSetName)
throws IOException {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
Path tmpDirectory = Files.createTempDirectory("configset-download-");
try {
configSetService.downloadConfig(configSetName, tmpDirectory);
- try (ZipOutputStream zipOut = new ZipOutputStream(baos)) {
- Files.walkFileTree(
- tmpDirectory,
- new SimpleFileVisitor<>() {
- @Override
- public FileVisitResult preVisitDirectory(Path dir,
BasicFileAttributes attrs)
- throws IOException {
- if (Files.isHidden(dir)) {
- return FileVisitResult.SKIP_SUBTREE;
- }
- String dirName = tmpDirectory.relativize(dir).toString();
- if (!dirName.isEmpty()) {
- if (!dirName.endsWith("/")) {
- dirName += "/";
- }
- zipOut.putNextEntry(new ZipEntry(dirName));
- zipOut.closeEntry();
- }
- return FileVisitResult.CONTINUE;
- }
-
- @Override
- public FileVisitResult visitFile(Path file, BasicFileAttributes
attrs)
- throws IOException {
- if (!Files.isHidden(file)) {
- try (InputStream fis = Files.newInputStream(file)) {
- ZipEntry zipEntry = new
ZipEntry(tmpDirectory.relativize(file).toString());
- zipOut.putNextEntry(zipEntry);
- fis.transferTo(zipOut);
- }
- }
- return FileVisitResult.CONTINUE;
- }
- });
- }
+ return Utils.zipDirectory(tmpDirectory);
} finally {
PathUtils.deleteDirectory(tmpDirectory);
}
- return baos.toByteArray();
}
}
diff --git a/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java
b/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java
index e1df239b51b..a814ba529de 100644
--- a/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java
+++ b/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java
@@ -17,6 +17,13 @@
package org.apache.solr.cli;
+import java.io.ByteArrayInputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.util.SecurityJson;
import org.junit.BeforeClass;
@@ -57,4 +64,52 @@ public class CreateToolTest extends SolrCloudTestCase {
assertEquals(0, runTool(args, CreateTool.class));
}
+
+ @Test
+ public void testCreateCollectionUploadsNewConfigSet() throws Exception {
+ String[] args = {
+ "create",
+ "-c",
+ "testCreateCollectionUploadsNewConfigSet",
+ "-d",
+ configset("cloud-minimal").toString(),
+ "-n",
+ "cloud-minimal-uploaded",
+ "-z",
+ cluster.getZkClient().getZkServerAddress(),
+ "--credentials",
+ SecurityJson.USER_PASS,
+ "--verbose"
+ };
+
+ assertEquals(0, CLITestHelper.runTool(args, CreateTool.class));
+
assertTrue(cluster.getZkClient().exists("/configs/cloud-minimal-uploaded"));
+ }
+
+ @Test
+ public void testZipConfigSetSkipsHiddenFilesAndIncludesDirectoryEntries()
throws Exception {
+ Path confDir = createTempDir("zipConfigSetTest");
+ Files.writeString(confDir.resolve("solrconfig.xml"), "<config/>");
+ Files.writeString(confDir.resolve(".hidden-file"), "should not be zipped");
+ Path langDir = Files.createDirectory(confDir.resolve("lang"));
+ Files.writeString(langDir.resolve("stopwords.txt"), "the\na\n");
+ Path hiddenDir = Files.createDirectory(confDir.resolve(".hiddenDir"));
+ Files.writeString(hiddenDir.resolve("nope.txt"), "should not be zipped
either");
+
+ byte[] zipBytes = CreateTool.zipConfigSet(confDir);
+
+ Set<String> entryNames = new HashSet<>();
+ try (ZipInputStream zipIn = new ZipInputStream(new
ByteArrayInputStream(zipBytes))) {
+ ZipEntry entry;
+ while ((entry = zipIn.getNextEntry()) != null) {
+ entryNames.add(entry.getName());
+ }
+ }
+
+ assertTrue(entryNames.contains("solrconfig.xml"));
+ assertTrue(entryNames.contains("lang/"));
+ assertTrue(entryNames.contains("lang/stopwords.txt"));
+ assertFalse(entryNames.contains(".hidden-file"));
+ assertTrue(entryNames.stream().noneMatch(name ->
name.startsWith(".hiddenDir")));
+ }
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
index 71cdbb59c23..810cf086132 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
@@ -43,6 +43,11 @@ import java.net.URL;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.AbstractMap;
@@ -69,6 +74,8 @@ import java.util.function.Function;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.LinkedHashMapWriter;
import org.apache.solr.common.MapWriter;
@@ -1045,4 +1052,53 @@ public class Utils {
return bos.getByteBuffer();
}
}
+
+ /**
+ * Zips the contents of {@code rootPath} into an in-memory archive. Hidden
files and directories
+ * (as determined by {@link Files#isHidden}) are skipped, directory entries
are written for
+ * non-empty subdirectories, and zip entry names are normalized to use
{@code /} separators
+ * regardless of platform.
+ *
+ * @param rootPath the directory to zip
+ * @return the zipped bytes
+ */
+ public static byte[] zipDirectory(Path rootPath) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (ZipOutputStream zipOut = new ZipOutputStream(baos)) {
+ Files.walkFileTree(
+ rootPath,
+ new SimpleFileVisitor<>() {
+ @Override
+ public FileVisitResult preVisitDirectory(Path dir,
BasicFileAttributes attrs)
+ throws IOException {
+ if (Files.isHidden(dir)) {
+ return FileVisitResult.SKIP_SUBTREE;
+ }
+ String dirName =
rootPath.relativize(dir).toString().replace('\\', '/');
+ if (!dirName.isEmpty()) {
+ if (!dirName.endsWith("/")) {
+ dirName += "/";
+ }
+ zipOut.putNextEntry(new ZipEntry(dirName));
+ zipOut.closeEntry();
+ }
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes
attrs)
+ throws IOException {
+ if (Files.isHidden(file)) {
+ return FileVisitResult.CONTINUE;
+ }
+ String entryName =
rootPath.relativize(file).toString().replace('\\', '/');
+ zipOut.putNextEntry(new ZipEntry(entryName));
+ Files.copy(file, zipOut);
+ zipOut.closeEntry();
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ }
+ return baos.toByteArray();
+ }
}
diff --git
a/solr/test-framework/src/java/org/apache/solr/cloud/RemoteSolrBackend.java
b/solr/test-framework/src/java/org/apache/solr/cloud/RemoteSolrBackend.java
index 78cb06ecee0..3ef526a0176 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/RemoteSolrBackend.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/RemoteSolrBackend.java
@@ -20,8 +20,6 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
import org.apache.solr.SolrBackend;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
@@ -30,6 +28,7 @@ import
org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.ConfigSetAdminRequest;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.URLUtil;
+import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
/**
@@ -68,7 +67,7 @@ public class RemoteSolrBackend implements SolrBackend {
public void uploadConfigSet(Path configDir, String name) throws
SolrServerException, IOException {
Path tempZip = Files.createTempFile("configset-", ".zip");
try {
- zipDirectory(configDir, tempZip);
+ Files.write(tempZip, Utils.zipDirectory(configDir));
new ConfigSetAdminRequest.Upload()
.setConfigSetName(name)
.setUploadFile(tempZip, "application/zip")
@@ -83,24 +82,6 @@ public class RemoteSolrBackend implements SolrBackend {
return new
ConfigSetAdminRequest.List().process(adminClient).getConfigSets().contains(name);
}
- /** Packages {@code sourceDir} contents into a zip file. */
- private static void zipDirectory(Path sourceDir, Path targetZip) throws
IOException {
- try (ZipOutputStream zos = new
ZipOutputStream(Files.newOutputStream(targetZip))) {
- Files.walk(sourceDir)
- .filter(Files::isRegularFile)
- .forEach(
- file -> {
- try {
- zos.putNextEntry(new
ZipEntry(sourceDir.relativize(file).toString()));
- Files.copy(file, zos);
- zos.closeEntry();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- });
- }
- }
-
@Override
public void createCollection(CollectionAdminRequest.Create create)
throws SolrServerException, IOException {
