This is an automated email from the ASF dual-hosted git repository.
hossman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 58527ba SOLR-15708: SolrJ support for ConfigSet uploading
58527ba is described below
commit 58527ba5fd45ab8b2238f23850d46c5e58497446
Author: Chris Hostetter <[email protected]>
AuthorDate: Thu Oct 21 16:02:05 2021 -0700
SOLR-15708: SolrJ support for ConfigSet uploading
---
solr/CHANGES.txt | 2 +
.../org/apache/solr/cloud/TestConfigSetsAPI.java | 154 +++++++++++++--------
.../solrj/request/ConfigSetAdminRequest.java | 143 ++++++++++++++++++-
.../solrj/request/TestConfigSetAdminRequest.java | 33 +++++
4 files changed, 273 insertions(+), 59 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index df4a92d..1ae7271 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -401,6 +401,8 @@ New Features
* SOLR-7642: opt-in support to create ZK chroot on startup (Timothy Potter,
Shawn Heisey, Mark Miller, Tomas Eduardo Fernandez Lobbe,
Jan Høydahl, Steve Molloy, Isabelle Giguere, David Eric Pugh, Gus Heck,
Christine Poerschke, Houston Putman)
+* SOLR-15708: SolrJ support for ConfigSet uploading (ab, hossman)
+
Improvements
---------------------
(No changes)
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
index 7691c6b..aa4f2bd 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
@@ -59,7 +59,6 @@ import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.message.BasicHeader;
import org.apache.http.util.EntityUtils;
-import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
@@ -68,6 +67,7 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.ConfigSetAdminRequest;
import org.apache.solr.client.solrj.request.ConfigSetAdminRequest.Create;
import org.apache.solr.client.solrj.request.ConfigSetAdminRequest.Delete;
+import org.apache.solr.client.solrj.request.ConfigSetAdminRequest.Upload;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.ConfigSetAdminResponse;
@@ -81,10 +81,11 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.common.util.ValidatingJsonMap;
import org.apache.solr.core.ConfigSetProperties;
import org.apache.solr.core.ConfigSetService;
import org.apache.solr.core.TestSolrConfigHandler;
-import org.apache.solr.security.AuthenticationPlugin;
+import org.apache.solr.security.BasicAuthPlugin;
import org.apache.solr.security.AuthorizationContext;
import org.apache.solr.security.AuthorizationPlugin;
import org.apache.solr.security.AuthorizationResponse;
@@ -877,6 +878,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
private static String getSecurityJson() throws KeeperException,
InterruptedException {
String securityJson = "{\n" +
" 'authentication':{\n" +
+ " 'blockUnknown': false,\n" +
" 'class':'" + MockAuthenticationPlugin.class.getName() +
"'},\n" +
" 'authorization':{\n" +
" 'class':'" + MockAuthorizationPlugin.class.getName() + "'}}";
@@ -917,82 +919,110 @@ public class TestConfigSetsAPI extends SolrCloudTestCase
{
}
private long uploadConfigSet(String configSetName, String suffix, String
username,
- boolean overwrite, boolean cleanup, boolean v2) throws IOException {
+ boolean overwrite, boolean cleanup, boolean v2)
throws IOException {
+
// Read zipped sample config
- ByteBuffer sampleZippedConfig = TestSolrConfigHandler
- .getFileContent(
- createTempZipFile("solr/configsets/upload/"+configSetName), false);
-
- return uploadGivenConfigSet(sampleZippedConfig, configSetName, suffix,
username, overwrite, cleanup, v2);
+ return
uploadGivenConfigSet(createTempZipFile("solr/configsets/upload/"+configSetName),
+ configSetName, suffix, username, overwrite,
cleanup, v2);
}
private long uploadBadConfigSet(String configSetName, String suffix, String
username,
- boolean overwrite, boolean cleanup, boolean v2)
throws IOException {
+ boolean overwrite, boolean cleanup, boolean
v2) throws IOException {
+
// Read single file from sample configs. This should fail the unzipping
- ByteBuffer sampleBadZippedFile =
TestSolrConfigHandler.getFileContent(SolrTestCaseJ4.getFile("solr/configsets/upload/regular/solrconfig.xml").getAbsolutePath(),
false);
-
- return uploadGivenConfigSet(sampleBadZippedFile, configSetName, suffix,
username, overwrite, cleanup, v2);
+ return
uploadGivenConfigSet(SolrTestCaseJ4.getFile("solr/configsets/upload/regular/solrconfig.xml"),
+ configSetName, suffix, username, overwrite,
cleanup, v2);
}
- private long uploadGivenConfigSet(ByteBuffer file, String configSetName,
String suffix, String username,
- boolean overwrite, boolean cleanup, boolean v2)
throws IOException {
- String uriEnding;
- boolean usePut = false;
+ private long uploadGivenConfigSet(File file, String configSetName, String
suffix, String username,
+ boolean overwrite, boolean cleanup,
boolean v2) throws IOException {
+
if (v2) {
- uriEnding = "/api/cluster/configs/" + configSetName+suffix +
(!overwrite? "?overwrite=false" : "") + (cleanup? "?cleanup=true" : "");
- usePut = true;
- } else {
- uriEnding =
"/solr/admin/configs?action=UPLOAD&name="+configSetName+suffix + (overwrite?
"&overwrite=true" : "") + (cleanup? "&cleanup=true" : "");
+ // TODO: switch to using V2Request
+
+ final ByteBuffer fileBytes =
TestSolrConfigHandler.getFileContent(file.getAbsolutePath(), false);
+ final String uriEnding = "/api/cluster/configs/" + configSetName+suffix
+ (!overwrite? "?overwrite=false" : "") + (cleanup? "?cleanup=true" : "");
+ final boolean usePut = true;
+ Map<?, ?> map = postDataAndGetResponse(cluster.getSolrClient(),
+
cluster.getJettySolrRunners().get(0).getBaseUrl().toString().replace("/solr",
"") + uriEnding,
+ fileBytes, username, usePut);
+ assertNotNull(map);
+ long statusCode = (long) getObjectByPath(map, false,
Arrays.asList("responseHeader", "status"));
+ return statusCode;
+
+ } // else "not" a V2 request...
+
+ try {
+ return (new Upload())
+ .setConfigSetName(configSetName + suffix)
+ .setUploadFile(file, "application/zip")
+ .setOverwrite(overwrite ? true : null) // expect server default to be
'false'
+ .setCleanup(cleanup ? true : null) // expect server default to be
'false'
+ .setBasicAuthCredentials(username, username) // for our
MockAuthenticationPlugin
+ .process(cluster.getSolrClient())
+ .getStatus();
+ } catch (SolrServerException e1) {
+ throw new AssertionError("Server error uploading configset: " +
e1.toString(), e1);
+ } catch (SolrException e2) {
+ return e2.code();
}
-
- Map<?, ?> map = postDataAndGetResponse(cluster.getSolrClient(),
-
cluster.getJettySolrRunners().get(0).getBaseUrl().toString().replace("/solr",
"") + uriEnding,
- file, username, usePut);
- assertNotNull(map);
- long statusCode = (long) getObjectByPath(map, false,
Arrays.asList("responseHeader", "status"));
- return statusCode;
}
private long uploadSingleConfigSetFile(String configSetName, String suffix,
String username,
- String filePath, String uploadPath,
boolean overwrite, boolean cleanup, boolean v2) throws IOException {
+ String localFilePath, String
uploadPath, boolean overwrite, boolean cleanup, boolean v2) throws IOException {
// Read single file from sample configs
- ByteBuffer sampleConfigFile =
TestSolrConfigHandler.getFileContent(SolrTestCaseJ4.getFile(filePath).getAbsolutePath(),
false);
+ final File file = SolrTestCaseJ4.getFile(localFilePath);
- String uriEnding;
- boolean usePut = false;
if (v2) {
- uriEnding = "/api/cluster/configs/" + configSetName+suffix + "/" +
uploadPath + (!overwrite? "?overwrite=false" : "") + (cleanup? "?cleanup=true"
: "");
- usePut = true;
- } else {
- uriEnding =
"/solr/admin/configs?action=UPLOAD&name="+configSetName+suffix+"&filePath="+uploadPath
+ (overwrite? "&overwrite=true" : "") + (cleanup? "&cleanup=true" : "");
+ // TODO: switch to use V2Request
+
+ final ByteBuffer sampleConfigFile =
TestSolrConfigHandler.getFileContent(file.getAbsolutePath(), false);
+ final String uriEnding = "/api/cluster/configs/" + configSetName+suffix
+ "/" + uploadPath + (!overwrite? "?overwrite=false" : "") + (cleanup?
"?cleanup=true" : "");
+ final boolean usePut = true;
+
+ Map<?, ?> map = postDataAndGetResponse(cluster.getSolrClient(),
+
cluster.getJettySolrRunners().get(0).getBaseUrl().toString().replace("/solr",
"") + uriEnding,
+ sampleConfigFile, username,
usePut);
+ assertNotNull(map);
+ long statusCode = (long) getObjectByPath(map, false,
Arrays.asList("responseHeader", "status"));
+ return statusCode;
+
+ } // else "not" a V2 request...
+
+ try {
+ return (new Upload())
+ .setConfigSetName(configSetName + suffix)
+ .setFilePath(uploadPath)
+ .setUploadFile(file, "application/octet-stream") // NOTE: server
doesn't actually care, and test plumbing doesn't tell us
+ .setOverwrite(overwrite ? true : null) // expect server default to be
'false'
+ .setCleanup(cleanup ? true : null) // expect server default to be
'false'
+ .setBasicAuthCredentials(username, username) // for our
MockAuthenticationPlugin
+ .process(cluster.getSolrClient())
+ .getStatus();
+ } catch (SolrServerException e1) {
+ throw new AssertionError("Server error uploading file to configset: " +
e1.toString(), e1);
+ } catch (SolrException e2) {
+ return e2.code();
}
-
- Map<?, ?> map = postDataAndGetResponse(cluster.getSolrClient(),
-
cluster.getJettySolrRunners().get(0).getBaseUrl().toString().replace("/solr",
"") + uriEnding,
- sampleConfigFile, username, usePut);
- assertNotNull(map);
- long statusCode = (long) getObjectByPath(map, false,
Arrays.asList("responseHeader", "status"));
- return statusCode;
}
+
/**
* Create a zip file (in the temp directory) containing all the files within
the specified directory
- * and return the path for the zip file.
+ * and return the zip file.
*/
- private String createTempZipFile(String directoryPath) {
- File zipFile = new File(cluster.getBaseDir().toFile().getAbsolutePath() +
- File.separator + TestUtil.randomSimpleString(random(), 6, 8) + ".zip");
-
- File directory = SolrTestCaseJ4.getFile(directoryPath);
- if (log.isInfoEnabled()) {
- log.info("Directory: {}", directory.getAbsolutePath());
- }
+ private File createTempZipFile(String directoryPath) {
try {
+ final File zipFile = createTempFile("configset","zip").toFile();
+ final File directory = SolrTestCaseJ4.getFile(directoryPath);
+ if (log.isInfoEnabled()) {
+ log.info("Directory: {}", directory.getAbsolutePath());
+ }
zip (directory, zipFile);
if (log.isInfoEnabled()) {
log.info("Zipfile: {}", zipFile.getAbsolutePath());
}
- return zipFile.getAbsolutePath();
+ return zipFile;
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -1284,11 +1314,20 @@ public class TestConfigSetsAPI extends
SolrCloudTestCase {
}
}
- public static class MockAuthenticationPlugin extends AuthenticationPlugin {
+ public static class MockAuthenticationPlugin extends BasicAuthPlugin {
@Override
- public void init(Map<String, Object> pluginConfig) {
-
+ public AuthenticationProvider getAuthenticationProvider(Map<String,
Object> pluginConfig) {
+ return new AuthenticationProvider() {
+ @Override public void init(Map<String,Object> ignored) { }
+ @Override public ValidatingJsonMap getSpec() { return
Utils.getSpec("cluster.security.BasicAuth.Commands").getSpec(); }
+ @Override public boolean authenticate(String user, String pwd) {
+ return user.equals(pwd);
+ }
+ @Override public Map<String, String> getPromptHeaders() {
+ return Collections.emptyMap();
+ }
+ };
}
@Override
@@ -1296,10 +1335,9 @@ public class TestConfigSetsAPI extends SolrCloudTestCase
{
if (request.getHeader("user") != null) {
final Principal p = new BasicUserPrincipal("solr");
filterChain.doFilter(wrap(request, p, "solr"), response);
- } else {
- filterChain.doFilter(request, response);
+ return true;
}
- return true;
+ return super.doAuthenticate(request, response, filterChain);
}
HttpServletRequest wrap(HttpServletRequest request, Principal principal,
String username) {
diff --git
a/solr/solrj/src/java/org/apache/solr/client/solrj/request/ConfigSetAdminRequest.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/request/ConfigSetAdminRequest.java
index 233f99b..6614fc6 100644
---
a/solr/solrj/src/java/org/apache/solr/client/solrj/request/ConfigSetAdminRequest.java
+++
b/solr/solrj/src/java/org/apache/solr/client/solrj/request/ConfigSetAdminRequest.java
@@ -16,8 +16,14 @@
*/
package org.apache.solr.client.solrj.request;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+
import java.util.Map;
import java.util.Properties;
+import java.util.Collection;
+import java.util.Collections;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
@@ -27,8 +33,13 @@ import
org.apache.solr.common.params.ConfigSetParams.ConfigSetAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ContentStreamBase.FileStream;
+
import static org.apache.solr.common.params.CommonParams.NAME;
+import org.apache.commons.io.IOUtils;
+
/**
* This class is experimental and subject to change.
*
@@ -104,7 +115,137 @@ public abstract class ConfigSetAdminRequest
return SolrRequestType.ADMIN.toString();
}
- // CREATE request
+ /**
+ * Uploads files to create a new configset, or modify an existing config set.
+ *
+ * When creating a new configset, the file to be uploaded must be a ZIP file
containing the entire configset being uploaded.
+ * When modifing an existing configset, the file to be uploaded should
either be a ZIP file containing the entire configset
+ * being uploaded, or an individual file to upload if {@link #setFilePath}
is being used.
+ */
+ public static class Upload extends ConfigSetSpecificAdminRequest<Upload> {
+ private static final String NO_STREAM_ERROR = "There must be a
ContentStream or File to Upload";
+
+ protected ContentStream stream;
+ protected String filePath;
+
+ protected Boolean overwrite;
+ protected Boolean cleanup;
+
+ public Upload() {
+ action = ConfigSetAction.UPLOAD;
+ setMethod(SolrRequest.METHOD.POST);
+ }
+
+ @Override
+ protected Upload getThis() {
+ return this;
+ }
+
+ /** Optional {@link ConfigSetParams#FILE_PATH} to indicate a single file
is being uploaded into an existing configset */
+ public final Upload setFilePath(final String filePath) {
+ this.filePath = filePath;
+ return getThis();
+ }
+
+ /** @see #setFilePath */
+ public final String getFilePath() {
+ return filePath;
+ }
+
+ /**
+ * A convinience method for specifying an existing File to use as the
upload data.
+ *
+ * This should either be a ZIP file containing the entire configset being
uploaded, or
+ * an individual file to upload into an existing configset if {@link
#setFilePath} is being used.
+ *
+ * @see #setUploadStream
+ */
+ public final Upload setUploadFile(final File file, final String
contentType) {
+ final FileStream fileStream = new FileStream(file);
+ fileStream.setContentType(contentType);
+ return setUploadStream(fileStream);
+ }
+
+ /** @see ConfigSetParams#OVERWRITE */
+ public final Upload setOverwrite(final Boolean overwrite) {
+ this.overwrite = overwrite;
+ return getThis();
+ }
+
+ /** @see #setOverwrite */
+ public final Boolean getOverwrite() {
+ return overwrite;
+ }
+
+ /** @see ConfigSetParams#CLEANUP */
+ public final Upload setCleanup(final Boolean cleanup) {
+ this.cleanup = cleanup;
+ return getThis();
+ }
+
+ /** @see #setCleanup */
+ public final Boolean getCleanup() {
+ return cleanup;
+ }
+
+ /**
+ * Specify the ContentStream to upload.
+ *
+ * This should either be a ZIP file containing the entire configset being
uploaded, or
+ * an individual file to upload into an existing configset if {@link
#setFilePath} is being used.
+ *
+ * @see #setUploadStream
+ */
+ public final Upload setUploadStream(final ContentStream stream) {
+ this.stream = stream;
+ return getThis();
+ }
+
+ @Override
+ public Collection<ContentStream> getContentStreams() throws IOException {
+ return Collections.singletonList(stream);
+ }
+
+ @Override
+ public RequestWriter.ContentWriter getContentWriter(String expectedType) {
+ if (null == stream) {
+ throw new NullPointerException(NO_STREAM_ERROR);
+ }
+ return new RequestWriter.ContentWriter() {
+ @Override
+ public void write(OutputStream os) throws IOException {
+ try(var inStream = stream.getStream()) {
+ IOUtils.copy(inStream, os);
+ }
+ }
+
+ @Override
+ public String getContentType() {
+ return stream.getContentType();
+ }
+ };
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = new
ModifiableSolrParams(super.getParams());
+
+ if (null == stream) {
+ throw new NullPointerException(NO_STREAM_ERROR);
+ }
+
+ params.setNonNull(ConfigSetParams.FILE_PATH, filePath);
+ params.setNonNull(ConfigSetParams.CLEANUP, cleanup);
+ params.setNonNull(ConfigSetParams.OVERWRITE, overwrite);
+
+ return params;
+ }
+ }
+
+ /**
+ * Creates a new config set by cloning an existing "base" configset.
+ * To create a new configset from scratch using a ZIP file you wish to
upload, use the {@link Upload} command instead
+ */
public static class Create extends ConfigSetSpecificAdminRequest<Create> {
protected static String PROPERTY_PREFIX = "configSetProp";
protected String baseConfigSetName;
diff --git
a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestConfigSetAdminRequest.java
b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestConfigSetAdminRequest.java
index f1da3fe..252805c 100644
---
a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestConfigSetAdminRequest.java
+++
b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestConfigSetAdminRequest.java
@@ -16,9 +16,13 @@
*/
package org.apache.solr.client.solrj.request;
+import java.io.File;
+
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.response.ConfigSetAdminResponse;
+import org.apache.solr.common.params.ConfigSetParams;
+
import org.junit.Test;
/**
@@ -33,6 +37,35 @@ public class TestConfigSetAdminRequest extends
SolrTestCaseJ4 {
}
@Test
+ public void testUpload() throws Exception {
+ final File tmpFile = createTempFile().toFile();
+ ConfigSetAdminRequest.Upload upload = new ConfigSetAdminRequest.Upload();
+ verifyException(upload, "ConfigSet");
+
+ upload.setConfigSetName("name");
+ verifyException(upload, "There must be a ContentStream");
+
+ upload.setUploadFile(tmpFile, "application/zip");
+
+ assertEquals(1, upload.getContentStreams().size());
+ assertEquals("application/zip",
upload.getContentStreams().stream().findFirst().get().getContentType());
+
+ assertNull(upload.getParams().get(ConfigSetParams.FILE_PATH));
+ assertNull(upload.getParams().get(ConfigSetParams.OVERWRITE));
+ assertNull(upload.getParams().get(ConfigSetParams.CLEANUP));
+
+ upload.setUploadFile(tmpFile, "application/xml")
+ .setFilePath("solrconfig.xml")
+ .setOverwrite(true);
+
+ assertEquals(1, upload.getContentStreams().size());
+ assertEquals("application/xml",
upload.getContentStreams().stream().findFirst().get().getContentType());
+
+ assertEquals("solrconfig.xml",
upload.getParams().get(ConfigSetParams.FILE_PATH));
+ assertEquals("true", upload.getParams().get(ConfigSetParams.OVERWRITE));
+ }
+
+ @Test
public void testCreate() {
ConfigSetAdminRequest.Create create = new ConfigSetAdminRequest.Create();
verifyException(create, "ConfigSet");
