This is an automated email from the ASF dual-hosted git repository.
houston pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 7d20587344b SOLR-16480: Add overridable allow-list for ConfigSet file
types
7d20587344b is described below
commit 7d20587344b6184cee1b0e646c9e261b8ee11084
Author: Houston Putman <[email protected]>
AuthorDate: Mon Jan 9 16:59:23 2023 -0500
SOLR-16480: Add overridable allow-list for ConfigSet file types
(cherry picked from commit 28d6b0163316376ef3b5429b3554c5041b47b5be)
---
solr/CHANGES.txt | 2 +
.../org/apache/solr/cloud/ZkConfigSetService.java | 24 ++-
.../solr/core/FileSystemConfigSetService.java | 21 ++-
.../org/apache/solr/core/backup/BackupManager.java | 31 ++--
.../backup/repository/BackupRepositoryFactory.java | 1 -
.../handler/configsets/UploadConfigSetFileAPI.java | 7 +
.../org/apache/solr/cloud/TestConfigSetsAPI.java | 162 +++++++++++++++++++--
.../configuration-guide/pages/config-sets.adoc | 19 +++
.../configuration-guide/pages/configsets-api.adoc | 2 +
.../pages/major-changes-in-solr-9.adoc | 2 +
.../solr/common/cloud/ZkMaintenanceUtils.java | 44 +++++-
11 files changed, 277 insertions(+), 38 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 10663a00ca4..1b71262252e 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -202,6 +202,8 @@ Bug Fixes
* SOLR-16588: Fixed problem with default knn algorithm (Elia Porciani via
Alessandro Benedetti)
+* SOLR-16480: ConfigSets now have an overridable allow-list for filetypes.
(Houston Putman)
+
================== 9.1.0 ==================
New Features
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
index b67d4e7cc50..88f6a7f9f91 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
@@ -199,8 +199,12 @@ public class ZkConfigSetService extends ConfigSetService {
throws IOException {
String filePath = CONFIGS_ZKNODE + "/" + configName + "/" + fileName;
try {
- // if overwriteOnExists is true then zkClient#makePath failOnExists is
set to false
- zkClient.makePath(filePath, data, CreateMode.PERSISTENT, null,
!overwriteOnExists, true);
+ if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
+ log.warn("Not including uploading file to config, as it is a forbidden
type: {}", fileName);
+ } else {
+ // if overwriteOnExists is true then zkClient#makePath failOnExists is
set to false
+ zkClient.makePath(filePath, data, CreateMode.PERSISTENT, null,
!overwriteOnExists, true);
+ }
} catch (KeeperException.NodeExistsException nodeExistsException) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
@@ -327,9 +331,19 @@ public class ZkConfigSetService extends ConfigSetService {
private void copyData(String fromZkFilePath, String toZkFilePath)
throws KeeperException, InterruptedException {
- log.debug("Copying zk node {} to {}", fromZkFilePath, toZkFilePath);
- byte[] data = zkClient.getData(fromZkFilePath, null, null, true);
- zkClient.makePath(toZkFilePath, data, true);
+ if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fromZkFilePath)) {
+ log.warn(
+ "Skipping copy of file in ZK, as the source file is a forbidden
type: {}",
+ fromZkFilePath);
+ } else if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(toZkFilePath)) {
+ log.warn(
+ "Skipping download of file from ZK, as the target file is a
forbidden type: {}",
+ toZkFilePath);
+ } else {
+ log.debug("Copying zk node {} to {}", fromZkFilePath, toZkFilePath);
+ byte[] data = zkClient.getData(fromZkFilePath, null, null, true);
+ zkClient.makePath(toZkFilePath, data, true);
+ }
}
public SolrCloudManager getSolrCloudManager() {
diff --git
a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
index 75d8e288834..5ada2f99cfb 100644
--- a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
+++ b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
@@ -35,6 +35,7 @@ import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.common.util.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -146,9 +147,13 @@ public class FileSystemConfigSetService extends
ConfigSetService {
public void uploadFileToConfig(
String configName, String fileName, byte[] data, boolean
overwriteOnExists)
throws IOException {
- Path filePath =
getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
- if (!Files.exists(filePath) || overwriteOnExists) {
- Files.write(filePath, data);
+ if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
+ log.warn("Not including uploading file to config, as it is a forbidden
type: {}", fileName);
+ } else {
+ Path filePath =
getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
+ if (!Files.exists(filePath) || overwriteOnExists) {
+ Files.write(filePath, data);
+ }
}
}
@@ -195,8 +200,14 @@ public class FileSystemConfigSetService extends
ConfigSetService {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes
attrs)
throws IOException {
- Files.copy(
- file, target.resolve(source.relativize(file).toString()),
REPLACE_EXISTING);
+ if
(ZkMaintenanceUtils.isFileForbiddenInConfigSets(file.getFileName().toString()))
{
+ log.warn(
+ "Not including uploading file to config, as it is a
forbidden type: {}",
+ file.getFileName());
+ } else {
+ Files.copy(
+ file, target.resolve(source.relativize(file).toString()),
REPLACE_EXISTING);
+ }
return FileVisitResult.CONTINUE;
}
});
diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
index 0d6cd77f3df..cd50bb5deef 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
@@ -36,6 +36,7 @@ import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.ConfigSetService;
@@ -337,11 +338,16 @@ public class BackupManager {
// checking for '/' is correct for a directory since
ConfigSetService#getAllConfigFiles
// always separates file paths with '/'
if (!filePath.endsWith("/")) {
- log.debug("Writing file {}", filePath);
- // ConfigSetService#downloadFileFromConfig requires '/' in fle path
separator
- byte[] data = configSetService.downloadFileFromConfig(configName,
filePath);
- try (OutputStream os = repository.createOutput(uri)) {
- os.write(data);
+ if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(filePath)) {
+ log.warn(
+ "Not including zookeeper file in backup, as it is a forbidden
type: {}", filePath);
+ } else {
+ log.debug("Writing file {}", filePath);
+ // ConfigSetService#downloadFileFromConfig requires '/' in fle path
separator
+ byte[] data = configSetService.downloadFileFromConfig(configName,
filePath);
+ try (OutputStream os = repository.createOutput(uri)) {
+ os.write(data);
+ }
}
} else {
if (!repository.exists(uri)) {
@@ -361,11 +367,16 @@ public class BackupManager {
switch (t) {
case FILE:
{
- try (IndexInput is = repository.openInput(sourceDir, file,
IOContext.DEFAULT)) {
- // probably ok since the config file should be small.
- byte[] arr = new byte[(int) is.length()];
- is.readBytes(arr, 0, (int) is.length());
- configSetService.uploadFileToConfig(configName, filePath, arr,
false);
+ if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(filePath)) {
+ log.warn(
+ "Not including zookeeper file in restore, as it is a
forbidden type: {}", file);
+ } else {
+ try (IndexInput is = repository.openInput(sourceDir, file,
IOContext.DEFAULT)) {
+ // probably ok since the config file should be small.
+ byte[] arr = new byte[(int) is.length()];
+ is.readBytes(arr, 0, (int) is.length());
+ configSetService.uploadFileToConfig(configName, filePath, arr,
false);
+ }
}
break;
}
diff --git
a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java
b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java
index 6a5ae42f165..17fc9b4bd76 100644
---
a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java
+++
b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java
@@ -95,7 +95,6 @@ public class BackupRepositoryFactory {
if (defaultBackupRepoPlugin != null) {
return newInstance(loader, defaultBackupRepoPlugin.name);
}
-
LocalFileSystemRepository repo = new LocalFileSystemRepository();
repo.init(new NamedList<>());
return repo;
diff --git
a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
index d44f3c5ff31..df877e2b8f8 100644
---
a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
+++
b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
@@ -23,6 +23,7 @@ import java.io.InputStream;
import org.apache.commons.io.IOUtils;
import org.apache.solr.api.EndPoint;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.common.params.ConfigSetParams;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.request.SolrQueryRequest;
@@ -71,6 +72,12 @@ public class UploadConfigSetFileAPI extends ConfigSetAPIBase
{
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"The file path provided for upload, '" + singleFilePath + "', is not
valid.");
+ } else if
(ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath)) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "The file type provided for upload, '"
+ + singleFilePath
+ + "', is forbidden for use in configSets.");
} else if (cleanup) {
// Cleanup is not allowed while using singleFilePath upload
throw new SolrException(
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
index c2f0f23dab8..7e7ff345d4e 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
@@ -83,6 +83,7 @@ import
org.apache.solr.client.solrj.response.schema.SchemaResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.common.params.ConfigSetParams;
import org.apache.solr.common.params.ConfigSetParams.ConfigSetAction;
@@ -579,13 +580,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
assertEquals(
"Can't overwrite an existing configset unless the overwrite
parameter is set",
400,
- uploadConfigSet(configsetName, configsetSuffix, null, false, false,
v2));
+ uploadConfigSet(configsetName, configsetSuffix, null, false, false,
v2, false));
unIgnoreException("The configuration regulartestOverwrite-1 already
exists in zookeeper");
assertEquals(
"Expecting version to remain equal",
solrconfigZkVersion,
getConfigZNodeVersion(zkClient, configsetName, configsetSuffix,
"solrconfig.xml"));
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null,
true, false, v2));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true,
false, v2, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -620,13 +622,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
for (String f : extraFiles) {
zkClient.makePath(f, true);
}
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null,
true, false, v2));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true,
false, v2, false));
for (String f : extraFiles) {
assertTrue(
"Expecting file " + f + " to exist in ConfigSet but it's gone",
zkClient.exists(f, true));
}
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null,
true, true, v2));
+ assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null,
true, true, v2, false));
for (String f : extraFiles) {
assertFalse(
"Expecting file " + f + " to be deleted from ConfigSet but it
wasn't",
@@ -636,6 +639,34 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
}
}
+ @Test
+ public void testOverwriteWithForbiddenFilesV1() throws Exception {
+ testOverwriteWithForbiddenFiles(false);
+ }
+
+ @Test
+ public void testOverwriteWithForbiddenFilesV2() throws Exception {
+ testOverwriteWithForbiddenFiles(true);
+ }
+
+ public void testOverwriteWithForbiddenFiles(boolean v2) throws Exception {
+ String configsetName = "regular";
+ String configsetSuffix = "testOverwriteWithForbiddenFiles-1-" + v2;
+ uploadConfigSetWithAssertions(configsetName, configsetSuffix, null);
+ try (SolrZkClient zkClient =
+ new SolrZkClient(
+ cluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT,
45000, null)) {
+ String configPath = "/configs/" + configsetName + configsetSuffix;
+ assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null,
true, false, v2, true));
+ for (String fileEnding :
ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
+ String f = configPath + "/test." + fileEnding;
+ assertFalse(
+ "Expecting file " + f + " to not exist, because it has a forbidden
file type",
+ zkClient.exists(f, true));
+ }
+ }
+ }
+
@Test
public void testOverwriteWithTrustV1() throws Exception {
testOverwriteWithTrust(false);
@@ -657,7 +688,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
int solrconfigZkVersion =
getConfigZNodeVersion(zkClient, configsetName, configsetSuffix,
"solrconfig.xml");
// Was untrusted, overwrite with untrusted
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null,
true, false, v2));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true,
false, v2, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -667,7 +699,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
getConfigZNodeVersion(zkClient, configsetName, configsetSuffix,
"solrconfig.xml");
// Was untrusted, overwrite with trusted but no cleanup
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr",
true, false, v2));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true,
false, v2, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -692,7 +725,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
"Either empty zipped data, or non-zipped data was passed. In order
to upload a configSet, you must zip a non-empty directory to upload.");
// Was untrusted, overwrite with trusted with cleanup
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr",
true, true, v2));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true,
true, v2, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -706,7 +740,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
assertEquals(
"Can't upload a trusted configset with an untrusted request",
400,
- uploadConfigSet(configsetName, configsetSuffix, null, true, false,
v2));
+ uploadConfigSet(configsetName, configsetSuffix, null, true, false,
v2, false));
assertEquals(
"Expecting version to remain equal",
solrconfigZkVersion,
@@ -718,7 +752,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
assertEquals(
"Can't upload a trusted configset with an untrusted request",
400,
- uploadConfigSet(configsetName, configsetSuffix, null, true, true,
v2));
+ uploadConfigSet(configsetName, configsetSuffix, null, true, true,
v2, false));
assertEquals(
"Expecting version to remain equal",
solrconfigZkVersion,
@@ -727,7 +761,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
unIgnoreException("Trying to make an unstrusted ConfigSet update on a
trusted configSet");
// Was trusted, overwrite with trusted no cleanup
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr",
true, false, v2));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true,
false, v2, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -737,7 +772,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
getConfigZNodeVersion(zkClient, configsetName, configsetSuffix,
"solrconfig.xml");
// Was trusted, overwrite with trusted with cleanup
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr",
true, true, v2));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true,
true, v2, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -1015,6 +1051,51 @@ public class TestConfigSetsAPI extends SolrCloudTestCase
{
}
}
+ @Test
+ public void testSingleFileForbiddenTypeV1() throws Exception {
+ testSingleFileForbiddenType(false);
+ }
+
+ @Test
+ public void testSingleFileForbiddenTypeV2() throws Exception {
+ testSingleFileForbiddenType(true);
+ }
+
+ public void testSingleFileForbiddenType(boolean v2) throws Exception {
+ String configsetName = "regular";
+ String configsetSuffix = "testSingleFileForbiddenType-1-" + v2;
+ uploadConfigSetWithAssertions(configsetName, configsetSuffix, "solr");
+ try (SolrZkClient zkClient =
+ new SolrZkClient(
+ cluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT,
45000, null)) {
+ for (String fileType : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
+ ignoreException("is forbidden for use in configSets");
+ assertEquals(
+ "Can't upload a configset file with a forbidden type: " + fileType,
+ 400,
+ uploadSingleConfigSetFile(
+ configsetName,
+ configsetSuffix,
+ "solr",
+ "solr/configsets/upload/regular/solrconfig.xml",
+ "/test/different/path/solrconfig." + fileType,
+ false,
+ false,
+ v2));
+ assertFalse(
+ "New file should not exist, since the filetype is forbidden: " +
fileType,
+ zkClient.exists(
+ "/configs/"
+ + configsetName
+ + configsetSuffix
+ + "/test/different/path/solrconfig."
+ + fileType,
+ true));
+ unIgnoreException("is forbidden for use in configSets");
+ }
+ }
+ }
+
@Test
public void testSingleFileUntrustedV1() throws Exception {
testSingleFileUntrusted(false);
@@ -1388,7 +1469,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
String configSetName, String suffix, String username, SolrZkClient
zkClient, boolean v2)
throws IOException {
assertFalse(getConfigSetService().checkConfigExists(configSetName +
suffix));
- return uploadConfigSet(configSetName, suffix, username, false, false, v2);
+ return uploadConfigSet(configSetName, suffix, username, false, false, v2,
false);
}
private long uploadConfigSet(
@@ -1397,12 +1478,16 @@ public class TestConfigSetsAPI extends
SolrCloudTestCase {
String username,
boolean overwrite,
boolean cleanup,
- boolean v2)
+ boolean v2,
+ boolean forbiddenTypes)
throws IOException {
// Read zipped sample config
return uploadGivenConfigSet(
- createTempZipFile("solr/configsets/upload/" + configSetName),
+ forbiddenTypes
+ ? createTempZipFileWithForbiddenTypes(
+ "solr/configsets/upload/" + configSetName + "/solrconfig.xml")
+ : createTempZipFile("solr/configsets/upload/" + configSetName),
configSetName,
suffix,
username,
@@ -1554,6 +1639,55 @@ public class TestConfigSetsAPI extends SolrCloudTestCase
{
}
}
+ /**
+ * Create a zip file (in the temp directory) containing a file with all
forbidden types (named
+ * "test.fileType")
+ */
+ private File createTempZipFileWithForbiddenTypes(String file) {
+ try {
+ final File zipFile = createTempFile("configset", "zip").toFile();
+ final File directory = SolrTestCaseJ4.getFile(file);
+ if (log.isInfoEnabled()) {
+ log.info("Directory: {}", directory.getAbsolutePath());
+ }
+ zipWithForbiddenEndings(directory, zipFile);
+ if (log.isInfoEnabled()) {
+ log.info("Zipfile: {}", zipFile.getAbsolutePath());
+ }
+ return zipFile;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static void zipWithForbiddenEndings(File file, File zipfile) throws
IOException {
+ OutputStream out = new FileOutputStream(zipfile);
+ ZipOutputStream zout = new ZipOutputStream(out);
+ try {
+ for (String fileType : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
+ zout.putNextEntry(new ZipEntry("test." + fileType));
+
+ InputStream in = new FileInputStream(file);
+ try {
+ byte[] buffer = new byte[1024];
+ while (true) {
+ int readCount = in.read(buffer);
+ if (readCount < 0) {
+ break;
+ }
+ zout.write(buffer, 0, readCount);
+ }
+ } finally {
+ in.close();
+ }
+
+ zout.closeEntry();
+ }
+ } finally {
+ zout.close();
+ }
+ }
+
private static void zip(File directory, File zipfile) throws IOException {
URI base = directory.toURI();
Deque<File> queue = new ArrayDeque<>();
diff --git
a/solr/solr-ref-guide/modules/configuration-guide/pages/config-sets.adoc
b/solr/solr-ref-guide/modules/configuration-guide/pages/config-sets.adoc
index 782aeab1e61..a04d6e41d18 100644
--- a/solr/solr-ref-guide/modules/configuration-guide/pages/config-sets.adoc
+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/config-sets.adoc
@@ -100,3 +100,22 @@ To upload a file to a configset already stored on
ZooKeeper, you can use xref:de
CAUTION: By default, ZooKeeper's file size limit is 1MB.
If your files are larger than this, you'll need to either
xref:deployment-guide:zookeeper-ensemble.adoc#increasing-the-file-size-limit[increase
the ZooKeeper file size limit] or store them
xref:libs.adoc#lib-directives-in-solrconfig[on the filesystem] of every node in
a cluster.
+
+=== Forbidden File Types
+
+Solr does not accept all file types when uploading or downloading configSets.
+By default the excluded file types are:
+
+- `class`
+- `java`
+- `jar`
+- `tgz`
+- `zip`
+- `tar`
+- `gz`
+
+However, users can impose stricter or looser limits on their systems by
providing a comma separated list of file types
+(without the preceding dot, e.g. `jar,class,csv`), to either of the following
settings:
+
+- System Property: `-DsolrConfigSetForbiddenFileTypes`
+- Environment Variable: `SOLR_CONFIG_SET_FORBIDDEN_FILE_TYPES`
diff --git
a/solr/solr-ref-guide/modules/configuration-guide/pages/configsets-api.adoc
b/solr/solr-ref-guide/modules/configuration-guide/pages/configsets-api.adoc
index 6b8d78be67c..46087834555 100644
--- a/solr/solr-ref-guide/modules/configuration-guide/pages/configsets-api.adoc
+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/configsets-api.adoc
@@ -103,6 +103,8 @@ Upon creation of a collection using an "untrusted"
configset, the following func
If you use any of these parameters or features, you must have enabled security
features in your Solr installation and you must upload the configset as an
authenticated user.
+Not all file types are supported for use in configSets. Please see
xref:configuration-guide:config-sets.adoc#forbidden-file-types[] for more
information.
+
The `upload` command takes the following parameters:
`name`::
diff --git
a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
index 11f6b1c1f40..d110ffdf33b 100644
---
a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
+++
b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
@@ -70,6 +70,8 @@ Due to changes in Lucene 9, that isn't possible any more.
This is an improvement to the binary release artifact, but Jetty does not
allow web-apps (Solr) to share these libraries by default.
The `server/contexts/solr-jetty-context.xml` now explicitly removes these
restrictions, allowing Solr to share these "server" jars which now live in
`server/lib/ext`.
+* Solr no longer accepts all file types for configSets. Please see
xref:configuration-guide:config-sets.adoc#forbidden-file-types[ConfigSet
Forbidden File Types] for more information.
+
=== Tracing
* A new `opentelemetry` module is added, with support for OTEL tracing in
`OTLP` format using gRPC.
At the same time, the `jaegertracer-configurator` module is deprecated for
removal in Solr 10.
diff --git
a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
index 40da28d255d..4b12cbea817 100644
---
a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
+++
b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
@@ -30,6 +30,7 @@ import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
+import java.util.Set;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import org.apache.solr.client.solrj.SolrServerException;
@@ -328,13 +329,20 @@ public class ZkMaintenanceUtils {
public FileVisitResult visitFile(Path file, BasicFileAttributes
attrs)
throws IOException {
String filename = file.getFileName().toString();
- if (filenameExclusions != null &&
filenameExclusions.matcher(filename).matches()) {
+ if ((filenameExclusions != null &&
filenameExclusions.matcher(filename).matches())) {
log.info(
"uploadToZK skipping '{}' due to filenameExclusions '{}'",
filename,
filenameExclusions);
return FileVisitResult.CONTINUE;
}
+ if (isFileForbiddenInConfigSets(filename)) {
+ log.info(
+ "uploadToZK skipping '{}' due to forbidden file types '{}'",
+ filename,
+ USE_FORBIDDEN_FILE_TYPES);
+ return FileVisitResult.CONTINUE;
+ }
String zkNode = createZkNodeName(zkPath, rootPath, file);
try {
// if the path exists (and presumably we're uploading data to
it) just set its data
@@ -421,8 +429,12 @@ public class ZkMaintenanceUtils {
if (children.size() == 0) {
// If we didn't copy data down, then we also didn't create the file.
But we still need a
// marker on the local disk so create an empty file.
- if (copyDataDown(zkClient, zkPath, file) == 0) {
- Files.createFile(file);
+ if (isFileForbiddenInConfigSets(zkPath)) {
+ log.warn("Skipping download of file from ZK, as it is a forbidden
type: {}", zkPath);
+ } else {
+ if (copyDataDown(zkClient, zkPath, file) == 0) {
+ Files.createFile(file);
+ }
}
} else {
Files.createDirectories(file); // Make parent dir.
@@ -548,6 +560,32 @@ public class ZkMaintenanceUtils {
}
return ret;
}
+
+ public static final String FORBIDDEN_FILE_TYPES_PROP =
"solrConfigSetForbiddenFileTypes";
+ public static final String FORBIDDEN_FILE_TYPES_ENV =
"SOLR_CONFIG_SET_FORBIDDEN_FILE_TYPES";
+ public static final Set<String> DEFAULT_FORBIDDEN_FILE_TYPES =
+ Set.of("class", "java", "jar", "tgz", "zip", "tar", "gz");
+ private static volatile Set<String> USE_FORBIDDEN_FILE_TYPES = null;
+
+ public static boolean isFileForbiddenInConfigSets(String filePath) {
+ // Try to set the forbidden file types just once, since it is set by
SysProp/EnvVar
+ if (USE_FORBIDDEN_FILE_TYPES == null) {
+ synchronized (DEFAULT_FORBIDDEN_FILE_TYPES) {
+ if (USE_FORBIDDEN_FILE_TYPES == null) {
+ String userForbiddenFileTypes =
+ System.getProperty(
+ FORBIDDEN_FILE_TYPES_PROP,
System.getenv(FORBIDDEN_FILE_TYPES_ENV));
+ if (StringUtils.isEmpty(userForbiddenFileTypes)) {
+ USE_FORBIDDEN_FILE_TYPES = DEFAULT_FORBIDDEN_FILE_TYPES;
+ } else {
+ USE_FORBIDDEN_FILE_TYPES =
Set.of(userForbiddenFileTypes.split(","));
+ }
+ }
+ }
+ }
+ int lastDot = filePath.lastIndexOf(".");
+ return lastDot >= 0 &&
USE_FORBIDDEN_FILE_TYPES.contains(filePath.substring(lastDot + 1));
+ }
}
class ZkCopier implements ZkMaintenanceUtils.ZkVisitor {
