This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 15534754f49 SOLR-16949: Restrict certain file types from being
uploaded to or downloaded from Config Sets
15534754f49 is described below
commit 15534754f492079e52288dd11abaf1c4261b3ea4
Author: Jan Høydahl <[email protected]>
AuthorDate: Wed Dec 13 22:49:23 2023 +0100
SOLR-16949: Restrict certain file types from being uploaded to or
downloaded from Config Sets
---
solr/CHANGES.txt | 2 +
solr/core/build.gradle | 2 +
.../org/apache/solr/cli/ConfigSetUploadTool.java | 2 +
.../org/apache/solr/cloud/ZkConfigSetService.java | 21 ++-
.../solr/core/FileSystemConfigSetService.java | 28 +++-
.../org/apache/solr/core/backup/BackupManager.java | 23 ++-
.../handler/configsets/UploadConfigSetFileAPI.java | 8 +-
.../org/apache/solr/util/FileTypeMagicUtil.java | 166 +++++++++++++++++++++
solr/core/src/resources/magic/executables | 74 +++++++++
solr/core/src/test-files/magic/HelloWorld.java.txt | 5 +
.../test-files/magic/HelloWorldJavaClass.class.bin | Bin 0 -> 426 bytes
solr/core/src/test-files/magic/README.md | 29 ++++
solr/core/src/test-files/magic/hello.tar.bin | Bin 0 -> 4096 bytes
solr/core/src/test-files/magic/plain.txt | 1 +
solr/core/src/test-files/magic/shell.sh.txt | 2 +
.../org/apache/solr/cloud/TestConfigSetsAPI.java | 141 +++++++++++------
.../apache/solr/util/FileTypeMagicUtilTest.java | 54 +++++++
solr/licenses/simplemagic-1.17.jar.sha1 | 1 +
solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt | 15 ++
solr/licenses/simplemagic-NOTICE.txt | 0
.../solr/common/cloud/ZkMaintenanceUtils.java | 2 +
versions.lock | 1 +
versions.props | 1 +
23 files changed, 522 insertions(+), 56 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 40da8c435ac..2fd94304a24 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -169,6 +169,8 @@ Other Changes
* SOLR-17091: dev tools script cloud.sh became broken after changes in 9.3
added a new -slim.tgz file it was not expecting
cloud.sh has been updated to ignore the -slim.tgz version of the tarball.
+* SOLR-16949: Restrict certain file types from being uploaded to or downloaded
from Config Sets (janhoy, Houston Putman)
+
================== 9.4.0 ==================
New Features
---------------------
diff --git a/solr/core/build.gradle b/solr/core/build.gradle
index 61ecd1713af..ed2c8a370ae 100644
--- a/solr/core/build.gradle
+++ b/solr/core/build.gradle
@@ -159,6 +159,8 @@ dependencies {
compileOnly 'com.github.stephenc.jcip:jcip-annotations'
+ implementation 'com.j256.simplemagic:simplemagic'
+
// -- Test Dependencies
testRuntimeOnly 'org.slf4j:jcl-over-slf4j'
diff --git a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
index 5fd4a538bd7..6576742a195 100644
--- a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
@@ -27,6 +27,7 @@ import org.apache.solr.client.solrj.impl.SolrZkClientTimeout;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.core.ConfigSetService;
+import org.apache.solr.util.FileTypeMagicUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -101,6 +102,7 @@ public class ConfigSetUploadTool extends ToolBase {
+ cli.getOptionValue("confname")
+ " to ZooKeeper at "
+ zkHost);
+ FileTypeMagicUtil.assertConfigSetFolderLegal(confPath);
ZkMaintenanceUtils.uploadToZK(
zkClient,
confPath,
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
index f02404d636d..9abde098e1c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
@@ -22,6 +22,7 @@ import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
@@ -39,6 +40,7 @@ import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.util.FileTypeMagicUtil;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
@@ -199,6 +201,15 @@ public class ZkConfigSetService extends ConfigSetService {
try {
if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
log.warn("Not including uploading file to config, as it is a forbidden
type: {}", fileName);
+ } else if (FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
+ String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ String.format(
+ Locale.ROOT,
+ "Not uploading file %s to config, as it matched the MAGIC
signature of a forbidden mime type %s",
+ fileName,
+ mimeType));
} else {
// if overwriteOnExists is true then zkClient#makePath failOnExists is
set to false
zkClient.makePath(filePath, data, CreateMode.PERSISTENT, null,
!overwriteOnExists, true);
@@ -340,7 +351,15 @@ public class ZkConfigSetService extends ConfigSetService {
} else {
log.debug("Copying zk node {} to {}", fromZkFilePath, toZkFilePath);
byte[] data = zkClient.getData(fromZkFilePath, null, null, true);
- zkClient.makePath(toZkFilePath, data, true);
+ if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
+ zkClient.makePath(toZkFilePath, data, true);
+ } else {
+ String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
+ log.warn(
+ "Skipping copy of file {} in ZK, as it matched the MAGIC signature
of a forbidden mime type {}",
+ fromZkFilePath,
+ mimeType);
+ }
}
}
diff --git
a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
index 1f1a42b15e1..4b041252211 100644
--- a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
+++ b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
@@ -37,6 +37,7 @@ import java.util.stream.Stream;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.util.FileTypeMagicUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -150,9 +151,17 @@ public class FileSystemConfigSetService extends
ConfigSetService {
if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
log.warn("Not including uploading file to config, as it is a forbidden
type: {}", fileName);
} else {
- Path filePath =
getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
- if (!Files.exists(filePath) || overwriteOnExists) {
- Files.write(filePath, data);
+ if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
+ Path filePath =
getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
+ if (!Files.exists(filePath) || overwriteOnExists) {
+ Files.write(filePath, data);
+ }
+ } else {
+ String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
+ log.warn(
+ "Not including uploading file {}, as it matched the MAGIC
signature of a forbidden mime type {}",
+ fileName,
+ mimeType);
}
}
}
@@ -205,8 +214,17 @@ public class FileSystemConfigSetService extends
ConfigSetService {
"Not including uploading file to config, as it is a
forbidden type: {}",
file.getFileName());
} else {
- Files.copy(
- file, target.resolve(source.relativize(file).toString()),
REPLACE_EXISTING);
+ if
(!FileTypeMagicUtil.isFileForbiddenInConfigset(Files.newInputStream(file))) {
+ Files.copy(
+ file,
target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);
+ } else {
+ String mimeType =
+
FileTypeMagicUtil.INSTANCE.guessMimeType(Files.newInputStream(file));
+ log.warn(
+ "Not copying file {}, as it matched the MAGIC signature
of a forbidden mime type {}",
+ file.getFileName(),
+ mimeType);
+ }
}
return FileVisitResult.CONTINUE;
}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
index be6a1a83c2f..8ff78b27e08 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
@@ -40,6 +40,7 @@ import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.ConfigSetService;
import org.apache.solr.core.backup.repository.BackupRepository;
+import org.apache.solr.util.FileTypeMagicUtil;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@@ -349,8 +350,16 @@ public class BackupManager {
if (data == null) {
data = new byte[0];
}
- try (OutputStream os = repository.createOutput(uri)) {
- os.write(data);
+ if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
+ try (OutputStream os = repository.createOutput(uri)) {
+ os.write(data);
+ }
+ } else {
+ String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
+ log.warn(
+ "Not including zookeeper file {} in backup, as it matched the
MAGIC signature of a forbidden mime type {}",
+ filePath,
+ mimeType);
}
}
} else {
@@ -379,7 +388,15 @@ public class BackupManager {
// probably ok since the config file should be small.
byte[] arr = new byte[(int) is.length()];
is.readBytes(arr, 0, (int) is.length());
- configSetService.uploadFileToConfig(configName, filePath, arr,
false);
+ if (!FileTypeMagicUtil.isFileForbiddenInConfigset(arr)) {
+ configSetService.uploadFileToConfig(configName, filePath,
arr, false);
+ } else {
+ String mimeType =
FileTypeMagicUtil.INSTANCE.guessMimeType(arr);
+ log.warn(
+ "Not including zookeeper file {} in restore, as it
matched the MAGIC signature of a forbidden mime type {}",
+ filePath,
+ mimeType);
+ }
}
}
break;
diff --git
a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
index 98889aff563..2380a79a92b 100644
---
a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
+++
b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
@@ -27,6 +27,7 @@ import org.apache.solr.common.params.ConfigSetParams;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.util.FileTypeMagicUtil;
/**
* V2 API for adding or updating a single file within a configset.
@@ -67,11 +68,13 @@ public class UploadConfigSetFileAPI extends
ConfigSetAPIBase {
if (fixedSingleFilePath.charAt(0) == '/') {
fixedSingleFilePath = fixedSingleFilePath.substring(1);
}
+ byte[] data = inputStream.readAllBytes();
if (fixedSingleFilePath.isEmpty()) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"The file path provided for upload, '" + singleFilePath + "', is not
valid.");
- } else if
(ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath)) {
+ } else if
(ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath)
+ || FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"The file type provided for upload, '"
@@ -87,8 +90,7 @@ public class UploadConfigSetFileAPI extends ConfigSetAPIBase {
// For creating the baseNode, the cleanup parameter is only allowed to
be true when
// singleFilePath is not passed.
createBaseNode(configSetService, overwritesExisting, requestIsTrusted,
configSetName);
- configSetService.uploadFileToConfig(
- configSetName, fixedSingleFilePath, inputStream.readAllBytes(),
allowOverwrite);
+ configSetService.uploadFileToConfig(configSetName, fixedSingleFilePath,
data, allowOverwrite);
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java
b/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java
new file mode 100644
index 00000000000..cfb6c9fa0af
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util;
+
+import com.j256.simplemagic.ContentInfo;
+import com.j256.simplemagic.ContentInfoUtil;
+import com.j256.simplemagic.ContentType;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.Set;
+import org.apache.solr.common.SolrException;
+
+/** Utility class to guess the mime type of file based on its magic number. */
+public class FileTypeMagicUtil implements ContentInfoUtil.ErrorCallBack {
+ private final ContentInfoUtil util;
+ private static final Set<String> SKIP_FOLDERS = new
HashSet<>(Arrays.asList(".", ".."));
+
+ public static FileTypeMagicUtil INSTANCE = new FileTypeMagicUtil();
+
+ FileTypeMagicUtil() {
+ try {
+ util = new ContentInfoUtil("/magic/executables", this);
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error
parsing magic file", e);
+ }
+ }
+
+ /**
+ * Asserts that an entire configset folder is legal to upload.
+ *
+ * @param confPath the path to the folder
+ * @throws SolrException if an illegal file is found in the folder structure
+ */
+ public static void assertConfigSetFolderLegal(Path confPath) throws
IOException {
+ Files.walkFileTree(
+ confPath,
+ new SimpleFileVisitor<Path>() {
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes
attrs)
+ throws IOException {
+ // Read first 100 bytes of the file to determine the mime type
+ try (InputStream fileStream = Files.newInputStream(file)) {
+ byte[] bytes = new byte[100];
+ fileStream.read(bytes);
+ if (FileTypeMagicUtil.isFileForbiddenInConfigset(bytes)) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ String.format(
+ Locale.ROOT,
+ "Not uploading file %s to configset, as it matched the
MAGIC signature of a forbidden mime type %s",
+ file,
+ FileTypeMagicUtil.INSTANCE.guessMimeType(bytes)));
+ }
+ return FileVisitResult.CONTINUE;
+ }
+ }
+
+ @Override
+ public FileVisitResult preVisitDirectory(Path dir,
BasicFileAttributes attrs)
+ throws IOException {
+ if (SKIP_FOLDERS.contains(dir.getFileName().toString()))
+ return FileVisitResult.SKIP_SUBTREE;
+
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ }
+
+ /**
+ * Guess the mime type of file based on its magic number.
+ *
+ * @param stream input stream of the file
+ * @return string with content-type or "application/octet-stream" if unknown
+ */
+ public String guessMimeType(InputStream stream) {
+ try {
+ ContentInfo info = util.findMatch(stream);
+ if (info == null) {
+ return ContentType.OTHER.getMimeType();
+ }
+ return info.getContentType().getMimeType();
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ }
+
+ /**
+ * Guess the mime type of file bytes based on its magic number.
+ *
+ * @param bytes the first bytes at start of the file
+ * @return string with content-type or "application/octet-stream" if unknown
+ */
+ public String guessMimeType(byte[] bytes) {
+ return guessMimeType(new ByteArrayInputStream(bytes));
+ }
+
+ @Override
+ public void error(String line, String details, Exception e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ String.format(Locale.ROOT, "%s: %s", line, details),
+ e);
+ }
+
+ /**
+ * Determine forbidden file type based on magic bytes matching of the file
itself. Forbidden types
+ * are:
+ *
+ * <ul>
+ * <li><code>application/x-java-applet</code>: java class file
+ * <li><code>application/zip</code>: jar or zip archives
+ * <li><code>application/x-tar</code>: tar archives
+ * <li><code>text/x-shellscript</code>: shell or bash script
+ * </ul>
+ *
+ * @param fileStream stream from the file content
+ * @return true if file is among the forbidden mime-types
+ */
+ public static boolean isFileForbiddenInConfigset(InputStream fileStream) {
+ return
forbiddenTypes.contains(FileTypeMagicUtil.INSTANCE.guessMimeType(fileStream));
+ }
+
+ /**
+ * Determine forbidden file type based on magic bytes matching of the first
bytes of the file.
+ *
+ * @param bytes byte array of the file content
+ * @return true if file is among the forbidden mime-types
+ */
+ public static boolean isFileForbiddenInConfigset(byte[] bytes) {
+ if (bytes == null || bytes.length == 0)
+ return false; // A ZK znode may be a folder with no content
+ return isFileForbiddenInConfigset(new ByteArrayInputStream(bytes));
+ }
+
+ private static final Set<String> forbiddenTypes =
+ new HashSet<>(
+ Arrays.asList(
+ System.getProperty(
+ "solr.configset.upload.mimetypes.forbidden",
+
"application/x-java-applet,application/zip,application/x-tar,text/x-shellscript")
+ .split(",")));
+}
diff --git a/solr/core/src/resources/magic/executables
b/solr/core/src/resources/magic/executables
new file mode 100644
index 00000000000..04094eaf797
--- /dev/null
+++ b/solr/core/src/resources/magic/executables
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# POSIX tar archives
+# URL: https://en.wikipedia.org/wiki/Tar_(computing)
+# Reference:
https://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5&manpath=FreeBSD+8-current
+# header mainly padded with nul bytes
+500 quad 0
+!:strength /2
+# filename or extended attribute printable strings in range space null til
umlaut ue
+>0 ubeshort >0x1F00
+>>0 ubeshort <0xFCFD
+# last 4 header bytes often null but tar\0 in gtarfail2.tar gtarfail.tar-bad
+# at https://sourceforge.net/projects/s-tar/files/testscripts/
+>>>508 ubelong&0x8B9E8DFF 0
+# nul, space or ascii digit 0-7 at start of mode
+>>>>100 ubyte&0xC8 =0
+>>>>>101 ubyte&0xC8 =0
+# nul, space at end of check sum
+>>>>>>155 ubyte&0xDF =0
+# space or ascii digit 0 at start of check sum
+>>>>>>>148 ubyte&0xEF =0x20
+# check for specific 1st member name that indicates other mime type and file
name suffix
+>>>>>>>>0 string TpmEmuTpms/permall
+!:mime application/x-tar
+!:ext tar
+# other stuff in padding
+# some implementations add new fields to the blank area at the end of the
header record
+# created for example by DOS TAR 3.20g 1994 Tim V.Shapore with -j option
+>>257 ulong !0 tar archive (old)
+!:mime application/x-tar
+!:ext tar
+# magic in newer, GNU, posix variants
+>257 string =ustar
+# 2 last char of magic and UStar version because string expression does not
work
+# 2 space characters followed by a null for GNU variant
+>>261 ubelong =0x72202000 POSIX tar archive (GNU)
+!:mime application/x-gtar
+!:ext tar/gtar
+
+
+# Zip archives (Greg Roelofs, c/o [email protected])
+0 string PK\005\006 Zip archive data (empty)
+0 string PK\003\004 Zip archive data
+!:strength +1
+!:mime application/zip
+!:ext zip/cbz
+
+
+# JAVA
+0 belong 0xcafebabe
+>4 ubelong >30 compiled Java class data,
+!:mime application/x-java-applet
+#!:mime application/java-byte-code
+!:ext class
+
+
+# SHELL scripts
+#0 string/w : shell archive or script for
antique kernel text
+0 regex \^#!\\s?(/bin/|/usr/) POSIX shell script text
executable
+!:mime text/x-shellscript
+!:ext sh/bash
\ No newline at end of file
diff --git a/solr/core/src/test-files/magic/HelloWorld.java.txt
b/solr/core/src/test-files/magic/HelloWorld.java.txt
new file mode 100644
index 00000000000..ca9518d2afa
--- /dev/null
+++ b/solr/core/src/test-files/magic/HelloWorld.java.txt
@@ -0,0 +1,5 @@
+class HelloWorld {
+ public static void main(String[] args) {
+ System.out.println("Hellow world");
+ }
+}
\ No newline at end of file
diff --git a/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin
b/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin
new file mode 100644
index 00000000000..e15d0a6c5b9
Binary files /dev/null and
b/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin differ
diff --git a/solr/core/src/test-files/magic/README.md
b/solr/core/src/test-files/magic/README.md
new file mode 100644
index 00000000000..6e499a2f711
--- /dev/null
+++ b/solr/core/src/test-files/magic/README.md
@@ -0,0 +1,29 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+The two binary files were created by the following commands:
+
+```bash
+echo "Hello" > hello.txt && \
+ tar -cvf hello.tar.bin hello.txt && \
+ rm hello.txt
+
+cp HelloWorld.java.txt HelloWorld.java && \
+ javac HelloWorld.java && \
+ mv HelloWorld.class HelloWorldJavaClass.class.bin && \
+ rm HelloWorld.java
+```
\ No newline at end of file
diff --git a/solr/core/src/test-files/magic/hello.tar.bin
b/solr/core/src/test-files/magic/hello.tar.bin
new file mode 100644
index 00000000000..68ca23c362a
Binary files /dev/null and b/solr/core/src/test-files/magic/hello.tar.bin differ
diff --git a/solr/core/src/test-files/magic/plain.txt
b/solr/core/src/test-files/magic/plain.txt
new file mode 100644
index 00000000000..70c379b63ff
--- /dev/null
+++ b/solr/core/src/test-files/magic/plain.txt
@@ -0,0 +1 @@
+Hello world
\ No newline at end of file
diff --git a/solr/core/src/test-files/magic/shell.sh.txt
b/solr/core/src/test-files/magic/shell.sh.txt
new file mode 100644
index 00000000000..9ea411e111d
--- /dev/null
+++ b/solr/core/src/test-files/magic/shell.sh.txt
@@ -0,0 +1,2 @@
+#! /usr/bin/env bash
+echo Hello
\ No newline at end of file
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
index 1da60d85ff9..f7c9431c296 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
@@ -45,6 +45,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -592,14 +593,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
assertEquals(
"Can't overwrite an existing configset unless the overwrite
parameter is set",
400,
- uploadConfigSet(configsetName, configsetSuffix, null, false, false,
v2, false));
+ uploadConfigSet(configsetName, configsetSuffix, null, false, false,
v2, false, false));
unIgnoreException("The configuration regulartestOverwrite-1 already
exists in zookeeper");
assertEquals(
"Expecting version to remain equal",
solrconfigZkVersion,
getConfigZNodeVersion(zkClient, configsetName, configsetSuffix,
"solrconfig.xml"));
assertEquals(
- 0, uploadConfigSet(configsetName, configsetSuffix, null, true,
false, v2, false));
+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true,
false, v2, false, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -638,13 +639,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
zkClient.makePath(f, true);
}
assertEquals(
- 0, uploadConfigSet(configsetName, configsetSuffix, null, true,
false, v2, false));
+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true,
false, v2, false, false));
for (String f : extraFiles) {
assertTrue(
"Expecting file " + f + " to exist in ConfigSet but it's gone",
zkClient.exists(f, true));
}
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null,
true, true, v2, false));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true, true,
v2, false, false));
for (String f : extraFiles) {
assertFalse(
"Expecting file " + f + " to be deleted from ConfigSet but it
wasn't",
@@ -675,7 +677,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
.withConnTimeOut(45000, TimeUnit.MILLISECONDS)
.build()) {
String configPath = "/configs/" + configsetName + configsetSuffix;
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null,
true, false, v2, true));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true,
false, v2, true, false));
for (String fileEnding :
ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
String f = configPath + "/test." + fileEnding;
assertFalse(
@@ -710,7 +713,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
getConfigZNodeVersion(zkClient, configsetName, configsetSuffix,
"solrconfig.xml");
// Was untrusted, overwrite with untrusted
assertEquals(
- 0, uploadConfigSet(configsetName, configsetSuffix, null, true,
false, v2, false));
+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true,
false, v2, false, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -721,7 +724,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
// Was untrusted, overwrite with trusted but no cleanup
assertEquals(
- 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true,
false, v2, false));
+ 0,
+ uploadConfigSet(configsetName, configsetSuffix, "solr", true, false,
v2, false, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -747,7 +751,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
// Was untrusted, overwrite with trusted with cleanup
assertEquals(
- 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true,
true, v2, false));
+ 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true,
true, v2, false, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -761,7 +765,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
assertEquals(
"Can't upload a trusted configset with an untrusted request",
400,
- uploadConfigSet(configsetName, configsetSuffix, null, true, false,
v2, false));
+ uploadConfigSet(configsetName, configsetSuffix, null, true, false,
v2, false, false));
assertEquals(
"Expecting version to remain equal",
solrconfigZkVersion,
@@ -773,7 +777,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
assertEquals(
"Can't upload a trusted configset with an untrusted request",
400,
- uploadConfigSet(configsetName, configsetSuffix, null, true, true,
v2, false));
+ uploadConfigSet(configsetName, configsetSuffix, null, true, true,
v2, false, false));
assertEquals(
"Expecting version to remain equal",
solrconfigZkVersion,
@@ -783,7 +787,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
// Was trusted, overwrite with trusted no cleanup
assertEquals(
- 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true,
false, v2, false));
+ 0,
+ uploadConfigSet(configsetName, configsetSuffix, "solr", true, false,
v2, false, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -794,7 +799,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
// Was trusted, overwrite with trusted with cleanup
assertEquals(
- 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true,
true, v2, false));
+ 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true,
true, v2, false, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -1457,6 +1462,13 @@ public class TestConfigSetsAPI extends SolrCloudTestCase
{
.get("id"));
}
+ @Test
+ public void testUploadWithForbiddenContent() throws Exception {
+ // Uploads a config set containing a script, a class file and jar file,
will return 400 error
+ long res = uploadConfigSet("forbidden", "suffix", "foo", true, false,
true, false, true);
+ assertEquals(400, res);
+ }
+
private static String getSecurityJson() {
return "{\n"
+ " 'authentication':{\n"
@@ -1511,7 +1523,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
String configSetName, String suffix, String username, SolrZkClient
zkClient, boolean v2)
throws IOException {
assertFalse(getConfigSetService().checkConfigExists(configSetName +
suffix));
- return uploadConfigSet(configSetName, suffix, username, false, false, v2,
false);
+ return uploadConfigSet(configSetName, suffix, username, false, false, v2,
false, false);
}
private long uploadConfigSet(
@@ -1521,21 +1533,25 @@ public class TestConfigSetsAPI extends
SolrCloudTestCase {
boolean overwrite,
boolean cleanup,
boolean v2,
- boolean forbiddenTypes)
+ boolean forbiddenTypes,
+ boolean forbiddenContent)
throws IOException {
+ File zipFile;
+ if (forbiddenTypes) {
+ log.info("Uploading configset with forbidden file endings");
+ zipFile =
+ createTempZipFileWithForbiddenTypes(
+ "solr/configsets/upload/" + configSetName + "/solrconfig.xml");
+ } else if (forbiddenContent) {
+ log.info("Uploading configset with forbidden file content");
+ zipFile = createTempZipFileWithForbiddenContent("magic");
+ } else {
+ zipFile = createTempZipFile("solr/configsets/upload/" + configSetName);
+ }
+
// Read zipped sample config
- return uploadGivenConfigSet(
- forbiddenTypes
- ? createTempZipFileWithForbiddenTypes(
- "solr/configsets/upload/" + configSetName + "/solrconfig.xml")
- : createTempZipFile("solr/configsets/upload/" + configSetName),
- configSetName,
- suffix,
- username,
- overwrite,
- cleanup,
- v2);
+ return uploadGivenConfigSet(zipFile, configSetName, suffix, username,
overwrite, cleanup, v2);
}
private long uploadBadConfigSet(String configSetName, String suffix, String
username, boolean v2)
@@ -1702,31 +1718,68 @@ public class TestConfigSetsAPI extends
SolrCloudTestCase {
}
}
- private static void zipWithForbiddenEndings(File file, File zipfile) throws
IOException {
- OutputStream out = new FileOutputStream(zipfile);
- ZipOutputStream zout = new ZipOutputStream(out);
+ /** Create a zip file (in the temp directory) containing files with
forbidden content */
+ private File createTempZipFileWithForbiddenContent(String resourcePath) {
try {
- for (String fileType : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
- zout.putNextEntry(new ZipEntry("test." + fileType));
+ final File zipFile = createTempFile("configset", "zip").toFile();
+ final File directory = SolrTestCaseJ4.getFile(resourcePath);
+ if (log.isInfoEnabled()) {
+ log.info("Directory: {}", directory.getAbsolutePath());
+ }
+ zipWithForbiddenContent(directory, zipFile);
+ if (log.isInfoEnabled()) {
+ log.info("Zipfile: {}", zipFile.getAbsolutePath());
+ }
+ return zipFile;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
- InputStream in = new FileInputStream(file);
- try {
- byte[] buffer = new byte[1024];
- while (true) {
- int readCount = in.read(buffer);
- if (readCount < 0) {
- break;
+ private static void zipWithForbiddenContent(File directory, File zipfile)
throws IOException {
+ OutputStream out = Files.newOutputStream(zipfile.toPath());
+ assertTrue(directory.isDirectory());
+ try (ZipOutputStream zout = new ZipOutputStream(out)) {
+ // Copy in all files from the directory
+ for (File file : Objects.requireNonNull(directory.listFiles())) {
+ zout.putNextEntry(new ZipEntry(file.getName()));
+ zout.write(Files.readAllBytes(file.toPath()));
+ zout.closeEntry();
+ }
+ }
+ }
+
+ private static void zipWithForbiddenEndings(File fileOrDirectory, File
zipfile)
+ throws IOException {
+ OutputStream out = new FileOutputStream(zipfile);
+ try (ZipOutputStream zout = new ZipOutputStream(out)) {
+ if (fileOrDirectory.isFile()) {
+ // Create entries with given file, one for each forbidden endding
+ for (String fileType :
ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
+ zout.putNextEntry(new ZipEntry("test." + fileType));
+
+ try (InputStream in = new FileInputStream(fileOrDirectory)) {
+ byte[] buffer = new byte[1024];
+ while (true) {
+ int readCount = in.read(buffer);
+ if (readCount < 0) {
+ break;
+ }
+ zout.write(buffer, 0, readCount);
}
- zout.write(buffer, 0, readCount);
}
- } finally {
- in.close();
- }
- zout.closeEntry();
+ zout.closeEntry();
+ }
+ }
+ if (fileOrDirectory.isDirectory()) {
+ // Copy in all files from the directory
+ for (File file : Objects.requireNonNull(fileOrDirectory.listFiles())) {
+ zout.putNextEntry(new ZipEntry(file.getName()));
+ zout.write(Files.readAllBytes(file.toPath()));
+ zout.closeEntry();
+ }
}
- } finally {
- zout.close();
}
}
diff --git a/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java
b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java
new file mode 100644
index 00000000000..b8e9a35a3d9
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util;
+
+import org.apache.solr.SolrTestCaseJ4;
+
+public class FileTypeMagicUtilTest extends SolrTestCaseJ4 {
+ public void testGuessMimeType() {
+ assertEquals(
+ "application/x-java-applet",
+ FileTypeMagicUtil.INSTANCE.guessMimeType(
+
FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin")));
+ assertEquals(
+ "application/zip",
+ FileTypeMagicUtil.INSTANCE.guessMimeType(
+ FileTypeMagicUtil.class.getResourceAsStream(
+ "/runtimecode/containerplugin.v.1.jar.bin")));
+ assertEquals(
+ "application/x-tar",
+ FileTypeMagicUtil.INSTANCE.guessMimeType(
+
FileTypeMagicUtil.class.getResourceAsStream("/magic/hello.tar.bin")));
+ assertEquals(
+ "text/x-shellscript",
+ FileTypeMagicUtil.INSTANCE.guessMimeType(
+
FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt")));
+ }
+
+ public void testIsFileForbiddenInConfigset() {
+ assertTrue(
+ FileTypeMagicUtil.isFileForbiddenInConfigset(
+
FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin")));
+ assertTrue(
+ FileTypeMagicUtil.isFileForbiddenInConfigset(
+
FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt")));
+ assertFalse(
+ FileTypeMagicUtil.isFileForbiddenInConfigset(
+ FileTypeMagicUtil.class.getResourceAsStream("/magic/plain.txt")));
+ }
+}
diff --git a/solr/licenses/simplemagic-1.17.jar.sha1
b/solr/licenses/simplemagic-1.17.jar.sha1
new file mode 100644
index 00000000000..cf101094cc8
--- /dev/null
+++ b/solr/licenses/simplemagic-1.17.jar.sha1
@@ -0,0 +1 @@
+b6e2d1e47d7172e57fa858a2e3940c09a590e61e
diff --git a/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt
b/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt
new file mode 100644
index 00000000000..9228230f933
--- /dev/null
+++ b/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt
@@ -0,0 +1,15 @@
+ISC License (https://opensource.org/licenses/ISC)
+
+Copyright 2021, Gray Watson
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
\ No newline at end of file
diff --git a/solr/licenses/simplemagic-NOTICE.txt
b/solr/licenses/simplemagic-NOTICE.txt
new file mode 100644
index 00000000000..e69de29bb2d
diff --git
a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
index d571339880c..e40294a6683 100644
---
a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
+++
b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
@@ -348,6 +348,7 @@ public class ZkMaintenanceUtils {
USE_FORBIDDEN_FILE_TYPES);
return FileVisitResult.CONTINUE;
}
+ // TODO: Cannot check MAGIC header for file since FileTypeGuesser
is in core
String zkNode = createZkNodeName(zkPath, rootPath, file);
try {
// if the path exists (and presumably we're uploading data to
it) just set its data
@@ -437,6 +438,7 @@ public class ZkMaintenanceUtils {
if (isFileForbiddenInConfigSets(zkPath)) {
log.warn("Skipping download of file from ZK, as it is a forbidden
type: {}", zkPath);
} else {
+ // TODO: Cannot check MAGIC header for file since FileTypeGuesser is
in core
if (copyDataDown(zkClient, zkPath, file) == 0) {
Files.createFile(file);
}
diff --git a/versions.lock b/versions.lock
index 9c0f9ef53d4..6b6ea467e0d 100644
--- a/versions.lock
+++ b/versions.lock
@@ -62,6 +62,7 @@ com.googlecode.plist:dd-plist:1.24 (1 constraints: 300c84f5)
com.healthmarketscience.jackcess:jackcess:4.0.2 (1 constraints: 5d0cf201)
com.healthmarketscience.jackcess:jackcess-encrypt:4.0.1 (1 constraints:
5c0cf101)
com.ibm.icu:icu4j:70.1 (1 constraints: a90f1784)
+com.j256.simplemagic:simplemagic:1.17 (1 constraints: dd04f830)
com.jayway.jsonpath:json-path:2.8.0 (2 constraints: 6c12952c)
com.lmax:disruptor:3.4.4 (1 constraints: 0d050a36)
com.mchange:c3p0:0.9.5.5 (1 constraints: c80c571b)
diff --git a/versions.props b/versions.props
index 10b583c43e2..cfa127e1094 100644
--- a/versions.props
+++ b/versions.props
@@ -13,6 +13,7 @@ com.google.cloud:google-cloud-bom=0.204.0
com.google.errorprone:*=2.23.0
com.google.guava:guava=32.1.3-jre
com.google.re2j:re2j=1.7
+com.j256.simplemagic:simplemagic=1.17
com.jayway.jsonpath:json-path=2.8.0
com.lmax:disruptor=3.4.4
com.tdunning:t-digest=3.1
