This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 2c68426fc9 [TIKA-4408] Fix Python env shebang detection (#3473)
2c68426fc9 is described below
commit 2c68426fc910d6461de6f8b31621a060b5165ec6
Author: Lynnbb88 <[email protected]>
AuthorDate: Tue Nov 3 04:40:45 2026 +1100
[TIKA-4408] Fix Python env shebang detection (#3473)
Match the complete env Python shebang at file offset zero, with the
existing no-space and single-space convention. Add regression coverage
for content-only detection, filename hints, and Shell controls.
Generated-by: OpenAI Codex
Co-authored-by: Lynnbb88 <[email protected]>
---
CHANGES.txt | 3 +
.../org/apache/tika/mime/tika-mimetypes.xml | 3 +-
.../tika/mime/PythonShebangDetectionTest.java | 75 ++++++++++++++++++++++
3 files changed, 80 insertions(+), 1 deletion(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index f8fdf8c164..b8a32f721b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -50,6 +50,9 @@ Release 4.3.0 - ???
EXTRACTION AND FORMATS
+ * Detect Python scripts with /usr/bin/env python shebangs as text/x-python
+ instead of application/x-sh (TIKA-4408).
+
* PPTX slides are extracted in presentation order (p:sldIdLst in
presentation.xml)
instead of relationship-id order, which put rId10 before rId2;
embedded-resource
page numbers follow the same order. Slides that are related but missing
from the
diff --git
a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
index 37be9a47b7..be233e391f 100644
--- a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
+++ b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
@@ -8582,7 +8582,8 @@
<match value="#!/usr/local/bin/python" type="string" offset="0"/>
<match value="#! /usr/local/bin/python" type="string" offset="0"/>
<match value="eval "exec /usr/local/bin/python" type="string"
offset="0"/>
- <match value="/bin/env python" type="string" offset="1"/>
+ <match value="#!/usr/bin/env python" type="string" offset="0"/>
+ <match value="#! /usr/bin/env python" type="string" offset="0"/>
</magic>
<glob pattern="*.py"/>
<sub-class-of type="text/plain"/>
diff --git
a/tika-core/src/test/java/org/apache/tika/mime/PythonShebangDetectionTest.java
b/tika-core/src/test/java/org/apache/tika/mime/PythonShebangDetectionTest.java
new file mode 100644
index 0000000000..4b279f8ec0
--- /dev/null
+++
b/tika-core/src/test/java/org/apache/tika/mime/PythonShebangDetectionTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.mime;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import org.apache.tika.Tika;
+
+class PythonShebangDetectionTest {
+
+ private final Tika tika = new Tika();
+
+ static Stream<Arguments> scripts() {
+ return Stream.of(
+ Arguments.of("#!/usr/bin/env python", "print('hello')",
"text/x-python"),
+ Arguments.of("#!/usr/bin/env python3", "print('hello')",
"text/x-python"),
+ Arguments.of("#! /usr/bin/env python3", "print('hello')",
"text/x-python"),
+ Arguments.of("#!/usr/bin/env python3.11", "print('hello')",
"text/x-python"),
+ Arguments.of("#!/usr/bin/python3", "print('hello')",
"text/x-python"),
+ Arguments.of("#!/bin/python3", "print('hello')",
"text/x-python"),
+ Arguments.of("#!/bin/sh", "echo '/bin/env python3'",
"application/x-sh"),
+ Arguments.of("#!/usr/bin/env bash", "echo hello",
"application/x-sh")
+ );
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("scripts")
+ void testContentOnly(String shebang, String body, String expected) {
+ byte[] content = (shebang + "\n" + body + "\n").getBytes(UTF_8);
+ String actual = tika.detect(content);
+ assertEquals(expected, actual, "Content-only detection: " + shebang);
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("scripts")
+ void testWithPythonFilenameHint(String shebang, String body, String
expected) {
+ byte[] content = (shebang + "\n" + body + "\n").getBytes(UTF_8);
+ String actual = tika.detect(content, "script.py");
+ assertEquals(expected, actual, "Detection with script.py hint: " +
shebang);
+ }
+
+ @Test
+ void testPythonFilenameOnly() {
+ String actual = tika.detect("script.py");
+ assertEquals("text/x-python", actual);
+ }
+
+ @Test
+ void testPythonShebangMustBeAtStart() {
+ byte[] content = "\n#!/usr/bin/env
python3\nprint('hello')\n".getBytes(UTF_8);
+ assertEquals("text/plain", tika.detect(content));
+ }
+}
