This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch branch_3x
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/branch_3x by this push:
new 1f17a44efe TIKA-5065 branch 3x (#3634)
1f17a44efe is described below
commit 1f17a44efe2b5ad2d2cd5bb3475048cee9db43b9
Author: Tim Allison <[email protected]>
AuthorDate: Tue Nov 17 11:22:13 2026 -0500
TIKA-5065 branch 3x (#3634)
* TIKA-5065: improve file type precision for external tools and libraries
that process files (backport of #3630)
---
CHANGES.txt | 8 +
.../org/apache/tika/embedder/ExternalEmbedder.java | 29 +++-
.../java/org/apache/tika/io/IsolatedInput.java | 175 +++++++++++++++++++
.../tika/parser/external/ExternalParser.java | 9 +-
.../tika/parser/external2/ExternalParser.java | 29 +++-
.../tika/embedder/ExternalEmbedderInputTest.java | 53 ++++++
.../java/org/apache/tika/io/InputFileProbe.java | 85 ++++++++++
.../java/org/apache/tika/io/IsolatedInputTest.java | 137 +++++++++++++++
.../parser/external2/ExternalParserInputTest.java | 78 +++++++++
.../org/apache/tika/parser/gdal/GDALParser.java | 187 ++++++++++++---------
.../geoinfo/GeographicInformationParser.java | 7 +-
.../org/apache/tika/parser/grib/GribParser.java | 5 +-
.../java/org/apache/tika/parser/hdf/HDFParser.java | 9 +-
.../apache/tika/parser/isatab/ISArchiveParser.java | 35 +++-
.../apache/tika/parser/netcdf/NetCDFParser.java | 4 +-
.../apache/tika/parser/netcdf/NetcdfReaders.java | 115 +++++++++++++
.../apache/tika/parser/gdal/TestGDALParser.java | 50 +++++-
.../tika/parser/isatab/ISArchiveParserTest.java | 32 +++-
.../tika/parser/netcdf/NetcdfReadersTest.java | 119 +++++++++++++
.../parser/scientific/integration/TestParsers.java | 22 ++-
.../apache/tika/parser/ocr/ImagePreprocessor.java | 7 +-
.../org/apache/tika/parser/ocr/TesseractInput.java | 95 +++++++++++
.../apache/tika/parser/ocr/TesseractOCRParser.java | 72 +++++---
.../apache/tika/parser/ocr/TesseractInputTest.java | 85 ++++++++++
.../tika/parser/ocr/TesseractOCRParserTest.java | 17 ++
.../java/org/apache/tika/parser/pkg/RarParser.java | 17 +-
.../org/apache/tika/parser/pkg/UnrarParser.java | 21 ++-
27 files changed, 1360 insertions(+), 142 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 70eeff45dc..3ccccfba11 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,13 @@
Release 3.4.0 - ???
+ * External tools and libraries are told which formats to read, and get the
document as
+ "input" plus an extension of its detected type, in a directory of its own.
GDALParser reads
+ and claims only the drivers in allowedFormats (default GTiff, netCDF,
GRIB, FITS, NITF,
+ DTED); the external2 ExternalParser's inputFileExtension overrides the
extension;
+ ISArchiveParser reads files beside the study only with readSiblingFiles
(or when given a
+ directory); Tesseract and ImageMagick refuse input that is not a supported
image
+ (TIKA-5065).
+
* UTF-8 text that was misdetected as GB18030 is now detected as UTF-8
(juniversalchardet 2.5.2).
diff --git
a/tika-core/src/main/java/org/apache/tika/embedder/ExternalEmbedder.java
b/tika-core/src/main/java/org/apache/tika/embedder/ExternalEmbedder.java
index c58d57345f..5d8c3e23a1 100644
--- a/tika-core/src/main/java/org/apache/tika/embedder/ExternalEmbedder.java
+++ b/tika-core/src/main/java/org/apache/tika/embedder/ExternalEmbedder.java
@@ -34,6 +34,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.IsolatedInput;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
@@ -345,6 +346,7 @@ public class ExternalEmbedder implements Embedder {
TikaInputStream tikaInputStream = TikaInputStream.get(inputStream);
File tempOutputFile = null;
+ IsolatedInput isolated = null;
List<String> commandMetadataSegments = null;
if (hasMetadataCommandArguments) {
@@ -356,12 +358,21 @@ public class ExternalEmbedder implements Embedder {
List<String> cmd = new ArrayList<>();
for (String commandSegment : origCmd) {
if (commandSegment.contains(ExternalParser.INPUT_FILE_TOKEN)) {
+ if (isolated == null) {
+ isolated = metadata == null ?
IsolatedInput.of(tikaInputStream.getPath())
+ : IsolatedInput.of(tikaInputStream.getPath(),
metadata);
+ }
commandSegment =
commandSegment.replace(ExternalParser.INPUT_FILE_TOKEN,
- tikaInputStream.getFile().toString());
+ isolated.getPath().toString());
inputToStdIn = false;
}
if (commandSegment.contains(ExternalParser.OUTPUT_FILE_TOKEN)) {
- tempOutputFile = tmp.createTemporaryFile();
+ try {
+ tempOutputFile = tmp.createTemporaryFile();
+ } catch (IOException | RuntimeException e) {
+ IOUtils.closeQuietly(isolated);
+ throw e;
+ }
commandSegment = commandSegment
.replace(ExternalParser.OUTPUT_FILE_TOKEN,
tempOutputFile.toString());
outputFromStdOut = false;
@@ -400,10 +411,15 @@ public class ExternalEmbedder implements Embedder {
// Execute
Process process;
- if (cmd.toArray().length == 1) {
- process = Runtime.getRuntime().exec(cmd.toArray(String[]::new)[0]);
- } else {
- process = Runtime.getRuntime().exec(cmd.toArray(String[]::new));
+ try {
+ if (cmd.toArray().length == 1) {
+ process =
Runtime.getRuntime().exec(cmd.toArray(String[]::new)[0]);
+ } else {
+ process =
Runtime.getRuntime().exec(cmd.toArray(String[]::new));
+ }
+ } catch (IOException | RuntimeException e) {
+ IOUtils.closeQuietly(isolated);
+ throw e;
}
UnsynchronizedByteArrayOutputStream stdErrOutputStream =
UnsynchronizedByteArrayOutputStream.builder().get();
@@ -449,6 +465,7 @@ public class ExternalEmbedder implements Embedder {
// TemporaryResources.createTemporaryFile)
IOUtils.closeQuietly(tikaInputStream);
}
+ IOUtils.closeQuietly(isolated);
IOUtils.closeQuietly(outputStream);
IOUtils.closeQuietly(stdErrOutputStream);
if (process.exitValue() != 0) {
diff --git a/tika-core/src/main/java/org/apache/tika/io/IsolatedInput.java
b/tika-core/src/main/java/org/apache/tika/io/IsolatedInput.java
new file mode 100644
index 0000000000..3ec841cfd1
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/io/IsolatedInput.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.io;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.regex.Pattern;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.mime.MimeType;
+import org.apache.tika.mime.MimeTypeException;
+import org.apache.tika.mime.MimeTypes;
+
+/**
+ * A document under a fixed name in a directory of its own, for handing to an
external
+ * program or a library that opens files by path.
+ * <p>
+ * The file is a hard link where the file system allows one; else a copy if
the document is
+ * small, a symbolic link if it is large, and a copy if neither link can be
made. Closing
+ * removes the directory and whatever the reader left in it; the source keeps
its content.
+ */
+public final class IsolatedInput implements Closeable {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(IsolatedInput.class);
+ // above this a copy costs more than it is worth; a symbolic link is used
instead
+ private static final long MAX_SMALL_COPY = 64L * 1024 * 1024;
+ private static final Pattern EXTENSION =
Pattern.compile("(\\.[A-Za-z0-9]{1,16})?");
+
+ private final Path dir;
+ private final Path path;
+
+ private IsolatedInput(Path dir, Path path) {
+ this.dir = dir;
+ this.path = path;
+ }
+
+ public static IsolatedInput of(Path source) throws IOException {
+ return of(source, "");
+ }
+
+ /**
+ * The file gets an extension of the media type Tika gave the document,
since some readers
+ * need one: the document's own extension if it is one of that type's,
else the type's
+ * usual one.
+ */
+ public static IsolatedInput of(Path source, Metadata metadata) throws
IOException {
+ return of(source, extensionFor(metadata));
+ }
+
+ static String extensionFor(Metadata metadata) {
+ MediaType type =
MediaType.parse(metadata.get(HttpHeaders.CONTENT_TYPE));
+ if (type == null) {
+ return "";
+ }
+ try {
+ MimeType registered = MimeTypes.getDefaultMimeTypes()
+ .getRegisteredMimeType(type.getBaseType().toString());
+ if (registered == null) {
+ return "";
+ }
+ String own = FilenameUtils.getSuffixFromPath(
+
String.valueOf(metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY)));
+ for (String known : registered.getExtensions()) {
+ if (known.equalsIgnoreCase(own) &&
EXTENSION.matcher(own).matches()) {
+ return own;
+ }
+ }
+ String extension = registered.getExtension();
+ return EXTENSION.matcher(extension).matches() ? extension : "";
+ } catch (MimeTypeException e) {
+ return "";
+ }
+ }
+
+ /**
+ * @param extension appended to the fixed name, for a reader that needs one
+ */
+ public static IsolatedInput of(Path source, String extension) throws
IOException {
+ if (!EXTENSION.matcher(extension).matches()) {
+ throw new IllegalArgumentException("not a file extension: " +
extension);
+ }
+ return of(source, extension, Placement.values());
+ }
+
+ static IsolatedInput of(Path source, String extension, Placement...
placements)
+ throws IOException {
+ Path dir = Files.createTempDirectory("tika-input-");
+ Path target = dir.resolve("input" + extension);
+ try {
+ // the file itself, not a link to it
+ place(source.toRealPath(), target, placements);
+ } catch (IOException | RuntimeException e) {
+ FileUtils.deleteQuietly(dir.toFile());
+ throw e;
+ }
+ return new IsolatedInput(dir, target);
+ }
+
+ /** In the order tried. */
+ enum Placement {
+ HARD_LINK, SMALL_COPY, SYMBOLIC_LINK, COPY
+ }
+
+ private static void place(Path source, Path target, Placement[] placements)
+ throws IOException {
+ IOException last = null;
+ for (Placement placement : placements) {
+ try {
+ switch (placement) {
+ case HARD_LINK:
+ Files.createLink(target, source);
+ return;
+ case SMALL_COPY:
+ if (Files.size(source) > MAX_SMALL_COPY) {
+ continue;
+ }
+ Files.copy(source, target);
+ return;
+ case SYMBOLIC_LINK:
+ Files.createSymbolicLink(target, source);
+ return;
+ default:
+ Files.copy(source, target);
+ return;
+ }
+ } catch (IOException e) {
+ // another file system, a file this user may not link to, no
links here
+ last = e;
+ } catch (UnsupportedOperationException e) {
+ last = new IOException(e);
+ }
+ }
+ throw last != null ? last : new IOException("no way to place " +
source);
+ }
+
+ /**
+ * @return the document, as an absolute path
+ */
+ public Path getPath() {
+ return path;
+ }
+
+ @Override
+ public void close() {
+ try {
+ // does not follow links
+ FileUtils.deleteDirectory(dir.toFile());
+ } catch (IOException e) {
+ LOG.warn("couldn't delete {}", dir, e);
+ }
+ }
+}
diff --git
a/tika-core/src/main/java/org/apache/tika/parser/external/ExternalParser.java
b/tika-core/src/main/java/org/apache/tika/parser/external/ExternalParser.java
index 429258e461..4c63e7d22b 100644
---
a/tika-core/src/main/java/org/apache/tika/parser/external/ExternalParser.java
+++
b/tika-core/src/main/java/org/apache/tika/parser/external/ExternalParser.java
@@ -44,6 +44,7 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.IsolatedInput;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
@@ -284,6 +285,7 @@ public class ExternalParser implements Parser {
boolean hasPatterns = (metadataPatterns != null &&
!metadataPatterns.isEmpty());
File output = null;
+ IsolatedInput input = null;
// Build our command
String[] cmd;
@@ -295,7 +297,12 @@ public class ExternalParser implements Parser {
}
for (int i = 0; i < cmd.length; i++) {
if (cmd[i].contains(INPUT_FILE_TOKEN)) {
- cmd[i] = cmd[i].replace(INPUT_FILE_TOKEN,
stream.getFile().getPath());
+ if (input == null) {
+ // under a fixed name, in a directory of its own
+ input = IsolatedInput.of(stream.getPath(), metadata);
+ tmp.addResource(input);
+ }
+ cmd[i] = cmd[i].replace(INPUT_FILE_TOKEN,
input.getPath().toString());
inputToStdIn = false;
}
if (cmd[i].contains(OUTPUT_FILE_TOKEN)) {
diff --git
a/tika-core/src/main/java/org/apache/tika/parser/external2/ExternalParser.java
b/tika-core/src/main/java/org/apache/tika/parser/external2/ExternalParser.java
index 5dbea57f04..c509c9eee4 100644
---
a/tika-core/src/main/java/org/apache/tika/parser/external2/ExternalParser.java
+++
b/tika-core/src/main/java/org/apache/tika/parser/external2/ExternalParser.java
@@ -42,6 +42,7 @@ import org.apache.tika.config.Param;
import org.apache.tika.config.TikaTaskTimeout;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.IsolatedInput;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.ExternalProcess;
@@ -92,6 +93,8 @@ public class ExternalParser implements Parser, Initializable {
private int maxStdOut = 10000;
+ private String inputFileExtension;
+
@Override
public Set<MediaType> getSupportedTypes(ParseContext context) {
return supportedTypes;
@@ -104,7 +107,12 @@ public class ExternalParser implements Parser,
Initializable {
Path outFile = null;
try (TemporaryResources tmp = new TemporaryResources()) {
TikaInputStream tis = TikaInputStream.get(stream, tmp, metadata);
- Path p = tis.getPath();
+ // under a fixed name, in a directory of its own
+ IsolatedInput input = inputFileExtension == null
+ ? IsolatedInput.of(tis.getPath(), metadata)
+ : IsolatedInput.of(tis.getPath(), inputFileExtension);
+ tmp.addResource(input);
+ Path p = input.getPath();
List<String> thisCommandLine = new ArrayList<>();
Matcher inputMatcher = INPUT_TOKEN_MATCHER.matcher("");
Matcher outputMatcher = OUTPUT_TOKEN_MATCHER.matcher("");
@@ -279,6 +287,25 @@ public class ExternalParser implements Parser,
Initializable {
return outputParser;
}
+ public String getInputFileExtension() {
+ return inputFileExtension;
+ }
+
+ /**
+ * The program is given the document as a file named {@code input} in a
directory of its
+ * own. By default the file keeps the document's extension if that is one
of the detected
+ * media type's, and otherwise gets that type's usual extension. Set this
to use one fixed
+ * extension instead, such as {@code ".wav"}, or the empty string for none.
+ */
+ @Field
+ public void setInputFileExtension(String inputFileExtension) {
+ if (inputFileExtension != null &&
!inputFileExtension.matches("(\\.[A-Za-z0-9]{1,16})?")) {
+ throw new IllegalArgumentException(
+ "inputFileExtension must be empty or a dot and letters or
digits");
+ }
+ this.inputFileExtension = inputFileExtension;
+ }
+
@Override
public void initialize(Map<String, Param> params) throws
TikaConfigException {
//no-op
diff --git
a/tika-core/src/test/java/org/apache/tika/embedder/ExternalEmbedderInputTest.java
b/tika-core/src/test/java/org/apache/tika/embedder/ExternalEmbedderInputTest.java
new file mode 100644
index 0000000000..9881b21f65
--- /dev/null
+++
b/tika-core/src/test/java/org/apache/tika/embedder/ExternalEmbedderInputTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.embedder;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.io.InputFileProbe;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.external.ExternalParser;
+
+public class ExternalEmbedderInputTest {
+
+ @Test
+ public void testInputIsGivenAsInputInItsOwnDirectory(@TempDir Path dir)
throws Exception {
+ Path doc = Files.write(dir.resolve("report.txt"),
"hello".getBytes(UTF_8));
+ ExternalEmbedder embedder = new ExternalEmbedder();
+
embedder.setCommand(InputFileProbe.command(ExternalParser.INPUT_FILE_TOKEN));
+ embedder.setMetadataCommandArguments(null);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (TikaInputStream tis = TikaInputStream.get(doc)) {
+ embedder.embed(new Metadata(), tis, out, new ParseContext());
+ }
+ Path given = InputFileProbe.arg(out.toString(UTF_8));
+ assertEquals("input",
given.getFileName().toString().replaceFirst("\\..*", ""));
+ assertFalse(given.startsWith(dir));
+ assertFalse(Files.exists(given.getParent()));
+ }
+}
diff --git a/tika-core/src/test/java/org/apache/tika/io/InputFileProbe.java
b/tika-core/src/test/java/org/apache/tika/io/InputFileProbe.java
new file mode 100644
index 0000000000..56d7fd04cc
--- /dev/null
+++ b/tika-core/src/test/java/org/apache/tika/io/InputFileProbe.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.io;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+/**
+ * Stands in for an external program in tests: run in a JVM of its own, it
reports the file
+ * it was given. No shell, so it behaves the same on every OS.
+ */
+public class InputFileProbe {
+
+ private static final String ARG = "arg=";
+ private static final String ENTRY = "entry=";
+ private static final String CONTENT = "content=";
+
+ public static void main(String[] args) throws IOException {
+ Path input = Paths.get(args[0]);
+ StringBuilder sb = new StringBuilder();
+ sb.append(ARG).append(args[0]).append('\n');
+ try (Stream<Path> entries = Files.list(input.getParent())) {
+ entries.map(p -> p.getFileName().toString()).sorted()
+ .forEach(name ->
sb.append(ENTRY).append(name).append('\n'));
+ }
+ sb.append(CONTENT).append(new String(Files.readAllBytes(input),
UTF_8)).append('\n');
+ System.out.write(sb.toString().getBytes(UTF_8));
+ System.out.flush();
+ }
+
+ /** A command line that runs this class on {@code inputToken}. */
+ public static String[] command(String inputToken) throws
URISyntaxException {
+ Path classes =
Paths.get(InputFileProbe.class.getProtectionDomain().getCodeSource()
+ .getLocation().toURI());
+ return new String[]{
+ Paths.get(System.getProperty("java.home"), "bin",
"java").toString(),
+ "-cp", classes.toString(), InputFileProbe.class.getName(),
inputToken};
+ }
+
+ /** The path the program was given. */
+ public static Path arg(String stdout) {
+ return Paths.get(values(stdout, ARG).get(0));
+ }
+
+ /** The names in the directory that holds the given path. */
+ public static List<String> entries(String stdout) {
+ return values(stdout, ENTRY);
+ }
+
+ public static String content(String stdout) {
+ return values(stdout, CONTENT).get(0);
+ }
+
+ private static List<String> values(String stdout, String prefix) {
+ List<String> values = new ArrayList<>();
+ for (String line : stdout.split("\r?\n")) {
+ if (line.startsWith(prefix)) {
+ values.add(line.substring(prefix.length()));
+ }
+ }
+ return values;
+ }
+}
diff --git a/tika-core/src/test/java/org/apache/tika/io/IsolatedInputTest.java
b/tika-core/src/test/java/org/apache/tika/io/IsolatedInputTest.java
new file mode 100644
index 0000000000..72ff4fc4f8
--- /dev/null
+++ b/tika-core/src/test/java/org/apache/tika/io/IsolatedInputTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.io;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+
+public class IsolatedInputTest {
+
+ @Test
+ public void testFixedNameInItsOwnDirectory(@TempDir Path tmp) throws
Exception {
+ Path source = tmp.resolve("document.dat");
+ Files.write(source, "content".getBytes(UTF_8));
+ Files.write(tmp.resolve("other.dat"), "other".getBytes(UTF_8));
+ Path dir;
+ try (IsolatedInput input = IsolatedInput.of(source)) {
+ Path path = input.getPath();
+ dir = path.getParent();
+ assertTrue(path.isAbsolute());
+ assertEquals("input", path.getFileName().toString());
+ assertNotEquals(tmp, dir);
+ assertEquals(List.of("input"), names(dir));
+ assertArrayEquals("content".getBytes(UTF_8),
Files.readAllBytes(path));
+ // what a reader leaves behind goes with the directory
+ Files.write(dir.resolve("input.idx"), new byte[]{1});
+ }
+ assertFalse(Files.exists(dir));
+ assertArrayEquals("content".getBytes(UTF_8),
Files.readAllBytes(source));
+ assertEquals(List.of("document.dat", "other.dat"), names(tmp));
+ }
+
+ @Test
+ public void testExtensionOfTheMediaType(@TempDir Path tmp) throws
Exception {
+ Path source = tmp.resolve("document.dat");
+ Files.write(source, "content".getBytes(UTF_8));
+ Metadata metadata = new Metadata();
+ metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "document.dat");
+ try (IsolatedInput input = IsolatedInput.of(source, metadata)) {
+ assertEquals("input", input.getPath().getFileName().toString());
+ }
+ metadata.set(HttpHeaders.CONTENT_TYPE, "application/pdf; charset=x");
+ try (IsolatedInput input = IsolatedInput.of(source, metadata)) {
+ assertEquals("input.pdf",
input.getPath().getFileName().toString());
+ }
+ // the document's own spelling is kept when it is one of the type's
extensions
+ metadata.set(HttpHeaders.CONTENT_TYPE, "image/jpeg");
+ metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "photo.JPEG");
+ try (IsolatedInput input = IsolatedInput.of(source, metadata)) {
+ assertEquals("input.JPEG",
input.getPath().getFileName().toString());
+ }
+ // otherwise the type's usual one
+ metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "photo.txt");
+ try (IsolatedInput input = IsolatedInput.of(source, metadata)) {
+ assertEquals("input.jpg",
input.getPath().getFileName().toString());
+ }
+ metadata.set(HttpHeaders.CONTENT_TYPE, "application/x-not-registered");
+ try (IsolatedInput input = IsolatedInput.of(source, metadata)) {
+ assertEquals("input", input.getPath().getFileName().toString());
+ }
+ }
+
+ @Test
+ public void testEveryPlacementReadsTheSameAndLeavesTheSource(@TempDir Path
tmp)
+ throws Exception {
+ Path source = tmp.resolve("document.dat");
+ Files.write(source, "content".getBytes(UTF_8));
+ for (IsolatedInput.Placement placement :
IsolatedInput.Placement.values()) {
+ Path dir;
+ try (IsolatedInput input = IsolatedInput.of(source, ".dat",
placement)) {
+ dir = input.getPath().getParent();
+ assertEquals(List.of("input.dat"), names(dir),
placement.toString());
+ assertArrayEquals("content".getBytes(UTF_8),
Files.readAllBytes(input.getPath()));
+ assertEquals(placement ==
IsolatedInput.Placement.SYMBOLIC_LINK,
+ Files.isSymbolicLink(input.getPath()),
placement.toString());
+ } catch (IOException e) {
+ // not available on this file system
+ continue;
+ }
+ assertFalse(Files.exists(dir), placement.toString());
+ assertArrayEquals("content".getBytes(UTF_8),
Files.readAllBytes(source));
+ }
+ }
+
+ @Test
+ public void testSourceThatIsARelativeLink(@TempDir Path tmp) throws
Exception {
+ Files.write(tmp.resolve("document.dat"), "content".getBytes(UTF_8));
+ Path link = tmp.resolve("link.dat");
+ try {
+ Files.createSymbolicLink(link, Paths.get("document.dat"));
+ } catch (IOException | UnsupportedOperationException e) {
+ assumeTrue(false, "no symbolic links here: " + e);
+ }
+ try (IsolatedInput input = IsolatedInput.of(link)) {
+ assertArrayEquals("content".getBytes(UTF_8),
Files.readAllBytes(input.getPath()));
+ }
+ }
+
+ private static List<String> names(Path dir) throws Exception {
+ try (Stream<Path> s = Files.list(dir)) {
+ return s.map(p ->
p.getFileName().toString()).sorted().collect(Collectors.toList());
+ }
+ }
+}
diff --git
a/tika-core/src/test/java/org/apache/tika/parser/external2/ExternalParserInputTest.java
b/tika-core/src/test/java/org/apache/tika/parser/external2/ExternalParserInputTest.java
new file mode 100644
index 0000000000..3c421ebb4e
--- /dev/null
+++
b/tika-core/src/test/java/org/apache/tika/parser/external2/ExternalParserInputTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.external2;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.io.InputFileProbe;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.sax.BodyContentHandler;
+
+public class ExternalParserInputTest {
+
+ /** The program is given the document under a fixed name in a directory of
its own. */
+ @Test
+ public void testInputFileNameAndDirectory(@TempDir Path tmp)
+ throws Exception {
+ Path document = tmp.resolve("document.bin");
+ Files.write(document, "content".getBytes(UTF_8));
+ Files.write(tmp.resolve("other.bin"), "other".getBytes(UTF_8));
+
+ ExternalParser parser = new ExternalParser();
+ parser.setSupportedTypes(List.of("audio/vnd.wave"));
+
parser.setCommandLine(List.of(InputFileProbe.command(ExternalParser.INPUT_FILE_TOKEN)));
+ Metadata metadata = new Metadata();
+ metadata.set(HttpHeaders.CONTENT_TYPE, "audio/vnd.wave");
+ BodyContentHandler handler = new BodyContentHandler();
+ try (TikaInputStream tis = TikaInputStream.get(document)) {
+ parser.parse(tis, handler, metadata, new ParseContext());
+ }
+ String stdout = handler.toString();
+ Path given = InputFileProbe.arg(stdout);
+ assertEquals("input.wav", given.getFileName().toString(), stdout);
+ assertFalse(given.startsWith(tmp), stdout);
+ assertEquals(List.of("input.wav"), InputFileProbe.entries(stdout),
stdout);
+ assertEquals("content", InputFileProbe.content(stdout), stdout);
+ }
+
+ @Test
+ public void testInputFileExtension() {
+ ExternalParser parser = new ExternalParser();
+ assertNull(parser.getInputFileExtension());
+ parser.setInputFileExtension(".wav");
+ assertEquals(".wav", parser.getInputFileExtension());
+ parser.setInputFileExtension("");
+ assertEquals("", parser.getInputFileExtension());
+ assertThrows(IllegalArgumentException.class,
+ () -> parser.setInputFileExtension("a.b"));
+ assertThrows(IllegalArgumentException.class, () ->
parser.setInputFileExtension("wav"));
+ }
+}
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/gdal/GDALParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/gdal/GDALParser.java
index f784411f04..cd40668391 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/gdal/GDALParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/gdal/GDALParser.java
@@ -24,13 +24,14 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.io.StringReader;
-import java.util.Arrays;
-import java.util.Collections;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.Set;
+import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -41,7 +42,9 @@ import org.xml.sax.SAXException;
import org.apache.tika.config.Field;
import org.apache.tika.config.TikaTaskTimeout;
+import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.IsolatedInput;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.ExternalProcess;
@@ -80,67 +83,36 @@ public class GDALParser implements Parser {
public static final long DEFAULT_TIMEOUT_MS = 60000;
- private static final Set<MediaType> SUPPORTED_TYPES =
Collections.unmodifiableSet(new HashSet<>(
- Arrays.asList(MediaType.application("x-netcdf"),
MediaType.application("vrt"),
- MediaType.image("geotiff"), MediaType.image("nitf"),
- MediaType.application("x-rpf-toc"),
MediaType.application("x-ecrg-toc"),
- MediaType.image("hfa"), MediaType.image("sar-ceos"),
MediaType.image("ceos"),
- MediaType.application("jaxa-pal-sar"),
MediaType.application("gff"),
- MediaType.application("elas"),
MediaType.application("aig"),
- MediaType.application("aaigrid"),
MediaType.application("grass-ascii-grid"),
- MediaType.application("sdts-raster"),
MediaType.application("dted"),
- MediaType.image("png"), MediaType.image("jpeg"),
MediaType.image("raster"),
- MediaType.application("jdem"), MediaType.image("gif"),
- MediaType.image("big-gif"), MediaType.image("envisat"),
MediaType.image("fits"),
- MediaType.application("fits"), MediaType.image("bsb"),
- MediaType.application("xpm"), MediaType.image("bmp"),
- MediaType.image("x-dimap"), MediaType.image("x-airsar"),
- MediaType.application("x-rs2"),
MediaType.application("x-pcidsk"),
- MediaType.application("pcisdk"),
MediaType.image("x-pcraster"),
- MediaType.image("ilwis"), MediaType.image("sgi"),
- MediaType.application("x-srtmhgt"),
MediaType.application("leveller"),
- MediaType.application("terragen"),
MediaType.application("x-gmt"),
- MediaType.application("x-isis3"),
MediaType.application("x-isis2"),
- MediaType.application("x-pds"),
MediaType.application("x-til"),
- MediaType.application("x-ers"),
MediaType.application("x-l1b"),
- MediaType.image("fit"), MediaType.application("x-grib"),
MediaType.image("jp2"),
- MediaType.application("x-rmf"),
MediaType.application("x-wcs"),
- MediaType.application("x-wms"),
MediaType.application("x-msgn"),
- MediaType.application("x-wms"),
MediaType.application("x-wms"),
- MediaType.application("x-rst"),
MediaType.application("x-ingr"),
- MediaType.application("x-gsag"),
MediaType.application("x-gsbg"),
- MediaType.application("x-gs7bg"),
MediaType.application("x-cosar"),
- MediaType.application("x-tsx"),
MediaType.application("x-coasp"),
- MediaType.application("x-r"),
MediaType.application("x-map"),
- MediaType.application("x-pnm"),
MediaType.application("x-doq1"),
- MediaType.application("x-doq2"),
MediaType.application("x-envi"),
- MediaType.application("x-envi-hdr"),
MediaType.application("x-generic-bin"),
- MediaType.application("x-p-aux"), MediaType.image("x-mff"),
- MediaType.image("x-mff2"), MediaType.image("x-fujibas"),
- MediaType.application("x-gsc"),
MediaType.application("x-fast"),
- MediaType.application("x-bt"),
MediaType.application("x-lan"),
- MediaType.application("x-cpg"), MediaType.image("ida"),
- MediaType.application("x-ndf"), MediaType.image("eir"),
- MediaType.application("x-dipex"),
MediaType.application("x-lcp"),
- MediaType.application("x-gtx"),
MediaType.application("x-los-las"),
- MediaType.application("x-ntv2"),
MediaType.application("x-ctable2"),
- MediaType.application("x-ace2"),
MediaType.application("x-snodas"),
- MediaType.application("x-kro"), MediaType.image("arg"),
- MediaType.application("x-rik"),
MediaType.application("x-usgs-dem"),
- MediaType.application("x-gxf"),
MediaType.application("x-dods"),
- MediaType.application("x-http"),
MediaType.application("x-bag"),
- MediaType.application("x-hdf"),
MediaType.image("x-hdf5-image"),
- MediaType.application("x-nwt-grd"),
MediaType.application("x-nwt-grc"),
- MediaType.image("adrg"), MediaType.image("x-srp"),
- MediaType.application("x-blx"),
MediaType.application("x-rasterlite"),
- MediaType.application("x-epsilon"),
MediaType.application("x-sdat"),
- MediaType.application("x-kml"),
MediaType.application("x-xyz"),
- MediaType.application("x-geo-pdf"),
MediaType.image("x-ozi"),
- MediaType.application("x-ctg"),
MediaType.application("x-e00-grid"),
- MediaType.application("x-zmap"),
MediaType.application("x-webp"),
- MediaType.application("x-ngs-geoid"),
MediaType.application("x-mbtiles"),
- MediaType.application("x-ppi"),
MediaType.application("x-cappi"))));
-
+ /**
+ * The GDAL drivers gdalinfo may open a file with, unless configured
otherwise.
+ */
+ public static final List<String> DEFAULT_ALLOWED_FORMATS =
+ List.of("GTiff", "netCDF", "GRIB", "FITS", "NITF", "DTED");
+
+ // the types this parser takes for each driver; a driver not listed here
adds none
+ private static final Map<String, Set<MediaType>> FORMAT_TYPES =
Map.ofEntries(
+ Map.entry("GTiff", Set.of(MediaType.image("geotiff"))),
+ Map.entry("netCDF", Set.of(MediaType.application("x-netcdf"))),
+ Map.entry("GRIB", Set.of(MediaType.application("x-grib"))),
+ Map.entry("FITS", Set.of(MediaType.image("fits"),
MediaType.application("fits"))),
+ Map.entry("NITF", Set.of(MediaType.image("nitf"))),
+ Map.entry("HFA", Set.of(MediaType.image("hfa"))),
+ Map.entry("DTED", Set.of(MediaType.application("dted"))),
+ Map.entry("HDF4", Set.of(MediaType.application("x-hdf"))),
+ Map.entry("HDF5", Set.of(MediaType.application("x-hdf"),
+ MediaType.image("x-hdf5-image"))),
+ Map.entry("ENVI", Set.of(MediaType.application("x-envi"),
+ MediaType.application("x-envi-hdr"))),
+ Map.entry("PNG", Set.of(MediaType.image("png"))),
+ Map.entry("JPEG", Set.of(MediaType.image("jpeg"))),
+ Map.entry("GIF", Set.of(MediaType.image("gif"))),
+ Map.entry("BMP", Set.of(MediaType.image("bmp"))),
+ Map.entry("WMS", Set.of(MediaType.application("x-wms"))),
+ Map.entry("WCS", Set.of(MediaType.application("x-wcs"))));
+
+ private static final Pattern FORMAT_NAME =
Pattern.compile("[A-Za-z0-9_]+");
+
+ private List<String> allowedFormats = DEFAULT_ALLOWED_FORMATS;
private String command;
@@ -176,13 +148,60 @@ public class GDALParser implements Parser {
return pCommand;
}
+ public List<String> getAllowedFormats() {
+ return allowedFormats;
+ }
+
+ /**
+ * GDAL driver names (as <code>gdalinfo --formats</code> prints them) that
gdalinfo may
+ * open a file with. This replaces {@link #DEFAULT_ALLOWED_FORMATS}, it
does not add to
+ * it.
+ */
+ @Field
+ public void setAllowedFormats(List<String> allowedFormats) throws
TikaConfigException {
+ if (allowedFormats == null || allowedFormats.isEmpty()) {
+ throw new TikaConfigException("allowedFormats must name at least
one GDAL driver");
+ }
+ for (String format : allowedFormats) {
+ if (!FORMAT_NAME.matcher(format).matches()) {
+ throw new TikaConfigException("allowedFormats: not a GDAL
driver name: "
+ + format);
+ }
+ if (!FORMAT_TYPES.containsKey(format)) {
+ for (String known : FORMAT_TYPES.keySet()) {
+ if (known.equalsIgnoreCase(format)) {
+ throw new TikaConfigException(
+ "allowedFormats: " + format + " should be " +
known);
+ }
+ }
+ LOG.warn("allowedFormats: {} adds no media type; the names
with types are {}",
+ format, new TreeSet<>(FORMAT_TYPES.keySet()));
+ }
+ }
+ this.allowedFormats = List.copyOf(allowedFormats);
+ }
+
// Split before substituting so whitespace in the input path stays inside
one argument.
- static String[] buildCommandLine(String command, String inputPath) {
- String[] args = command.split("\\s+", -1);
- for (int i = 0; i < args.length; i++) {
- args[i] = args[i].replace(INPUT_FILE_TOKEN, inputPath);
+ // Each -if names a driver gdalinfo may try; with any present it tries no
others.
+ static String[] buildCommandLine(String command, String inputPath,
+ List<String> allowedFormats) {
+ String[] split = command.split("\\s+", -1);
+ List<String> args = new ArrayList<>();
+ // before the input argument, or after the program if no argument is
the input alone
+ int formatsAt = 1;
+ for (int i = 0; i < split.length; i++) {
+ if (i > 0 && formatsAt == 1 && split[i].equals(INPUT_FILE_TOKEN)) {
+ formatsAt = i;
+ }
+ args.add(split[i].replace(INPUT_FILE_TOKEN, inputPath));
}
- return args;
+ List<String> formats = new ArrayList<>();
+ for (String format : allowedFormats) {
+ formats.add("-if");
+ formats.add(format);
+ }
+ args.addAll(formatsAt, formats);
+ return args.toArray(new String[0]);
}
// VRT and GTI open any file whose header merely contains their XML, so a
document of a
@@ -195,7 +214,11 @@ public class GDALParser implements Parser {
@Override
public Set<MediaType> getSupportedTypes(ParseContext context) {
- return SUPPORTED_TYPES;
+ Set<MediaType> types = new HashSet<>();
+ for (String format : allowedFormats) {
+ types.addAll(FORMAT_TYPES.getOrDefault(format, Set.of()));
+ }
+ return types;
}
@Override
@@ -206,18 +229,18 @@ public class GDALParser implements Parser {
return;
}
- // first set up and run GDAL
- // process the command
- TemporaryResources tmp = new TemporaryResources();
- TikaInputStream tis = TikaInputStream.get(stream, tmp, metadata);
-
- String[] runCommand = buildCommandLine(this.command,
tis.getFile().getPath());
-
+ FileProcessResult result;
long localTimeoutMillis = TikaTaskTimeout.getTimeoutMillis(context,
timeoutMs);
- ProcessBuilder pb = new ProcessBuilder(runCommand);
- restrictEnvironment(pb.environment());
- FileProcessResult result = ProcessUtils.execute(pb,
- localTimeoutMillis, maxStdOut, maxStdErr);
+ // under a fixed name, in a directory of its own
+ try (TemporaryResources tmp = new TemporaryResources();
+ IsolatedInput input = IsolatedInput.of(
+ TikaInputStream.get(stream, tmp, metadata).getPath(),
metadata)) {
+ String[] runCommand =
+ buildCommandLine(this.command, input.getPath().toString(),
allowedFormats);
+ ProcessBuilder pb = new ProcessBuilder(runCommand);
+ restrictEnvironment(pb.environment());
+ result = ProcessUtils.execute(pb, localTimeoutMillis, maxStdOut,
maxStdErr);
+ }
metadata.set(ExternalProcess.IS_TIMEOUT, result.isTimeout());
metadata.set(ExternalProcess.EXIT_VALUE, result.getExitValue());
@@ -228,6 +251,10 @@ public class GDALParser implements Parser {
metadata.set(ExternalProcess.STD_OUT, result.getStdout());
metadata.set(ExternalProcess.STD_ERR, result.getStderr());
+ if (result.getExitValue() != 0) {
+ throw new TikaException("GDALParser bad exit value " +
result.getExitValue()
+ + " err msg: " + result.getStderr());
+ }
String output = result.getStdout();
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/geoinfo/GeographicInformationParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/geoinfo/GeographicInformationParser.java
index 02712886e2..f043d3b849 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/geoinfo/GeographicInformationParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/geoinfo/GeographicInformationParser.java
@@ -61,6 +61,7 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.IsolatedInput;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
@@ -92,8 +93,10 @@ public class GeographicInformationParser implements Parser {
TemporaryResources tmp =
TikaInputStream.isTikaInputStream(inputStream) ? null : new
TemporaryResources();
- try (TikaInputStream tikaInputStream =
TikaInputStream.get(inputStream, tmp, metadata)) {
- File file = tikaInputStream.getFile();
+ // under a fixed name, in a directory of its own
+ try (TikaInputStream tikaInputStream =
TikaInputStream.get(inputStream, tmp, metadata);
+ IsolatedInput input = IsolatedInput.of(tikaInputStream.getPath(),
".xml")) {
+ File file = input.getPath().toFile();
try (DataStore dataStore = DataStores.open(file)) {
DefaultMetadata defaultMetadata = new
DefaultMetadata(dataStore.getMetadata());
extract(xhtmlContentHandler, metadata, defaultMetadata);
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/grib/GribParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/grib/GribParser.java
index 19228f828f..256a6d23fe 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/grib/GribParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/grib/GribParser.java
@@ -31,7 +31,6 @@ import ucar.nc2.Attribute;
import ucar.nc2.Dimension;
import ucar.nc2.NetcdfFile;
import ucar.nc2.Variable;
-import ucar.nc2.dataset.NetcdfDataset;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
@@ -40,6 +39,7 @@ import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.netcdf.NetcdfReaders;
import org.apache.tika.sax.XHTMLContentHandler;
public class GribParser implements Parser {
@@ -68,7 +68,8 @@ public class GribParser implements Parser {
Path gribFile = Files.createTempFile(tmpDir, "tika-file",
".grib2");
Files.copy(stream, gribFile, StandardCopyOption.REPLACE_EXISTING);
- try (NetcdfFile ncFile =
NetcdfDataset.openFile(gribFile.toString(), null)) {
+ try (NetcdfFile ncFile = NetcdfReaders.open(gribFile,
NetcdfReaders.GRIB1,
+ NetcdfReaders.GRIB2)) {
// first parse out the set of global attributes
for (Attribute attr : ncFile.getGlobalAttributes()) {
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/hdf/HDFParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/hdf/HDFParser.java
index 9d7eefc7fe..5c05c30bae 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/hdf/HDFParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/hdf/HDFParser.java
@@ -33,11 +33,11 @@ import ucar.nc2.NetcdfFile;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.netcdf.NetCDFParser;
+import org.apache.tika.parser.netcdf.NetcdfReaders;
import org.apache.tika.sax.XHTMLContentHandler;
/**
@@ -81,12 +81,9 @@ public class HDFParser implements Parser {
UnsynchronizedByteArrayOutputStream os =
UnsynchronizedByteArrayOutputStream.builder().get();
IOUtils.copy(stream, os);
- String name = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
- if (name == null) {
- name = "";
- }
try {
- NetcdfFile ncFile = NetcdfFile.openInMemory(name,
os.toByteArray());
+ NetcdfFile ncFile = NetcdfReaders.openInMemory(os.toByteArray(),
+ NetcdfReaders.HDF5, NetcdfReaders.HDF4);
unravelStringMet(ncFile, null, metadata);
} catch (IOException e) {
throw new TikaException("HDF parse error", e);
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/isatab/ISArchiveParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/isatab/ISArchiveParser.java
index 810ac89767..49ba23df92 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/isatab/ISArchiveParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/isatab/ISArchiveParser.java
@@ -26,6 +26,7 @@ import java.util.Set;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
+import org.apache.tika.config.Field;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.FilenameUtils;
import org.apache.tika.io.TemporaryResources;
@@ -47,6 +48,8 @@ public class ISArchiveParser implements Parser {
Collections.singleton(MediaType.application("x-isatab"));
private final String location;
+ private boolean readSiblingFiles;
+
/**
* Default constructor.
*/
@@ -68,6 +71,21 @@ public class ISArchiveParser implements Parser {
location += File.separator;
}
this.location = location;
+ // an explicit directory is the operator's choice to read it
+ this.readSiblingFiles = location != null;
+ }
+
+ public boolean isReadSiblingFiles() {
+ return readSiblingFiles;
+ }
+
+ /**
+ * An ISA-Tab archive is a directory: the investigation file beside a
study names the
+ * assay files to read. Off by default, so only the study itself is parsed.
+ */
+ @Field
+ public void setReadSiblingFiles(boolean readSiblingFiles) {
+ this.readSiblingFiles = readSiblingFiles;
}
@Override
@@ -91,15 +109,19 @@ public class ISArchiveParser implements Parser {
}
String studyFileName = tis.getFile().getName();
- File locationFile = new File(location);
- String[] investigationList = locationFile.list((dir, name) ->
name.matches("i_.+\\.txt"));
-
XHTMLContentHandler xhtml = new XHTMLContentHandler(handler,
metadata);
xhtml.startDocument();
- parseInvestigation(location, studyFileName, investigationList,
xhtml, metadata, context);
+ if (readSiblingFiles) {
+ String[] investigationList =
+ new File(location).list((dir, name) ->
name.matches("i_.+\\.txt"));
+ parseInvestigation(location, studyFileName, investigationList,
xhtml, metadata,
+ context);
+ }
parseStudy(stream, xhtml, metadata, context);
- parseAssay(location, xhtml, metadata, context);
+ if (readSiblingFiles) {
+ parseAssay(location, xhtml, metadata, context);
+ }
xhtml.endDocument();
} finally {
@@ -131,7 +153,8 @@ public class ISArchiveParser implements Parser {
private void parseStudy(InputStream stream, XHTMLContentHandler xhtml,
Metadata metadata,
ParseContext context) throws IOException,
SAXException, TikaException {
- xhtml.element("h2", "STUDY " + metadata.get("Study Identifier"));
+ String id = metadata.get("Study Identifier");
+ xhtml.element("h2", id == null ? "STUDY" : "STUDY " + id);
ISATabUtils.parseStudy(stream, xhtml, metadata, context);
}
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetCDFParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetCDFParser.java
index 9f4ef020c6..019b7917cb 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetCDFParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetCDFParser.java
@@ -32,6 +32,7 @@ import ucar.nc2.NetcdfFile;
import ucar.nc2.Variable;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.IsolatedInput;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
@@ -83,7 +84,8 @@ public class NetCDFParser implements Parser {
TemporaryResources tmp =
TikaInputStream.isTikaInputStream(stream) ? null : new
TemporaryResources();
TikaInputStream tis = TikaInputStream.get(stream, tmp, metadata);
- try (NetcdfFile ncFile =
NetcdfFile.open(tis.getFile().getAbsolutePath())) {
+ try (IsolatedInput input = IsolatedInput.of(tis.getPath());
+ NetcdfFile ncFile = NetcdfReaders.open(input.getPath(),
NetcdfReaders.NETCDF3)) {
metadata.set("File-Type-Description",
ncFile.getFileTypeDescription());
// first parse out the set of global attributes
for (Attribute attr : ncFile.getGlobalAttributes()) {
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetcdfReaders.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetcdfReaders.java
new file mode 100644
index 0000000000..93d47ac8b1
--- /dev/null
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetcdfReaders.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.netcdf;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+
+import org.apache.commons.io.IOUtils;
+import ucar.nc2.NetcdfFile;
+
+import org.apache.tika.exception.TikaException;
+
+/**
+ * Opens a file with the netcdf-java reader for its format, chosen from the
file's signature.
+ * A parser passes the readers for the formats it reads; any other file is
refused.
+ */
+public final class NetcdfReaders {
+
+ public static final String NETCDF3 = "ucar.nc2.iosp.netcdf3.N3raf";
+ public static final String HDF5 = "ucar.nc2.iosp.hdf5.H5iosp";
+ public static final String HDF4 = "ucar.nc2.iosp.hdf4.H4iosp";
+ public static final String GRIB1 = "ucar.nc2.grib.collection.Grib1Iosp";
+ public static final String GRIB2 = "ucar.nc2.grib.collection.Grib2Iosp";
+
+ private static final int HEAD_LENGTH = 8;
+ private static final int BUFFER_SIZE = 8092;
+
+ private NetcdfReaders() {
+ }
+
+ public static NetcdfFile open(Path file, String... allowed)
+ throws IOException, TikaException {
+ byte[] head = new byte[HEAD_LENGTH];
+ int n;
+ try (InputStream is = Files.newInputStream(file)) {
+ n = IOUtils.read(is, head);
+ }
+ String reader = allowedReader(head, n, allowed);
+ try {
+ return NetcdfFile.open(file.toAbsolutePath().toString(), reader,
BUFFER_SIZE, null,
+ null);
+ } catch (ReflectiveOperationException e) {
+ throw new TikaException("netcdf-java reader is not available: " +
reader, e);
+ } catch (RuntimeException e) {
+ throw new TikaException("netcdf-java could not read the file", e);
+ }
+ }
+
+ public static NetcdfFile openInMemory(byte[] data, String... allowed)
+ throws IOException, TikaException {
+ String reader = allowedReader(data, data.length, allowed);
+ try {
+ // a fixed name
+ return NetcdfFile.openInMemory("input", data, reader);
+ } catch (ReflectiveOperationException e) {
+ throw new TikaException("netcdf-java reader is not available: " +
reader, e);
+ } catch (RuntimeException e) {
+ throw new TikaException("netcdf-java could not read the file", e);
+ }
+ }
+
+ private static String allowedReader(byte[] head, int length, String[]
allowed)
+ throws TikaException {
+ String reader = forSignature(head, length);
+ if (reader == null || !Arrays.asList(allowed).contains(reader)) {
+ throw new TikaException("not a format this parser reads");
+ }
+ return reader;
+ }
+
+ /**
+ * @return the reader for the signature at the start of the file, or null
if none
+ */
+ static String forSignature(byte[] head, int length) {
+ if (length < HEAD_LENGTH) {
+ return null;
+ }
+ if (head[0] == 'C' && head[1] == 'D' && head[2] == 'F' && (head[3] ==
1 || head[3] == 2)) {
+ return NETCDF3;
+ }
+ if ((head[0] & 0xff) == 0x89 && head[1] == 'H' && head[2] == 'D' &&
head[3] == 'F'
+ && head[4] == '\r' && head[5] == '\n' && head[6] == 0x1a &&
head[7] == '\n') {
+ return HDF5;
+ }
+ if (head[0] == 0x0e && head[1] == 0x03 && head[2] == 0x13 && head[3]
== 0x01) {
+ return HDF4;
+ }
+ if (head[0] == 'G' && head[1] == 'R' && head[2] == 'I' && head[3] ==
'B') {
+ if (head[7] == 1) {
+ return GRIB1;
+ }
+ if (head[7] == 2) {
+ return GRIB2;
+ }
+ }
+ return null;
+ }
+}
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/gdal/TestGDALParser.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/gdal/TestGDALParser.java
index 80a3d4ee27..3df7ca147e 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/gdal/TestGDALParser.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/gdal/TestGDALParser.java
@@ -18,7 +18,9 @@ package org.apache.tika.parser.gdal;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
@@ -30,15 +32,20 @@ import java.net.ServerSocket;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
+import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.Test;
import org.apache.tika.TikaTest;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.external.ExternalParser;
import org.apache.tika.sax.BodyContentHandler;
@@ -186,8 +193,21 @@ public class TestGDALParser extends TikaTest {
@Test
public void testInputPathWithWhitespaceIsOneArgument() {
String path = "/data/survey 2024/tile.tif --config GDAL_HTTP_PROXY
attacker:8080";
- assertArrayEquals(new String[]{"gdalinfo", path},
- GDALParser.buildCommandLine(new GDALParser().getCommand(),
path));
+ assertArrayEquals(new String[]{"gdalinfo", "-if", "GTiff", "-if",
"FITS", path},
+ GDALParser.buildCommandLine(new GDALParser().getCommand(),
path,
+ List.of("GTiff", "FITS")));
+ }
+
+ @Test
+ public void testFormatsPrecedeTheInputInAWrappedCommand() {
+ assertArrayEquals(new String[]{"docker", "run", "img", "gdalinfo",
"-if", "GTiff", "/in.tif",
+ "-json"},
+ GDALParser.buildCommandLine("docker run img gdalinfo ${INPUT}
-json", "/in.tif",
+ List.of("GTiff")));
+ // not before an argument that only contains the input
+ assertArrayEquals(new String[]{"docker", "-if", "GTiff", "run", "-v",
"/in.tif:/in", "img"},
+ GDALParser.buildCommandLine("docker run -v ${INPUT}:/in img",
"/in.tif",
+ List.of("GTiff")));
}
@Test
@@ -224,6 +244,8 @@ public class TestGDALParser extends TikaTest {
document.getBytes(StandardCharsets.US_ASCII))) {
new GDALParser().parse(tis, new BodyContentHandler(), new
Metadata(),
new ParseContext());
+ } catch (TikaException e) {
+ // gdalinfo may refuse the file; what matters is that it
opened no URL
}
}
assertEquals(0, connections.get());
@@ -237,4 +259,28 @@ public class TestGDALParser extends TikaTest {
assertEquals("JPEG VRT GTI", env.get("GDAL_SKIP"));
assertEquals("", env.get("CPL_VSIL_CURL_ALLOWED_FILENAME"));
}
+
+ @Test
+ public void testDefaultFormatsAreASmallSet() {
+ GDALParser parser = new GDALParser();
+ assertEquals(GDALParser.DEFAULT_ALLOWED_FORMATS,
parser.getAllowedFormats());
+ Set<MediaType> types = parser.getSupportedTypes(new ParseContext());
+ assertTrue(types.contains(MediaType.application("x-netcdf")));
+ assertTrue(types.contains(MediaType.image("fits")));
+ // not in the default formats
+ assertFalse(types.contains(MediaType.image("png")));
+ }
+
+ @Test
+ public void testAllowedFormatsReplacesTheDefault() throws Exception {
+ GDALParser parser = new GDALParser();
+ parser.setAllowedFormats(List.of("PNG"));
+ assertEquals(Set.of(MediaType.image("png")),
parser.getSupportedTypes(new ParseContext()));
+ assertThrows(TikaConfigException.class, () ->
parser.setAllowedFormats(List.of()));
+ assertThrows(TikaConfigException.class,
+ () -> parser.setAllowedFormats(List.of("GTiff HFA")));
+ assertThrows(TikaConfigException.class, () ->
parser.setAllowedFormats(List.of("gtiff")));
+ // a driver with no media type here is allowed
+ parser.setAllowedFormats(List.of("GTiff", "AAIGrid"));
+ }
}
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/isatab/ISArchiveParserTest.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/isatab/ISArchiveParserTest.java
index d890c13c3b..dc50a1765e 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/isatab/ISArchiveParserTest.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/isatab/ISArchiveParserTest.java
@@ -19,6 +19,7 @@ package org.apache.tika.parser.isatab;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.io.InputStream;
@@ -109,7 +110,8 @@ public class ISArchiveParserTest {
@Test
public void testDirectoryNotCachedAcrossParses(@TempDir Path root) throws
Exception {
- Parser parser = new ISArchiveParser();
+ ISArchiveParser parser = new ISArchiveParser();
+ parser.setReadSiblingFiles(true);
assertEquals("FIRST", parseInvestigationId(parser, writeArchive(root,
"first", "FIRST")));
assertEquals("SECOND", parseInvestigationId(parser, writeArchive(root,
"second", "SECOND")));
}
@@ -135,4 +137,32 @@ public class ISArchiveParserTest {
}
return metadata.get("Investigation Identifier");
}
+
+ @Test
+ public void testSiblingFilesAreNotReadByDefault(@TempDir Path root) throws
Exception {
+ Files.write(root.resolve("i_test.txt"),
+ ("STUDY\n"
+ + "Study File Name\t\"s_test.txt\"\n"
+ + "Study Assay File Name\t\"a_test.txt\"\n")
+ .getBytes(StandardCharsets.UTF_8));
+ Files.write(root.resolve("a_test.txt"),
+ "\"ASSAY_CONTENT\"\n".getBytes(StandardCharsets.UTF_8));
+ Path study = root.resolve("s_test.txt");
+ Files.write(study, "\"Source
Name\"\n\"culture1\"\n".getBytes(StandardCharsets.UTF_8));
+
+ ContentHandler handler = new BodyContentHandler();
+ try (TikaInputStream tis = TikaInputStream.get(study)) {
+ new ISArchiveParser().parse(tis, handler, new Metadata(), new
ParseContext());
+ }
+ assertTrue(handler.toString().contains("culture1"),
handler.toString());
+ assertFalse(handler.toString().contains("ASSAY_CONTENT"),
handler.toString());
+
+ ISArchiveParser optedIn = new ISArchiveParser();
+ optedIn.setReadSiblingFiles(true);
+ ContentHandler withSiblings = new BodyContentHandler();
+ try (TikaInputStream tis = TikaInputStream.get(study)) {
+ optedIn.parse(tis, withSiblings, new Metadata(), new
ParseContext());
+ }
+ assertTrue(withSiblings.toString().contains("ASSAY_CONTENT"),
withSiblings.toString());
+ }
}
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/netcdf/NetcdfReadersTest.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/netcdf/NetcdfReadersTest.java
new file mode 100644
index 0000000000..da661f6f50
--- /dev/null
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/test/java/org/apache/tika/parser/netcdf/NetcdfReadersTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.netcdf;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.grib.GribParser;
+import org.apache.tika.parser.hdf.HDFParser;
+import org.apache.tika.sax.BodyContentHandler;
+
+/**
+ * Each parser opens only its own formats.
+ */
+public class NetcdfReadersTest extends TikaTest {
+
+ private static final String NETCDF = "sresa1b_ncar_ccsm3_0_run1_200001.nc";
+ private static final String GRIB = "gdas1.forecmwf.2014062612.grib2";
+
+ @TempDir
+ Path tmp;
+
+ @Test
+ public void testNetCDFParserReadsNetCDF() throws Exception {
+ Path nc = copy(NETCDF, "a.nc");
+ Metadata metadata = new Metadata();
+ parse(new NetCDFParser(), TikaInputStream.get(nc), metadata);
+ assertTrue(metadata.names().length > 5, metadata.toString());
+ assertEquals(List.of("a.nc"), names(tmp));
+ }
+
+ @Test
+ public void testNetCDFParserRefusesAnotherFormat() throws Exception {
+ // a GRIB file
+ Path grib = copy(GRIB, "a.nc");
+ assertRefused(new NetCDFParser(), TikaInputStream.get(grib));
+ assertEquals(List.of("a.nc"), names(tmp));
+ }
+
+ @Test
+ public void testHDFParserRefusesAnotherFormat() throws Exception {
+ assertRefused(new HDFParser(),
TikaInputStream.get(getResourceAsStream("/test-documents/" + NETCDF)));
+ }
+
+ @Test
+ public void testGribParserRefusesAnotherFormat() throws Exception {
+ assertRefused(new GribParser(), TikaInputStream.get(copy(NETCDF,
"a.grib2")));
+ }
+
+ @Test
+ public void testReaderFollowsTheSignature() {
+ assertEquals(NetcdfReaders.NETCDF3,
+ NetcdfReaders.forSignature(new byte[]{'C', 'D', 'F', 1, 0, 0,
0, 0}, 8));
+ assertEquals(NetcdfReaders.GRIB2,
+ NetcdfReaders.forSignature(new byte[]{'G', 'R', 'I', 'B', 0,
0, 0, 2}, 8));
+ assertNull(NetcdfReaders.forSignature(new byte[]{'G', 'R', 'I', 'B',
0, 0, 0, 3}, 8));
+ assertNull(NetcdfReaders.forSignature(new byte[]{'a', 'b', 'c', 'd',
'e', 'f', 'g', 'h'}, 8));
+ assertNull(NetcdfReaders.forSignature(new byte[]{'C', 'D', 'F', 1, 0,
0, 0, 0}, 4));
+ }
+
+ private void assertRefused(Parser parser, TikaInputStream stream) {
+ TikaException e = assertThrows(TikaException.class,
+ () -> parse(parser, stream, new Metadata()));
+ assertEquals("not a format this parser reads", e.getMessage());
+ }
+
+ private void parse(Parser parser, TikaInputStream stream, Metadata
metadata)
+ throws Exception {
+ try (TikaInputStream tis = stream) {
+ parser.parse(tis, new BodyContentHandler(-1), metadata, new
ParseContext());
+ }
+ }
+
+ private Path copy(String resource, String name) throws Exception {
+ Path target = tmp.resolve(name);
+ try (InputStream is = getResourceAsStream("/test-documents/" +
resource)) {
+ Files.copy(is, target);
+ }
+ return target;
+ }
+
+ private static List<String> names(Path dir) throws Exception {
+ try (Stream<Path> s = Files.list(dir)) {
+ return s.map(p ->
p.getFileName().toString()).sorted().collect(Collectors.toList());
+ }
+ }
+}
diff --git
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-package/src/test/java/org/apache/tika/parser/scientific/integration/TestParsers.java
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-package/src/test/java/org/apache/tika/parser/scientific/integration/TestParsers.java
index cb2d2236b0..ce03aa9fd9 100644
---
a/tika-parsers/tika-parsers-extended/tika-parser-scientific-package/src/test/java/org/apache/tika/parser/scientific/integration/TestParsers.java
+++
b/tika-parsers/tika-parsers-extended/tika-parser-scientific-package/src/test/java/org/apache/tika/parser/scientific/integration/TestParsers.java
@@ -17,6 +17,7 @@
package org.apache.tika.parser.scientific.integration;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.BufferedReader;
@@ -24,7 +25,9 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import org.junit.jupiter.api.Test;
@@ -32,8 +35,10 @@ import org.apache.tika.exception.TikaException;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.CompositeParser;
import org.apache.tika.parser.DefaultParser;
+import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.external.CompositeExternalParser;
+import org.apache.tika.parser.gdal.GDALParser;
import org.apache.tika.parser.ocr.TesseractOCRParser;
/**
@@ -48,6 +53,8 @@ import org.apache.tika.parser.ocr.TesseractOCRParser;
*/
public class TestParsers {
+ private static final String GDAL = GDALParser.class.toString();
+
@Test
public void testDiffsFrom241() throws Exception {
@@ -57,6 +64,11 @@ public class TestParsers {
path241 = "/2.4.1-tesseract.txt";
}
+ Set<String> gdalTypes = new HashSet<>();
+ for (MediaType type : new GDALParser().getSupportedTypes(new
ParseContext())) {
+ gdalTypes.add(type.toString());
+ }
+
int checked = 0;
//The initial lists were developed with exiftool installed. We have
since
//modified the 2.4.1-* files to act as if no exiftool is installed.
@@ -77,8 +89,14 @@ public class TestParsers {
if (external != null) {
parserClass = externalParser.getClass().toString();
}
- assertEquals(parserClass, currentDefault.get(mediaType),
- "for mediaType '" + mediaType + "'");
+ if (parserClass.equals(GDAL) &&
!gdalTypes.contains(mediaType)) {
+ // GDAL takes only the types of its allowedFormats; 2.4.1
gave it these too
+ assertNotEquals(GDAL, currentDefault.get(mediaType),
+ "for mediaType '" + mediaType + "'");
+ } else {
+ assertEquals(parserClass, currentDefault.get(mediaType),
+ "for mediaType '" + mediaType + "'");
+ }
checked++;
line = reader.readLine();
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/ImagePreprocessor.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/ImagePreprocessor.java
index 8932af5ada..560519e0c1 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/ImagePreprocessor.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/ImagePreprocessor.java
@@ -71,8 +71,13 @@ class ImagePreprocessor implements Serializable {
final List<String> filter = Arrays.asList("-filter",
config.getFilter());
final List<String> resize = Arrays.asList("-resize",
config.getResize() + "%");
final List<String> rotate = Arrays.asList("-rotate",
Double.toString(-angle));
+ // names the format explicitly
+ String format = TesseractInput.format(sourceFile);
+ if (format == null) {
+ throw new IOException("not an image ImageMagick should be
given");
+ }
final List<String> sourceFileArg =
-
Collections.singletonList(sourceFile.toAbsolutePath().toString());
+ Collections.singletonList(format + ":" +
sourceFile.toAbsolutePath());
final List<String> targFileArg =
Collections.singletonList(targFile.toAbsolutePath().toString());
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractInput.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractInput.java
new file mode 100644
index 0000000000..89d03c8ad3
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractInput.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.ocr;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.apache.commons.io.IOUtils;
+
+/**
+ * The image formats handed to tesseract, recognized by the signature at the
start of the
+ * file.
+ */
+final class TesseractInput {
+
+ // the longest signature below
+ private static final int HEAD_LENGTH = 12;
+
+ private static final byte[][] SIGNATURES = {
+ {(byte) 0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'},
+ {(byte) 0xff, (byte) 0xd8, (byte) 0xff},
+ {'I', 'I', 42, 0},
+ {'M', 'M', 0, 42},
+ {'I', 'I', 43, 0},
+ {'M', 'M', 0, 43},
+ {'B', 'M'},
+ {'G', 'I', 'F', '8', '7', 'a'},
+ {'G', 'I', 'F', '8', '9', 'a'},
+ {0, 0, 0, 0x0c, 'j', 'P', ' ', ' ', '\r', '\n', (byte) 0x87, '\n'}
+ };
+ // ImageMagick's name for each signature above
+ private static final String[] FORMATS = {"png", "jpeg", "tiff", "tiff",
"tiff", "tiff", "bmp", "gif", "gif",
+ "jp2"};
+
+ private TesseractInput() {
+ }
+
+ static boolean isAccepted(Path path) throws IOException {
+ return format(path) != null;
+ }
+
+ /**
+ * @return the image format the file's signature names, or null if it is
not accepted
+ */
+ static String format(Path path) throws IOException {
+ byte[] head = new byte[HEAD_LENGTH];
+ int n;
+ try (InputStream is = Files.newInputStream(path)) {
+ n = IOUtils.read(is, head);
+ }
+ return format(head, n);
+ }
+
+ static String format(byte[] head, int length) {
+ if (length < HEAD_LENGTH) {
+ return null;
+ }
+ for (int i = 0; i < SIGNATURES.length; i++) {
+ if (startsWith(head, SIGNATURES[i])) {
+ return FORMATS[i];
+ }
+ }
+ // PNM: P1 through P6 followed by whitespace
+ if (head[0] == 'P' && head[1] >= '1' && head[1] <= '6' &&
+ (head[2] == ' ' || head[2] == '\t' || head[2] == '\r' ||
head[2] == '\n')) {
+ return "pnm";
+ }
+ return null;
+ }
+
+ private static boolean startsWith(byte[] head, byte[] signature) {
+ for (int i = 0; i < signature.length; i++) {
+ if (head[i] != signature[i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
index 4bb95f5e0e..0a98d1260c 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
@@ -48,6 +48,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.imageio.ImageIO;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
@@ -116,6 +117,7 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser implements
private static final String OCR = "ocr-";
private static final Logger LOG =
LoggerFactory.getLogger(TesseractOCRParser.class);
private static final Object[] LOCK = new Object[0];
+ private static final long STREAM_JOIN_MILLIS = 10000;
private static final long serialVersionUID = -8167538283213097265L;
private static final Set<MediaType> SUPPORTED_TYPES =
Collections.unmodifiableSet(new HashSet<>(
Arrays.asList(
@@ -182,10 +184,10 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser implements
Map<String, String> env = pb.environment();
if (!StringUtils.isBlank(getTessdataPath())) {
- env.put(TESSDATA_PREFIX, getTessdataPath());
+ env.put(TESSDATA_PREFIX, absoluteDir(getTessdataPath()));
} else if (!StringUtils.isBlank(getTesseractPath())) {
//adding tessdata is required for at least >= 4.x
- env.put(TESSDATA_PREFIX, getTesseractPath() + "tessdata");
+ env.put(TESSDATA_PREFIX, absoluteDir(getTesseractPath()) +
"tessdata");
}
}
@@ -336,12 +338,16 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser implements
warnOnFirstParse();
validateLangString(config.getLanguage());
- File tmpTxtOutput = null;
+ String extension = config.getPageSegMode().equals("0") ? "osd" :
+ config.getOutputType().toString().toLowerCase(Locale.US);
+ // Tesseract appends the output type (.txt or .hocr or .osd) to output
file name
+ File tmpTxtOutput = new File(tmpOCROutputFile.getAbsolutePath() + "."
+ extension);
try {
Path input = tikaInputStream.getPath();
long size = tikaInputStream.getLength();
if (size >= config.getMinFileSizeToOcr() && size <=
config.getMaxFileSizeToOcr()) {
+ requireAccepted(input);
// Process image
if (config.isEnableImagePreprocessing() ||
config.isApplyRotation()) {
@@ -366,12 +372,6 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser implements
doOCR(input.toFile(), tmpOCROutputFile, config,
parseContext);
}
- String extension = config.getPageSegMode().equals("0") ? "osd"
:
-
config.getOutputType().toString().toLowerCase(Locale.US);
- // Tesseract appends the output type (.txt or .hocr or .osd)
to output file name
- tmpTxtOutput = new File(tmpOCROutputFile.getAbsolutePath() +
- "." + extension);
-
if (tmpTxtOutput.exists()) {
try (InputStream is = new FileInputStream(tmpTxtOutput)) {
if (config.getPageSegMode().equals("0")) {
@@ -385,9 +385,7 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser implements
}
}
} finally {
- if (tmpTxtOutput != null) {
- tmpTxtOutput.delete();
- }
+ tmpTxtOutput.delete();
}
}
@@ -444,10 +442,23 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser implements
*/
private void doOCR(File input, File output, TesseractOCRConfig config,
ParseContext parseContext)
throws IOException, TikaException {
+ // rechecked here: with preprocessing this is ImageMagick's output
+ requireAccepted(input.toPath());
+ runTesseract(input, output, config, parseContext);
+ }
+
+ private static void requireAccepted(Path input) throws IOException,
TikaException {
+ if (!TesseractInput.isAccepted(input)) {
+ throw new TikaException("TesseractOCRParser input is not an image
tesseract accepts");
+ }
+ }
+
+ private void runTesseract(File input, File output, TesseractOCRConfig
config,
+ ParseContext parseContext) throws IOException,
TikaException {
ArrayList<String> cmd = new ArrayList<>(
- Arrays.asList(getTesseractPath() + getTesseractProg(),
input.getPath(),
- output.getPath(), "--psm", config.getPageSegMode()));
+ Arrays.asList(tesseractCommand(), input.getAbsolutePath(),
+ output.getAbsolutePath(), "--psm",
config.getPageSegMode()));
//if --psm == 0, don't add anything else to the command line
if (! "0".equals(config.getPageSegMode())) {
if (!StringUtils.isBlank(config.getLanguage())) {
@@ -467,6 +478,9 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser implements
ProcessBuilder pb = new ProcessBuilder(cmd);
setEnv(pb);
+ // runs in a directory of its own
+ Path workingDir = Files.createTempDirectory("tika-ocr-");
+ pb.directory(workingDir.toFile());
Process process = null;
String id = null;
@@ -483,9 +497,23 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser implements
if (id != null) {
release(id);
}
+ FileUtils.deleteQuietly(workingDir.toFile());
}
}
+ // absolute: the process does not run in this JVM's working directory
+ private String tesseractCommand() {
+ String dir = getTesseractPath();
+ if (StringUtils.isBlank(dir)) {
+ return getTesseractProg();
+ }
+ return Paths.get(dir, getTesseractProg()).toAbsolutePath().toString();
+ }
+
+ private static String absoluteDir(String dir) {
+ return Paths.get(dir).toAbsolutePath() + File.separator;
+ }
+
private void runOCRProcess(Process process, long timeoutMillis) throws
IOException,
TikaException {
process.getOutputStream().close();
@@ -512,13 +540,17 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser implements
//this _should_ never be thrown
throw new TikaException("TesseractOCRParser timeout");
}
+ try {
+ outThread.join(STREAM_JOIN_MILLIS);
+ errThread.join(STREAM_JOIN_MILLIS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new TikaException("TesseractOCRParser interrupted", e);
+ }
+ if (outThread.isAlive() || errThread.isAlive()) {
+ throw new TikaException("TesseractOCRParser could not read the
process output");
+ }
if (exitValue > 0) {
- try {
- //make sure this thread is actually done
- errThread.join(1000);
- } catch (InterruptedException e) {
- //swallow
- }
throw new TikaException(
"TesseractOCRParser bad exit value " + exitValue + " err
msg: " +
errBuilder.toString());
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractInputTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractInputTest.java
new file mode 100644
index 0000000000..5f5b4719b1
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractInputTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.ocr;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.file.Paths;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.TikaTest;
+
+public class TesseractInputTest extends TikaTest {
+
+ private static final String PAD = "\0\0\0\0\0\0\0\0\0\0\0\0";
+
+ @Test
+ public void acceptsImages() throws Exception {
+ for (String name : new String[]{"testOCR_spacing.png", "testOCR.jpg",
"testTIFF_multipage.tif",
+ "testRotated+10.png"}) {
+ assertTrue(TesseractInput.isAccepted(
+ Paths.get(getResourceAsUri("/test-documents/" + name))));
+ }
+ }
+
+ @Test
+ public void acceptsSignatures() throws Exception {
+ for (String head : new String[]{
+ "\u0089PNG\r\n\u001a\n\0\0\0\rIHDR",
+ "\u00ff\u00d8\u00ff\u00e0\0\u0010JFIF\0\u0001",
+ "II*\0\b\0\0\0\0\0\0\0",
+ "MM\0*\0\0\0\b\0\0\0\0",
+ "II+\0\b\0\0\0\0\0\0\0",
+ "MM\0+\0\b\0\0\0\0\0\0",
+ "BM\u0010\u0010\0\0\0\0\0\0\0\0",
+ "GIF89a\u0010\0\u0010\0\0\0",
+ "P6\n16 16\n255\n",
+ "P1 # ok\n1 1\n1\n\n",
+ "\0\0\0\fjP \r\n\u0087\n"}) {
+ assertTrue(accepted(head), head);
+ }
+ }
+
+ @Test
+ public void rejects() throws Exception {
+ for (String head : new String[]{
+ "plain text\n" + PAD,
+ "short\n" + PAD,
+ "",
+ "BM\n",
+ "\u00ff\u00d8 plain text\n",
+ "GIF88a" + PAD,
+ "P6text text\n",
+ "P7\n" + PAD,
+ "RIFF\0\0\0\0WEBP",
+ "%PDF-1.7\n1 0 obj\n",
+ "%!PS-Adobe-3.0\n",
+ "spix" + PAD}) {
+ assertFalse(accepted(head), head);
+ }
+ }
+
+ private static boolean accepted(String head) {
+ byte[] bytes = head.getBytes(ISO_8859_1);
+ byte[] buffer = new byte[Math.max(bytes.length, 12)];
+ System.arraycopy(bytes, 0, buffer, 0, bytes.length);
+ return TesseractInput.format(buffer, bytes.length) != null;
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
index d33158ad18..6cd7dc5752 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
@@ -18,12 +18,14 @@ package org.apache.tika.parser.ocr;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
import java.io.File;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
@@ -37,6 +39,7 @@ import org.apache.tika.config.TikaConfig;
import org.apache.tika.config.TikaTaskTimeout;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
@@ -51,6 +54,7 @@ import org.apache.tika.parser.image.JpegParser;
import org.apache.tika.parser.image.PSDParser;
import org.apache.tika.parser.image.TiffParser;
import org.apache.tika.parser.image.WebPParser;
+import org.apache.tika.sax.BodyContentHandler;
public class TesseractOCRParserTest extends TikaTest {
@@ -61,6 +65,19 @@ public class TesseractOCRParserTest extends TikaTest {
}
+ @Test
+ public void testNonImageIsRefused() throws Exception {
+ assumeTrue(canRun(), "can run OCR");
+ byte[] notAnImage = "this is not an
image".getBytes(StandardCharsets.UTF_8);
+ try (TikaInputStream tis = TikaInputStream.get(notAnImage)) {
+ TikaException e = assertThrows(TikaException.class,
+ () -> new TesseractOCRParser().parse(tis, new
BodyContentHandler(),
+ new Metadata(), new ParseContext()));
+ assertEquals("TesseractOCRParser input is not an image tesseract
accepts",
+ e.getMessage());
+ }
+ }
+
@Test
public void testInterwordSpacing() throws Exception {
assumeTrue(canRun(), "can run OCR");
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/RarParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/RarParser.java
index 8b77d36cc6..b14254ee50 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/RarParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/RarParser.java
@@ -33,6 +33,7 @@ import org.apache.tika.exception.TikaException;
import org.apache.tika.exception.UnsupportedFormatException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
+import org.apache.tika.io.IsolatedInput;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
@@ -70,9 +71,12 @@ public class RarParser implements Parser {
throw new UnsupportedFormatException("Tika does not yet support
rar version 5.");
}
Archive rar = null;
+ IsolatedInput input = null;
try (TemporaryResources tmp = new TemporaryResources()) {
TikaInputStream tis = TikaInputStream.get(stream, tmp, metadata);
- rar = new Archive(tis.getFile());
+ // under a fixed name, in a directory of its own
+ input = IsolatedInput.of(tis.getPath(), ".rar");
+ rar = new Archive(input.getPath().toFile());
if (rar.isEncrypted()) {
throw new EncryptedDocumentException();
@@ -105,10 +109,15 @@ public class RarParser implements Parser {
} catch (RarException e) {
throw new TikaException("RarParser Exception", e);
} finally {
- if (rar != null) {
- rar.close();
+ try {
+ if (rar != null) {
+ rar.close();
+ }
+ } finally {
+ if (input != null) {
+ input.close();
+ }
}
-
}
xhtml.endDocument();
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/UnrarParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/UnrarParser.java
index 20ee89dd7a..05bfb2b01f 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/UnrarParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/UnrarParser.java
@@ -16,15 +16,18 @@
*/
package org.apache.tika.parser.pkg;
-import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
+import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Collections;
+import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
@@ -109,16 +112,20 @@ public class UnrarParser implements Parser {
xhtml.endDocument();
}
+ // regular files and directories only
private void processDirectory(Path baseDir, Path path,
XHTMLContentHandler xhtml,
EmbeddedDocumentExtractor extractor,
ParseContext context)
throws IOException, SAXException {
- for (File f : path.toFile().listFiles()) {
- if (f.isDirectory()) {
- processDirectory(baseDir, f.toPath(), xhtml, extractor,
- context);
- } else {
- processFile(baseDir, f.toPath(), xhtml, extractor, context);
+ List<Path> children;
+ try (Stream<Path> stream = Files.list(path)) {
+ children = stream.sorted().collect(Collectors.toList());
+ }
+ for (Path child : children) {
+ if (Files.isDirectory(child, LinkOption.NOFOLLOW_LINKS)) {
+ processDirectory(baseDir, child, xhtml, extractor, context);
+ } else if (Files.isRegularFile(child, LinkOption.NOFOLLOW_LINKS)) {
+ processFile(baseDir, child, xhtml, extractor, context);
}
}
}