This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch fix-testforasync-invalidpathexception in repository https://gitbox.apache.org/repos/asf/tika.git
commit 09e3adfd743733bbbadbc5e818734b5b6c0c29e1 Author: tallison <[email protected]> AuthorDate: Mon Aug 17 17:36:13 2026 -0400 TIKA-4808 - remove access to the network parser from cli --- CHANGES.txt | 22 ++- docs/modules/ROOT/pages/using-tika/cli/index.adoc | 12 +- .../src/main/java/org/apache/tika/cli/TikaCLI.java | 20 +-- .../java/org/apache/tika/parser/NetworkParser.java | 188 --------------------- .../org/apache/tika/parser/NetworkParserTest.java | 135 --------------- 5 files changed, 30 insertions(+), 347 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 558f6d9b81..c8f38747dc 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,14 +2,20 @@ Release 4.0.0 - ??? BREAKING CHANGES - * tika-app: the inline short forms -eX (output encoding), -pX (document - password) and -c<uri> (network client) were removed from standard mode. - Use --encoding=X, --password=X and --client=<uri>. These were the only - short flags that consumed an inline value, and matching them by prefix - meant a long name written with one dash was silently swallowed: - -config=tika.json set the network-client URI to "onfig=tika.json" and - loaded no config file, with no error. Every single-dash long name is now - rejected with a message naming the two-dash form (TIKA-4808). + * tika-app: the inline short forms -eX (output encoding) and -pX (document + password) were removed from standard mode. Use --encoding=X and + --password=X. These were the only short flags that consumed an inline + value, and matching them by prefix meant a long name written with one + dash was silently swallowed: -config=tika.json set the password to + "onfig=tika.json" and loaded no config file, with no error. Every + single-dash long name is now rejected with a message naming the + two-dash form (TIKA-4808). + + * tika-core/tika-app: NetworkParser and tika-app's -c/--client=<uri> + network-client mode were removed. It dispatched raw sockets + (including a bare telnet:// scheme) to an arbitrary user-supplied + host with no auth or TLS enforcement; use tika-server instead + (TIKA-4808). * Metadata's reserved tk: (and legacy X-TIKA:) namespace is now a trust boundary for String-keyed writes. A String write to a reserved key throws diff --git a/docs/modules/ROOT/pages/using-tika/cli/index.adoc b/docs/modules/ROOT/pages/using-tika/cli/index.adoc index cef8b5849b..121a1e3e12 100644 --- a/docs/modules/ROOT/pages/using-tika/cli/index.adoc +++ b/docs/modules/ROOT/pages/using-tika/cli/index.adoc @@ -64,10 +64,14 @@ Writing a long name with a single dash is always an error — `-input`, `-config `-encoding=UTF-8` and the like are rejected with a message naming the two-dash form. No short flag takes an inline value, so nothing is silently consumed as a flag's argument. -NOTE: In 4.0.0 the inline short forms `-eX` (encoding), `-pX` (password) and `-c<uri>` -(network client) were removed from standard mode. Use `--encoding=X`, `--password=X` and -`--client=<uri>`. Scripts using the old forms now fail with an unrecognized-option error -rather than silently misparsing. +NOTE: In 4.0.0 the inline short forms `-eX` (encoding) and `-pX` (password) were removed +from standard mode. Use `--encoding=X` and `--password=X`. Scripts using the old forms now +fail with an unrecognized-option error rather than silently misparsing. + +NOTE: In 4.0.0 network-client mode (`-c<uri>` / `--client=<uri>`, backed by +`NetworkParser`) was removed entirely. It dispatched raw sockets -- including a bare +`telnet://` scheme -- to an arbitrary user-supplied host with no authentication or TLS +enforcement. Use tika-server instead. `-T` means different things in the two modes, so read it in context: `--text-main` in standard mode, `--timeoutMillis=<ms>` in pipes mode. `-p` (`--pluginsDir`) and `-c` (`--config`) exist diff --git a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java index c6a31730e7..1089229760 100644 --- a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java +++ b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java @@ -32,7 +32,6 @@ import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.io.Writer; import java.lang.reflect.Field; -import java.net.URI; import java.net.URL; import java.nio.file.Files; import java.nio.file.InvalidPathException; @@ -91,7 +90,6 @@ import org.apache.tika.mime.MimeTypeException; import org.apache.tika.mime.MimeTypes; import org.apache.tika.parser.AutoDetectParser; import org.apache.tika.parser.CompositeParser; -import org.apache.tika.parser.NetworkParser; import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.Parser; import org.apache.tika.parser.ParserDecorator; @@ -136,7 +134,6 @@ public class TikaCLI { private TikaLoader tikaLoader; private String configFilePath; private boolean recursiveJSON = false; - private URI networkURI = null; /** * Output character encoding, or <code>null</code> for platform default */ @@ -447,8 +444,13 @@ public class TikaCLI { } if (args.length == 2) { - if (Files.isDirectory(Paths.get(args[0]))) { - return true; + try { + if (Files.isDirectory(Paths.get(args[0]))) { + return true; + } + } catch (InvalidPathException e) { + // Not a valid path (e.g. a URL passed as a raw single-dash + // arg on Windows) -- fall through to the other checks. } } @@ -607,8 +609,6 @@ public class TikaCLI { maxEmbeddedCount = Integer.parseInt(arg.substring("--maxEmbeddedCount=".length())); } else if (arg.equals("-r") || arg.equals("--pretty-print")) { prettyPrint = true; - } else if (arg.startsWith("--client=")) { - networkURI = new URI(arg.substring("--client=".length())); } else { // Any arg that reaches here is either "-" (stdin), an existing // file, a URL, or an unknown/typo'd flag. The default fallthrough @@ -1023,11 +1023,7 @@ public class TikaCLI { Files.deleteIfExists(tempConfig); } } - if (networkURI != null) { - parser = new NetworkParser(networkURI); - } else { - parser = tikaLoader.loadAutoDetectParser(); - } + parser = tikaLoader.loadAutoDetectParser(); // Load configs from tika-config.json and merge into existing context // (preserves EmbeddedDocumentExtractor and other items set before configure()) diff --git a/tika-core/src/main/java/org/apache/tika/parser/NetworkParser.java b/tika-core/src/main/java/org/apache/tika/parser/NetworkParser.java deleted file mode 100644 index f6bae48eef..0000000000 --- a/tika-core/src/main/java/org/apache/tika/parser/NetworkParser.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.tika.parser; - -import java.io.FilterOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.Socket; -import java.net.URI; -import java.net.URL; -import java.net.URLConnection; -import java.util.Collections; -import java.util.Set; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.io.function.IOSupplier; -import org.apache.commons.io.input.CloseShieldInputStream; -import org.xml.sax.Attributes; -import org.xml.sax.ContentHandler; -import org.xml.sax.SAXException; -import org.xml.sax.helpers.DefaultHandler; - -import org.apache.tika.exception.TikaException; -import org.apache.tika.io.TikaInputStream; -import org.apache.tika.metadata.KeyPrefix; -import org.apache.tika.metadata.Metadata; -import org.apache.tika.mime.MediaType; -import org.apache.tika.sax.TaggedContentHandler; -import org.apache.tika.sax.TeeContentHandler; -import org.apache.tika.utils.XMLReaderUtils; - - -public class NetworkParser implements Parser { - - // meta/@name in the remote parse service's XML response: naming a service's own inferred - // field is tool provenance, not file provenance (contrast dif:/gdal:/hdf:, which read names - // straight out of the file being parsed). - private static final KeyPrefix NETWORK = - KeyPrefix.tool("network:", "meta/@name from a remote parse service's XML response"); - - private final URI uri; - - private final Set<MediaType> supportedTypes; - - public NetworkParser(URI uri, Set<MediaType> supportedTypes) { - this.uri = uri; - this.supportedTypes = supportedTypes; - } - - public NetworkParser(URI uri) { - this(uri, Collections.singleton(MediaType.OCTET_STREAM)); - } - - public Set<MediaType> getSupportedTypes(ParseContext context) { - return supportedTypes; - } - - public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata, - ParseContext context) throws IOException, SAXException, TikaException { - if ("telnet".equals(uri.getScheme())) { - try (Socket socket = new Socket(uri.getHost(), uri.getPort())) { - new ParsingTask(tis, new FilterOutputStream(socket.getOutputStream()) { - @Override - public void close() throws IOException { - socket.shutdownOutput(); - } - }).parse(socket::getInputStream, handler, metadata, context); - } - } else { - URL url = uri.toURL(); - URLConnection connection = url.openConnection(); - connection.setDoOutput(true); - connection.connect(); - // getInputStream() must not be called until the request is written (see - // ParsingTask.parse() javadoc) -- get the output stream first. - OutputStream output = connection.getOutputStream(); - new ParsingTask(tis, output) - .parse(connection::getInputStream, handler, metadata, context); - } - - } - - private static class ParsingTask implements Runnable { - - private final TikaInputStream input; - - private final OutputStream output; - - private volatile Exception exception = null; - - public ParsingTask(TikaInputStream input, OutputStream output) { - this.input = input; - this.output = output; - } - - /** - * @param streamSupplier opens the response stream -- called only after the writer - * thread has started, so a supplier that blocks until the request - * is fully sent (e.g. {@code URLConnection#getInputStream}) doesn't - * deadlock waiting on a write that hasn't begun yet - */ - public void parse(IOSupplier<InputStream> streamSupplier, ContentHandler handler, - Metadata metadata, ParseContext context) - throws IOException, SAXException, TikaException { - Thread thread = new Thread(this, "Tika network parser"); - thread.start(); - - TaggedContentHandler tagged = - new TaggedContentHandler(handler); - // shield the real stream from the SAX parser's own close(); the try-with-resources - // below is what actually closes it, once parsing is done. - try (InputStream stream = streamSupplier.get()) { - XMLReaderUtils - .parseSAX(CloseShieldInputStream.wrap(stream), - new TeeContentHandler(tagged, new MetaHandler(metadata)), context); - } catch (SAXException e) { - tagged.throwIfCauseOf(e); - throw new TikaException("Invalid network parser output", e); - } catch (IOException e) { - throw new TikaException("Unable to read network parser output", e); - } finally { - try { - thread.join(1000); - } catch (InterruptedException e) { - throw new TikaException("Network parser interrupted", e); - } - - if (exception != null) { - input.throwIfCauseOf(exception); - throw new TikaException("Unexpected network parser error", exception); - } - } - } - - //----------------------------------------------------------<Runnable> - - public void run() { - try { - try { - IOUtils.copy(input, output); - } finally { - output.close(); - } - } catch (Exception e) { - exception = e; - } - } - - } - - private static class MetaHandler extends DefaultHandler { - - private final Metadata metadata; - - public MetaHandler(Metadata metadata) { - this.metadata = metadata; - } - - @Override - public void startElement(String uri, String localName, String qName, Attributes attributes) - throws SAXException { - if ("http://www.w3.org/1999/xhtml".equals(uri) && "meta".equals(localName)) { - String name = attributes.getValue("", "name"); - String content = attributes.getValue("", "content"); - if (name != null && content != null) { - metadata.add(NETWORK, name, content); - } - } - } - - } - -} diff --git a/tika-core/src/test/java/org/apache/tika/parser/NetworkParserTest.java b/tika-core/src/test/java/org/apache/tika/parser/NetworkParserTest.java deleted file mode 100644 index a59bfaf7a6..0000000000 --- a/tika-core/src/test/java/org/apache/tika/parser/NetworkParserTest.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.tika.parser; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - -import java.io.ByteArrayOutputStream; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.ServerSocket; -import java.net.Socket; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.junit.jupiter.api.Test; -import org.xml.sax.helpers.DefaultHandler; - -import org.apache.tika.io.TikaInputStream; -import org.apache.tika.metadata.Metadata; -import org.apache.tika.mime.MediaType; - -/** - * {@code meta/@name} in the remote service's XML response is tool-derived text, so - * {@link NetworkParser} routes it through the {@code network:} {@link - * org.apache.tika.metadata.KeyPrefix}. No test resource fixture is needed or wanted here: a - * loopback {@link ServerSocket} stands in for the remote parse service (raw sockets only -- - * {@code com.sun.net.httpserver} is forbiddenapis-banned as a non-portable internal JDK class). - * TIKA-4816. - */ -public class NetworkParserTest { - - private static final String RESPONSE_XML = "<html xmlns=\"http://www.w3.org/1999/xhtml\";>" - + "<head><meta name=\"foo\" content=\"bar\"/></head><body/></html>"; - - @Test - public void telnetSchemeRoutesMetaNameThroughNetworkKeyPrefix() throws Exception { - try (ServerSocket server = new ServerSocket(0)) { - Thread serverThread = new Thread(() -> { - try (Socket accepted = server.accept()) { - accepted.getInputStream().readAllBytes(); // client half-closes after writing - accepted.getOutputStream().write(RESPONSE_XML.getBytes(StandardCharsets.UTF_8)); - } catch (Exception ignore) { - // surfaced indirectly: the client-side assertions below fail instead - } - }, "test-network-parser-telnet-server"); - serverThread.start(); - - URI uri = URI.create("telnet://localhost:" + server.getLocalPort()); - assertMetaNameRoutedThroughNetworkKeyPrefix(uri); - serverThread.join(5000); - } - } - - @Test - public void httpSchemeRoutesMetaNameThroughNetworkKeyPrefix() throws Exception { - // Also exercises the getOutputStream()-before-getInputStream() ordering fix: the - // original code called URLConnection#getInputStream() before the request had been - // written, which deadlocks/fails against any real HTTP server (TIKA-4816). - try (ServerSocket server = new ServerSocket(0)) { - Thread serverThread = new Thread(() -> { - try (Socket accepted = server.accept()) { - // HttpURLConnection keeps the connection open for keep-alive (never - // half-closes), so -- unlike the telnet server above -- draining to EOF - // would hang forever; read exactly the declared request body instead. - readHttpRequest(accepted.getInputStream()); - byte[] body = RESPONSE_XML.getBytes(StandardCharsets.UTF_8); - OutputStream out = accepted.getOutputStream(); - out.write(("HTTP/1.1 200 OK\r\n" - + "Content-Type: text/xml; charset=utf-8\r\n" - + "Content-Length: " + body.length + "\r\n" - + "Connection: close\r\n\r\n").getBytes(StandardCharsets.UTF_8)); - out.write(body); - out.flush(); - } catch (Exception ignore) { - // surfaced indirectly: the client-side assertions below fail instead - } - }, "test-network-parser-http-server"); - serverThread.start(); - - URI uri = URI.create("http://localhost:"; + server.getLocalPort() + "/"); - assertMetaNameRoutedThroughNetworkKeyPrefix(uri); - serverThread.join(5000); - } - } - - private static final Pattern CONTENT_LENGTH = - Pattern.compile("(?i)content-length:\\s*(\\d+)"); - - /** Reads a minimal HTTP request (headers, then the declared Content-Length body bytes) off - * {@code in} without closing it and without needing the client to signal EOF. */ - private static void readHttpRequest(InputStream in) throws Exception { - ByteArrayOutputStream headerBytes = new ByteArrayOutputStream(); - int trailingCrLfCrLf = 0; // count of the last 4 bytes matching "\r\n\r\n" so far - int[] terminator = {'\r', '\n', '\r', '\n'}; - int b; - while (trailingCrLfCrLf < terminator.length && (b = in.read()) != -1) { - headerBytes.write(b); - trailingCrLfCrLf = (b == terminator[trailingCrLfCrLf]) ? trailingCrLfCrLf + 1 - : (b == '\r' ? 1 : 0); - } - Matcher m = CONTENT_LENGTH.matcher(headerBytes.toString(StandardCharsets.UTF_8)); - int contentLength = m.find() ? Integer.parseInt(m.group(1)) : 0; - in.readNBytes(contentLength); - } - - private void assertMetaNameRoutedThroughNetworkKeyPrefix(URI uri) throws Exception { - NetworkParser parser = new NetworkParser(uri, Collections.singleton(MediaType.OCTET_STREAM)); - Metadata metadata = new Metadata(); - try (TikaInputStream tis = - TikaInputStream.get("posted document".getBytes(StandardCharsets.UTF_8))) { - parser.parse(tis, new DefaultHandler(), metadata, new ParseContext()); - } - - assertEquals("bar", metadata.get("network:foo")); - assertNull(metadata.get("foo"), "the unprefixed legacy key must not appear"); - } -}
