This is an automated email from the ASF dual-hosted git repository.
yuqi1129 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 409f83447f [#11784] improvement(common): harden remote file fetch
against SSRF DNS-rebinding (#11785)
409f83447f is described below
commit 409f83447fde452eef58db4a86405826d71651ad
Author: YangJie <[email protected]>
AuthorDate: Tue Jul 7 17:24:30 2026 +0800
[#11784] improvement(common): harden remote file fetch against SSRF
DNS-rebinding (#11785)
### What changes were proposed in this pull request?
`FileFetcher` already checked a remote URI's host against an SSRF
denylist, but the download step then let the JDK re-resolve the hostname
when it connected. That leaves a DNS-rebinding window: the host can look
safe during validation and resolve to an internal address when the
download actually connects.
The fix resolves the host once in `RemoteUriValidator`, validates every
address it resolves to, and returns one for the caller to pin.
`RemoteFileDownloader` is a small HTTP/1.0 client that connects straight
to that pinned `InetAddress` and never looks the hostname up again,
while still using the original hostname for TLS/SNI verification. It
follows no redirects and accepts only Content-Length or chunked bodies,
so a truncated download can't be mistaken for a complete one.
Along the way it also:
- widens the denylist to 0.0.0.0/8, CGNAT 100.64/10 (covers the Alibaba
metadata IP), the Oracle metadata IP, broadcast, IPv6 unique-local, and
the IPv4-compatible/mapped/NAT64/6to4/ISATAP forms that hide an IPv4 in
an IPv6 literal;
- rejects `ftp://` when blocking is on, because its PASV data channel
connects to an address the server picks and can't be pinned;
- adds `SafeUri.redact` to strip userinfo and query strings from URIs
before they reach logs or error messages.
### Why are the changes needed?
The server downloads keytabs and jars from operator-supplied URIs.
Without pinning, someone who controls DNS for one of those hosts can
pass validation and still steer the connection to a cloud metadata
endpoint or an internal service. The extra denylist entries and the FTP
restriction remove the other ways to reach those addresses.
Fix: #11784
### Does this PR introduce _any_ user-facing change?
No new config or API. One behavior change: with
`gravitino.fetchFile.blockUnsafeRemoteUri` enabled (the default),
`ftp://` URIs are now rejected. Set it to false to allow them from a
trusted source.
### How was this patch tested?
Added 48 unit tests across four classes covering address classification
(including the IPv6-embedded forms), address pinning,
redirect/unframed/oversized/truncated responses, chunked decoding, FTP
rejection, and URI redaction. `./gradlew :common:test` and
`:common:spotlessCheck` pass.
---
.../org/apache/gravitino/utils/FileFetcher.java | 60 +-
.../gravitino/utils/RemoteFileDownloader.java | 396 ++++++++++++
.../apache/gravitino/utils/RemoteUriValidator.java | 140 ++++-
.../java/org/apache/gravitino/utils/SafeUri.java | 55 ++
.../apache/gravitino/utils/TestFileFetcher.java | 89 ++-
.../gravitino/utils/TestRemoteFileDownloader.java | 686 +++++++++++++++++++++
.../gravitino/utils/TestRemoteUriValidator.java | 118 ++--
.../org/apache/gravitino/utils/TestSafeUri.java | 51 ++
8 files changed, 1514 insertions(+), 81 deletions(-)
diff --git a/common/src/main/java/org/apache/gravitino/utils/FileFetcher.java
b/common/src/main/java/org/apache/gravitino/utils/FileFetcher.java
index 7291a9232a..dacd0cdcb8 100644
--- a/common/src/main/java/org/apache/gravitino/utils/FileFetcher.java
+++ b/common/src/main/java/org/apache/gravitino/utils/FileFetcher.java
@@ -20,11 +20,13 @@ package org.apache.gravitino.utils;
import java.io.File;
import java.io.IOException;
+import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
+import java.util.Locale;
import java.util.Optional;
import javax.annotation.Nullable;
import org.apache.commons.io.FileUtils;
@@ -90,17 +92,33 @@ public final class FileFetcher {
throws IOException {
try {
URI uri = new URI(fileUri);
- String scheme = Optional.ofNullable(uri.getScheme()).orElse("file");
+ // URI schemes are case-insensitive (RFC 3986); normalize so e.g. "HTTP"
routes like "http".
+ String scheme =
Optional.ofNullable(uri.getScheme()).orElse("file").toLowerCase(Locale.ROOT);
switch (scheme) {
case "http":
case "https":
case "ftp":
- RemoteUriValidator.validate(
- uri,
- blockUnsafeRemoteUri,
- String.format("'%s' to false", BLOCK_UNSAFE_REMOTE_URI_CONFIG));
- FileUtils.copyURLToFile(uri.toURL(), destFile, timeoutMs, timeoutMs);
+ if (!blockUnsafeRemoteUri) {
+ FileUtils.copyURLToFile(uri.toURL(), destFile, timeoutMs,
timeoutMs);
+ } else if (scheme.equals("ftp")) {
+ // FTP opens its data channel to an address the server chooses in
its PASV/EPSV reply,
+ // which cannot be pinned to the validated host and is therefore
vulnerable to SSRF. The
+ // operator must opt out of blocking to use FTP from a trusted
source.
+ throw new IllegalArgumentException(
+ String.format(
+ "Refusing to fetch ftp uri '%s' from the Gravitino server
side: FTP's data "
+ + "channel cannot be restricted to the validated
address. Set %s to false "
+ + "to allow it if the source is trusted.",
+ SafeUri.redact(uri), BLOCK_UNSAFE_REMOTE_URI_CONFIG));
+ } else {
+ // Resolve and validate the host exactly once, then pin the
download to the validated
+ // address so the hostname cannot be re-resolved to an unsafe
address (DNS rebinding).
+ InetAddress pinnedAddress =
+ RemoteUriValidator.resolveAndValidate(
+ uri, String.format("'%s' to false",
BLOCK_UNSAFE_REMOTE_URI_CONFIG));
+ RemoteFileDownloader.download(uri, pinnedAddress, destFile,
timeoutMs);
+ }
break;
case "file":
@@ -123,7 +141,15 @@ public final class FileFetcher {
}
private synchronized void linkLocalFile(URI uri, File destFile) throws
IOException {
- Path srcPath = new File(uri.getPath()).toPath().normalize();
+ String sourcePath = uri.getPath();
+ if (sourcePath == null) {
+ // Opaque file URIs (e.g. "file:relative", no authority) have a null
path; fail with a clear
+ // message rather than a context-free NullPointerException.
+ throw new IOException("file uri has no path: " + SafeUri.redact(uri));
+ }
+ // Resolve to an absolute path: a relative source would otherwise be
stored as the symlink
+ // target and resolved against the link's own directory, producing a
dangling symlink.
+ Path srcPath = new File(sourcePath).toPath().toAbsolutePath().normalize();
if (!Files.exists(srcPath)) {
throw new IOException(
String.format("Source file does not exist: %s",
srcPath.toAbsolutePath()));
@@ -142,7 +168,19 @@ public final class FileFetcher {
Path tmpPath = destPath.resolveSibling(destPath.getFileName() +
".symlink.tmp");
Files.deleteIfExists(tmpPath);
Files.createSymbolicLink(tmpPath, srcPath);
- Files.move(tmpPath, destPath, StandardCopyOption.REPLACE_EXISTING);
+ try {
+ Files.move(tmpPath, destPath, StandardCopyOption.REPLACE_EXISTING);
+ } catch (Throwable e) {
+ // Do not orphan the temporary symlink if the rename fails (e.g.
read-only or cross-device
+ // destination). This method is synchronized on the singleton, so the
fixed temp name cannot
+ // race with a concurrent local fetch.
+ try {
+ Files.deleteIfExists(tmpPath);
+ } catch (IOException suppressed) {
+ e.addSuppressed(suppressed);
+ }
+ throw e;
+ }
}
/**
@@ -156,7 +194,8 @@ public final class FileFetcher {
() ->
new IllegalArgumentException(
String.format(
- "A Hadoop configuration is required to fetch an 'hdfs'
uri: %s", uri)));
+ "A Hadoop configuration is required to fetch an 'hdfs'
uri: %s",
+ SafeUri.redact(uri))));
try {
Class<?> configurationClass =
Class.forName("org.apache.hadoop.conf.Configuration");
Class<?> fileSystemClass =
Class.forName("org.apache.hadoop.fs.FileSystem");
@@ -170,7 +209,8 @@ public final class FileFetcher {
.getMethod("copyToLocalFile", pathClass, pathClass)
.invoke(fileSystem, srcPath, destPath);
} catch (ReflectiveOperationException e) {
- throw new IOException(String.format("Failed to fetch file from hdfs uri:
%s", uri), e);
+ throw new IOException(
+ String.format("Failed to fetch file from hdfs uri: %s",
SafeUri.redact(uri)), e);
}
}
}
diff --git
a/common/src/main/java/org/apache/gravitino/utils/RemoteFileDownloader.java
b/common/src/main/java/org/apache/gravitino/utils/RemoteFileDownloader.java
new file mode 100644
index 0000000000..8f1fa8c1e4
--- /dev/null
+++ b/common/src/main/java/org/apache/gravitino/utils/RemoteFileDownloader.java
@@ -0,0 +1,396 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.utils;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import javax.net.ssl.SSLParameters;
+import javax.net.ssl.SSLSocket;
+import javax.net.ssl.SSLSocketFactory;
+
+/**
+ * Downloads an {@code http} or {@code https} file while pinning the TCP
connection to a
+ * pre-validated address.
+ *
+ * <p>This closes the DNS-rebinding TOCTOU window between SSRF validation and
the actual fetch: the
+ * host is resolved and validated once by {@link
RemoteUriValidator#resolveAndValidate}, and the
+ * resulting {@link InetAddress} is connected to directly here, so the
hostname is never
+ * re-resolved. A minimal HTTP/1.1 client is used so the connection can be
pinned at the socket
+ * level (the JDK exposes no per-connection address override for plain {@code
HttpURLConnection},
+ * and silently drops a {@code Host} header set via {@code
setRequestProperty}). The original
+ * hostname is still used for the HTTP {@code Host} header and, for {@code
https}, for TLS SNI and
+ * certificate-hostname verification, so virtual hosting and certificate
validation keep working.
+ * Redirects are not followed, because a redirect target would re-resolve to
an unvalidated address.
+ *
+ * <p>{@code ftp} is intentionally not handled here: the FTP data channel is
opened to an address
+ * the server chooses in its {@code PASV}/{@code EPSV} reply, which cannot be
pinned, so {@link
+ * FileFetcher} rejects {@code ftp} on the SSRF-blocking path instead.
+ *
+ * <p>Only responses framed by {@code Content-Length} or chunked
transfer-encoding are accepted; a
+ * connection-close-delimited body is rejected because a premature close
cannot be told apart from a
+ * complete one. Combined with streaming to a sibling temporary file that is
atomically moved into
+ * place only on success, this means a connection that drops mid-transfer
never leaves a truncated
+ * destination behind. Response and header sizes are bounded to prevent a
malicious pinned host from
+ * exhausting disk or memory.
+ */
+final class RemoteFileDownloader {
+
+ private static final int DEFAULT_HTTP_PORT = 80;
+ private static final int DEFAULT_HTTPS_PORT = 443;
+ private static final int MAX_HEADER_BYTES = 64 * 1024;
+ private static final long MAX_BODY_BYTES = 2L * 1024 * 1024 * 1024;
+ private static final int COPY_BUFFER_BYTES = 8192;
+
+ private RemoteFileDownloader() {}
+
+ /**
+ * Downloads {@code uri} into {@code destFile}, connecting only to {@code
pinnedAddress}.
+ *
+ * @param uri the source URI; must use the {@code http} or {@code https}
scheme
+ * @param pinnedAddress the validated address to connect to
+ * @param destFile the local destination file
+ * @param timeoutMs the connect/read timeout in milliseconds
+ * @throws IOException if the file cannot be fetched or the response is not
successful
+ */
+ static void download(URI uri, InetAddress pinnedAddress, File destFile, int
timeoutMs)
+ throws IOException {
+ String scheme =
Optional.ofNullable(uri.getScheme()).orElse("").toLowerCase(Locale.ROOT);
+ if (!scheme.equals("http") && !scheme.equals("https")) {
+ throw new IllegalArgumentException(
+ String.format("Pinned download does not support scheme '%s'",
scheme));
+ }
+
+ Path destPath = destFile.toPath().toAbsolutePath();
+ Path tempPath =
+ Files.createTempFile(destPath.getParent(), destPath.getFileName() +
".", ".tmp");
+ try {
+ httpDownload(uri, pinnedAddress, tempPath, timeoutMs,
scheme.equals("https"));
+ Files.move(tempPath, destPath, StandardCopyOption.REPLACE_EXISTING);
+ } catch (Throwable e) {
+ // Catch every throwable (incl. Error/OutOfMemoryError) so the temp file
is never orphaned in
+ // the staging directory. Precise rethrow keeps the declared `throws
IOException`.
+ try {
+ Files.deleteIfExists(tempPath);
+ } catch (IOException suppressed) {
+ e.addSuppressed(suppressed);
+ }
+ throw e;
+ }
+ }
+
+ private static void httpDownload(
+ URI uri, InetAddress pinnedAddress, Path destPath, int timeoutMs,
boolean tls)
+ throws IOException {
+ String host = uri.getHost();
+ int port = uri.getPort() != -1 ? uri.getPort() : (tls ? DEFAULT_HTTPS_PORT
: DEFAULT_HTTP_PORT);
+
+ Socket socket = new Socket();
+ try {
+ socket.connect(new InetSocketAddress(pinnedAddress, port), timeoutMs);
+ socket.setSoTimeout(timeoutMs);
+ if (tls) {
+ socket = startTls(socket, host, port);
+ }
+
+ sendGetRequest(socket.getOutputStream(), requestTarget(uri),
hostHeader(host, uri.getPort()));
+
+ InputStream in = new BufferedInputStream(socket.getInputStream());
+ int status = readStatusCode(in);
+ if (status < 200 || status >= 300) {
+ throw new IOException(
+ String.format(
+ "Unexpected HTTP status %d fetching pinned URI '%s'", status,
SafeUri.redact(uri)));
+ }
+ Map<String, String> headers = readHeaders(in);
+ try (OutputStream out = Files.newOutputStream(destPath)) {
+ writeBody(in, headers, out);
+ }
+ } finally {
+ socket.close();
+ }
+ }
+
+ /**
+ * Layers TLS over an already-connected plain socket. The handshake uses
{@code host} (the
+ * original hostname) for SNI and certificate verification ({@code HTTPS}
endpoint
+ * identification), while the TCP connection stays pinned to the address the
plain socket was
+ * connected to.
+ */
+ private static Socket startTls(Socket plainSocket, String host, int port)
throws IOException {
+ SSLSocketFactory factory = (SSLSocketFactory)
SSLSocketFactory.getDefault();
+ SSLSocket sslSocket = (SSLSocket) factory.createSocket(plainSocket, host,
port, true);
+ SSLParameters parameters = sslSocket.getSSLParameters();
+ parameters.setEndpointIdentificationAlgorithm("HTTPS");
+ sslSocket.setSSLParameters(parameters);
+ sslSocket.startHandshake();
+ return sslSocket;
+ }
+
+ private static void sendGetRequest(OutputStream out, String target, String
hostHeader)
+ throws IOException {
+ // HTTP/1.1 with "Connection: close" requests a single non-persistent
exchange while biasing the
+ // origin toward framing the body with Content-Length or chunked encoding.
An unframed
+ // (close-delimited) body is still rejected by writeBody because its
completeness cannot be
+ // verified.
+ String request =
+ "GET "
+ + target
+ + " HTTP/1.1\r\n"
+ + "Host: "
+ + hostHeader
+ + "\r\n"
+ + "Connection: close\r\n"
+ + "\r\n";
+ out.write(request.getBytes(StandardCharsets.US_ASCII));
+ out.flush();
+ }
+
+ private static int readStatusCode(InputStream in) throws IOException {
+ String statusLine = readLine(in);
+ if (statusLine == null) {
+ throw new IOException("Empty HTTP response from pinned host");
+ }
+ // Format: "HTTP/1.x <code> <reason>"; tolerate extra whitespace between
tokens.
+ String[] parts = statusLine.trim().split("\\s+", 3);
+ if (parts.length < 2) {
+ throw new IOException("Malformed HTTP status line: " + statusLine);
+ }
+ try {
+ return Integer.parseInt(parts[1]);
+ } catch (NumberFormatException e) {
+ throw new IOException("Malformed HTTP status line: " + statusLine, e);
+ }
+ }
+
+ private static Map<String, String> readHeaders(InputStream in) throws
IOException {
+ Map<String, String> headers = new HashMap<>();
+ int total = 0;
+ String line;
+ while ((line = readLine(in)) != null) {
+ if (line.isEmpty()) {
+ return headers;
+ }
+ // Reject obsolete line folding (RFC 7230 §3.2.4); a leading space/tab
continuation is treated
+ // differently by different parsers and is a smuggling vector.
+ if (line.charAt(0) == ' ' || line.charAt(0) == '\t') {
+ throw new IOException("Obsolete line folding in HTTP headers is
rejected from pinned host");
+ }
+ total += line.length();
+ if (total > MAX_HEADER_BYTES) {
+ throw new IOException("HTTP response headers exceed the allowed size
from pinned host");
+ }
+ int colon = line.indexOf(':');
+ if (colon <= 0) {
+ throw new IOException("Malformed HTTP header line from pinned host: "
+ line);
+ }
+ String name = line.substring(0, colon).trim().toLowerCase(Locale.ROOT);
+ String value = line.substring(colon + 1).trim();
+ String previous = headers.put(name, value);
+ // A duplicate Content-Length or Transfer-Encoding with a different
value (RFC 7230 §3.3.2 /
+ // §3.3.3) is a request-smuggling vector: parsers that disagree on which
copy wins frame the
+ // body differently. Reject the conflict rather than silently keeping
the last one.
+ if (previous != null
+ && !previous.equals(value)
+ && (name.equals("content-length") ||
name.equals("transfer-encoding"))) {
+ throw new IOException("Conflicting duplicate " + name + " headers from
pinned host");
+ }
+ }
+ throw new IOException("Unexpected end of stream while reading HTTP headers
from pinned host");
+ }
+
+ private static void writeBody(InputStream in, Map<String, String> headers,
OutputStream out)
+ throws IOException {
+ String transferEncoding = headers.get("transfer-encoding");
+ if (transferEncoding != null) {
+ // Accept only a sole "chunked" coding. A value such as "gzip, chunked"
is de-chunkable but
+ // still compressed, and we do not gunzip, so de-chunking alone would
write corrupt bytes; a
+ // substring like "xchunked" must likewise not be mistaken for chunked.
Reject anything other
+ // than an exact "chunked" rather than emit a mis-decoded body.
+ if (!transferEncoding.trim().equalsIgnoreCase("chunked")) {
+ throw new IOException(
+ "Unsupported Transfer-Encoding '"
+ + transferEncoding
+ + "' from pinned host; only 'chunked' is supported");
+ }
+ copyChunked(in, out);
+ return;
+ }
+ String contentLength = headers.get("content-length");
+ if (contentLength != null) {
+ copyExact(in, out, parseContentLength(contentLength));
+ return;
+ }
+ // Reject a connection-close-delimited body (no Content-Length, no chunked
framing): a premature
+ // close cannot be distinguished from a complete body, so the download's
completeness would be
+ // unverifiable and a truncated keytab/jar could be installed silently.
+ throw new IOException(
+ "Response from pinned host has neither Content-Length nor chunked
framing; "
+ + "its completeness cannot be verified");
+ }
+
+ private static long parseContentLength(String value) throws IOException {
+ String trimmed = value.trim();
+ // RFC 7230 §3.3.2 defines Content-Length as 1*DIGIT; reject anything else
(e.g. "+5", "0x10")
+ // that Long.parseLong would otherwise accept, to avoid framing
disagreement with upstreams.
+ if (trimmed.isEmpty() || !isAsciiDigits(trimmed)) {
+ throw new IOException("Malformed Content-Length header from pinned host:
" + value);
+ }
+ long length;
+ try {
+ length = Long.parseLong(trimmed);
+ } catch (NumberFormatException e) {
+ throw new IOException("Malformed Content-Length header from pinned host:
" + value, e);
+ }
+ if (length > MAX_BODY_BYTES) {
+ throw new IOException("Content-Length " + length + " exceeds the maximum
allowed size");
+ }
+ return length;
+ }
+
+ private static boolean isAsciiDigits(String value) {
+ for (int i = 0; i < value.length(); i++) {
+ char c = value.charAt(i);
+ if (c < '0' || c > '9') {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /** Copies exactly {@code length} bytes, failing if the stream ends early. */
+ private static void copyExact(InputStream in, OutputStream out, long length)
throws IOException {
+ byte[] buffer = new byte[COPY_BUFFER_BYTES];
+ long remaining = length;
+ while (remaining > 0) {
+ int read = in.read(buffer, 0, (int) Math.min(buffer.length, remaining));
+ if (read == -1) {
+ throw new IOException(
+ String.format(
+ "Truncated response: %d of %d body bytes missing from pinned
host",
+ remaining, length));
+ }
+ out.write(buffer, 0, read);
+ remaining -= read;
+ }
+ }
+
+ private static void copyChunked(InputStream in, OutputStream out) throws
IOException {
+ long total = 0;
+ while (true) {
+ String sizeLine = readLine(in);
+ if (sizeLine == null) {
+ throw new IOException("Unexpected end of stream reading chunk size
from pinned host");
+ }
+ int extension = sizeLine.indexOf(';');
+ String hex = (extension >= 0 ? sizeLine.substring(0, extension) :
sizeLine).trim();
+ int chunkSize;
+ try {
+ chunkSize = Integer.parseInt(hex, 16);
+ } catch (NumberFormatException e) {
+ throw new IOException("Malformed chunk size from pinned host: " +
sizeLine, e);
+ }
+ if (chunkSize < 0) {
+ throw new IOException("Negative chunk size from pinned host: " +
sizeLine);
+ }
+ if (chunkSize == 0) {
+ break;
+ }
+ total += chunkSize;
+ if (total > MAX_BODY_BYTES) {
+ throw new IOException("Chunked response exceeds the maximum allowed
size from pinned host");
+ }
+ copyExact(in, out, chunkSize);
+ // Each chunk's data is followed by a CRLF, which readLine returns as an
empty line.
+ String terminator = readLine(in);
+ if (terminator == null || !terminator.isEmpty()) {
+ throw new IOException("Malformed chunk terminator from pinned host");
+ }
+ }
+ // Discard any trailer headers up to the final blank line, bounding their
total size so a server
+ // cannot flood unbounded trailers.
+ int trailerBytes = 0;
+ String line;
+ while ((line = readLine(in)) != null && !line.isEmpty()) {
+ trailerBytes += line.length();
+ if (trailerBytes > MAX_HEADER_BYTES) {
+ throw new IOException("Chunked trailer headers exceed the allowed size
from pinned host");
+ }
+ }
+ }
+
+ /**
+ * Reads a CRLF- or LF-terminated line, returning it without the line
terminator, or {@code null}
+ * at end of stream. The line is capped at {@link #MAX_HEADER_BYTES} to
bound memory.
+ */
+ private static String readLine(InputStream in) throws IOException {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ boolean read = false;
+ int b;
+ while ((b = in.read()) != -1) {
+ read = true;
+ if (b == '\n') {
+ break;
+ }
+ buffer.write(b);
+ if (buffer.size() > MAX_HEADER_BYTES) {
+ throw new IOException("HTTP line exceeds the allowed size from pinned
host");
+ }
+ }
+ if (!read) {
+ return null;
+ }
+ // LF terminates the line; strip a single trailing CR (CRLF), but preserve
any other bytes so
+ // the parser does not silently mangle content that legitimately contains
a CR.
+ byte[] bytes = buffer.toByteArray();
+ int length = bytes.length;
+ if (length > 0 && bytes[length - 1] == '\r') {
+ length--;
+ }
+ return new String(bytes, 0, length, StandardCharsets.US_ASCII);
+ }
+
+ static String requestTarget(URI uri) {
+ String path = uri.getRawPath();
+ if (path == null || path.isEmpty()) {
+ path = "/";
+ }
+ String query = uri.getRawQuery();
+ return query == null ? path : path + "?" + query;
+ }
+
+ static String hostHeader(String host, int port) {
+ return port == -1 ? host : host + ":" + port;
+ }
+}
diff --git
a/common/src/main/java/org/apache/gravitino/utils/RemoteUriValidator.java
b/common/src/main/java/org/apache/gravitino/utils/RemoteUriValidator.java
index 41a60fa870..29f0a471b4 100644
--- a/common/src/main/java/org/apache/gravitino/utils/RemoteUriValidator.java
+++ b/common/src/main/java/org/apache/gravitino/utils/RemoteUriValidator.java
@@ -21,6 +21,7 @@ package org.apache.gravitino.utils;
import java.io.IOException;
import java.net.InetAddress;
import java.net.URI;
+import java.net.UnknownHostException;
/** Validates remote URI hosts before server-side downloads. */
public final class RemoteUriValidator {
@@ -28,23 +29,24 @@ public final class RemoteUriValidator {
private RemoteUriValidator() {}
/**
- * Resolves the host in the given URI and rejects unsafe addresses when
blocking is enabled.
+ * Resolves the host in the given URI exactly once, rejects unsafe
addresses, and returns a
+ * validated address for the caller to pin the subsequent connection to.
+ *
+ * <p>Returning the resolved address closes the DNS-rebinding TOCTOU window:
a caller that
+ * connects to the returned {@link InetAddress} never re-resolves the
hostname, so an attacker
+ * cannot make validation see a safe address and the fetch see an unsafe one.
*
* @param uri The remote URI to validate.
- * @param blockUnsafeAddress Whether unsafe addresses should be blocked.
* @param blockUnsafeAddressHint The configuration hint that disables unsafe
address blocking.
+ * @return The first resolved address, guaranteed to be safe.
* @throws IOException If host resolution fails.
* @throws IllegalArgumentException If the URI has no host or resolves to an
unsafe address.
*/
- public static void validate(URI uri, boolean blockUnsafeAddress, String
blockUnsafeAddressHint)
+ public static InetAddress resolveAndValidate(URI uri, String
blockUnsafeAddressHint)
throws IOException {
String host = uri.getHost();
if (host == null) {
- throw new IllegalArgumentException("URI has no host: " + uri);
- }
-
- if (!blockUnsafeAddress) {
- return;
+ throw new IllegalArgumentException("URI has no host: " +
SafeUri.redact(uri));
}
InetAddress[] addresses = InetAddress.getAllByName(host);
@@ -56,9 +58,10 @@ public final class RemoteUriValidator {
+ "Access to local, private, link-local, multicast,
unspecified, and cloud "
+ "metadata addresses is disabled by default to prevent
SSRF. If this URI is "
+ "trusted and this access is required, set %s.",
- uri, address.getHostAddress(), blockUnsafeAddressHint));
+ SafeUri.redact(uri), address.getHostAddress(),
blockUnsafeAddressHint));
}
}
+ return addresses[0];
}
private static boolean isUnsafeAddress(InetAddress address) {
@@ -71,19 +74,122 @@ public final class RemoteUriValidator {
}
byte[] bytes = address.getAddress();
- if (isCloudMetadataAddress(bytes)) {
+ if (isUnsafeIpv4Address(bytes)) {
return true;
}
+ if (bytes.length == 16) {
+ if (isIpv6UniqueLocalAddress(bytes)) {
+ return true;
+ }
+ // Several IPv6 forms embed an IPv4 address the platform checks above
miss (IPv4-compatible,
+ // NAT64, 6to4, ISATAP; plus IPv4-mapped defensively). Re-classify the
embedded IPv4 so a
+ // blocked address such as cloud metadata cannot be reached via an IPv6
literal.
+ byte[] embeddedIpv4 = embeddedIpv4(bytes);
+ if (embeddedIpv4 != null) {
+ try {
+ return isUnsafeAddress(InetAddress.getByAddress(embeddedIpv4));
+ } catch (UnknownHostException e) {
+ // getByAddress only rejects a wrong-length array; a 4-byte array
never reaches here. Fail
+ // closed if it somehow does.
+ return true;
+ }
+ }
+ }
+ return false;
+ }
- return isIpv6UniqueLocalAddress(bytes);
+ /**
+ * Returns the IPv4 address embedded in a 16-byte IPv6 address whose
embedded IPv4 is the literal
+ * connection target, or {@code null} if there is none. Covers the
IPv4-compatible ({@code
+ * ::a.b.c.d}), NAT64 ({@code 64:ff9b::a.b.c.d}, RFC 6052), 6to4 ({@code
2002:a.b.c.d::}, RFC
+ * 3056) and ISATAP ({@code ::0:5efe:a.b.c.d}, RFC 5214) forms. The
IPv4-mapped ({@code
+ * ::ffff:a.b.c.d}) form is handled defensively: the JDK normally surfaces
it as a 4-byte {@link
+ * java.net.Inet4Address} validated by {@link #isUnsafeIpv4Address}, so this
branch is a fallback.
+ *
+ * <p>Teredo ({@code 2001:0::/32}) is intentionally excluded: its embedded
IPv4 is the
+ * client/relay identifier rather than the connection destination, and it
requires a non-default
+ * Teredo tunnel to route at all.
+ */
+ private static byte[] embeddedIpv4(byte[] bytes) {
+ boolean highTenZero = true;
+ for (int i = 0; i < 10; i++) {
+ if (bytes[i] != 0) {
+ highTenZero = false;
+ break;
+ }
+ }
+ if (highTenZero) {
+ boolean compatible = bytes[10] == 0 && bytes[11] == 0;
+ // Defensive: the JDK normally collapses IPv4-mapped addresses to a
4-byte Inet4Address.
+ boolean mapped = (bytes[10] & 0xFF) == 0xFF && (bytes[11] & 0xFF) ==
0xFF;
+ if (compatible || mapped) {
+ return lowestFourBytes(bytes);
+ }
+ }
+ // NAT64 well-known prefix 64:ff9b::/96 (RFC 6052): the IPv4 is the low 32
bits. Only the
+ // well-known prefix is matched; network-specific prefixes (RFC 6052 §2.2,
e.g. RFC 8215's
+ // 64:ff9b:1::/48) are site-defined and routable only where explicitly
deployed, so detecting
+ // them would require configuration this validator does not have.
+ if ((bytes[0] & 0xFF) == 0x00
+ && (bytes[1] & 0xFF) == 0x64
+ && (bytes[2] & 0xFF) == 0xFF
+ && (bytes[3] & 0xFF) == 0x9B
+ && isZero(bytes, 4, 12)) {
+ return lowestFourBytes(bytes);
+ }
+ // 6to4 (2002::/16, RFC 3056): the gateway IPv4 is the 32 bits after the
prefix. 2002::/16 is
+ // reserved exclusively for 6to4, so no legitimate non-6to4 host occupies
it.
+ if ((bytes[0] & 0xFF) == 0x20 && (bytes[1] & 0xFF) == 0x02) {
+ return new byte[] {bytes[2], bytes[3], bytes[4], bytes[5]};
+ }
+ // ISATAP interface identifier (RFC 5214): the low 64 bits are
00:00:5e:fe:a.b.c.d or
+ // 02:00:5e:fe:a.b.c.d, embedding the IPv4 in the low 32 bits regardless
of the /64 prefix.
+ if ((bytes[8] & 0xFD) == 0x00
+ && bytes[9] == 0x00
+ && (bytes[10] & 0xFF) == 0x5E
+ && (bytes[11] & 0xFF) == 0xFE) {
+ return lowestFourBytes(bytes);
+ }
+ return null;
+ }
+
+ private static byte[] lowestFourBytes(byte[] bytes) {
+ return new byte[] {bytes[12], bytes[13], bytes[14], bytes[15]};
+ }
+
+ private static boolean isZero(byte[] bytes, int fromInclusive, int
toExclusive) {
+ for (int i = fromInclusive; i < toExclusive; i++) {
+ if (bytes[i] != 0) {
+ return false;
+ }
+ }
+ return true;
}
- private static boolean isCloudMetadataAddress(byte[] bytes) {
- return bytes.length == 4
- && (bytes[0] & 0xFF) == 100
- && (bytes[1] & 0xFF) == 100
- && (bytes[2] & 0xFF) == 100
- && (bytes[3] & 0xFF) == 200;
+ private static boolean isUnsafeIpv4Address(byte[] bytes) {
+ if (bytes.length != 4) {
+ return false;
+ }
+ int b0 = bytes[0] & 0xFF;
+ int b1 = bytes[1] & 0xFF;
+ int b2 = bytes[2] & 0xFF;
+ int b3 = bytes[3] & 0xFF;
+
+ // 0.0.0.0/8 "this network" (RFC 1122). isAnyLocalAddress only covers the
single 0.0.0.0.
+ if (b0 == 0) {
+ return true;
+ }
+ // 100.64.0.0/10 carrier-grade NAT / shared address space (RFC 6598). This
range also covers
+ // the Alibaba Cloud metadata endpoint 100.100.100.200. Not caught by
isSiteLocalAddress.
+ if (b0 == 100 && (b1 & 0xC0) == 0x40) {
+ return true;
+ }
+ // 192.0.0.192 Oracle Cloud Infrastructure instance-metadata endpoint.
+ if (b0 == 192 && b1 == 0 && b2 == 0 && b3 == 192) {
+ return true;
+ }
+ // 255.255.255.255 limited broadcast (RFC 919).
+ return b0 == 255 && b1 == 255 && b2 == 255 && b3 == 255;
}
private static boolean isIpv6UniqueLocalAddress(byte[] bytes) {
diff --git a/common/src/main/java/org/apache/gravitino/utils/SafeUri.java
b/common/src/main/java/org/apache/gravitino/utils/SafeUri.java
new file mode 100644
index 0000000000..aa39772dce
--- /dev/null
+++ b/common/src/main/java/org/apache/gravitino/utils/SafeUri.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.utils;
+
+import java.net.URI;
+
+/** Renders URIs for logs and error messages without their credential-bearing
parts. */
+final class SafeUri {
+
+ private SafeUri() {}
+
+ /**
+ * Returns a log-safe rendering of {@code uri} with the userinfo (e.g.
{@code user:password@}) and
+ * query string (which may carry presigned-URL tokens) removed. The scheme,
host, port and path
+ * are kept so the message stays useful for diagnosis.
+ *
+ * @param uri the URI to redact; may be {@code null}
+ * @return a redacted string safe to log
+ */
+ static String redact(URI uri) {
+ if (uri == null) {
+ return "null";
+ }
+ StringBuilder builder = new StringBuilder();
+ if (uri.getScheme() != null) {
+ builder.append(uri.getScheme()).append("://");
+ }
+ if (uri.getHost() != null) {
+ builder.append(uri.getHost());
+ if (uri.getPort() != -1) {
+ builder.append(':').append(uri.getPort());
+ }
+ }
+ if (uri.getRawPath() != null) {
+ builder.append(uri.getRawPath());
+ }
+ return builder.length() == 0 ? "<redacted uri>" : builder.toString();
+ }
+}
diff --git
a/common/src/test/java/org/apache/gravitino/utils/TestFileFetcher.java
b/common/src/test/java/org/apache/gravitino/utils/TestFileFetcher.java
index 0397e044ee..e088e3a8bf 100644
--- a/common/src/test/java/org/apache/gravitino/utils/TestFileFetcher.java
+++ b/common/src/test/java/org/apache/gravitino/utils/TestFileFetcher.java
@@ -25,6 +25,8 @@ import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
@@ -37,11 +39,15 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+/**
+ * Tests for {@link FileFetcher}.
+ *
+ * <p>The hdfs happy path is exercised reflectively against Hadoop and is
covered by the catalog
+ * Kerberos integration tests; the common module has no Hadoop on its test
classpath, so here we
+ * only assert that an hdfs uri without a Hadoop configuration is rejected.
+ */
public class TestFileFetcher {
- // The hdfs happy path is exercised reflectively against Hadoop and is
covered by the catalog
- // Kerberos integration tests; the common module has no Hadoop on its test
classpath, so here we
- // only assert that an hdfs uri without a Hadoop configuration is rejected.
@TempDir File tempDir;
@Test
@@ -67,6 +73,15 @@ public class TestFileFetcher {
srcFile.toPath().normalize(),
Files.readSymbolicLink(destFile.toPath()).normalize());
}
+ @Test
+ public void testOpaqueFileUriWithNullPathShouldFail() {
+ // An opaque file URI (no authority/path) must fail with a clear
IOException, not a raw NPE.
+ File destFile = new File(tempDir, "dest_opaque");
+ Assertions.assertThrows(
+ IOException.class,
+ () -> FileFetcher.get().fetchFileFromUri("file:relative", destFile,
10, null));
+ }
+
@Test
public void testMissingLocalFileShouldFail() {
File destFile = new File(tempDir, "dest_missing");
@@ -186,6 +201,74 @@ public class TestFileFetcher {
}
}
+ @Test
+ public void testRelativeLocalSourceProducesResolvableSymlink() throws
Exception {
+ File srcFile = new File(tempDir, "rel_src");
+ Assertions.assertTrue(srcFile.createNewFile());
+ File destFile = new File(tempDir, "rel_dest");
+ // A relative source path (as an operator might configure) must still
produce a symlink that
+ // resolves to the real file, not a dangling one.
+ Path cwd = Paths.get("").toAbsolutePath();
+ String relative =
cwd.relativize(srcFile.toPath().toAbsolutePath()).toString();
+
+ FileFetcher.get().fetchFileFromUri(relative, destFile, 10, null);
+
+ Assertions.assertTrue(Files.isSymbolicLink(destFile.toPath()));
+ Assertions.assertTrue(
+ Files.exists(destFile.toPath()), "symlink must resolve to an existing
file, not dangle");
+ }
+
+ @Test
+ public void testLocalFileMoveFailureCleansTempSymlink() throws Exception {
+ File srcFile = new File(tempDir, "src_movefail");
+ Assertions.assertTrue(srcFile.createNewFile());
+ // Make the destination a non-empty directory so the final atomic rename
fails.
+ File destDir = new File(tempDir, "dest_is_dir");
+ Assertions.assertTrue(destDir.mkdir());
+ Assertions.assertTrue(new File(destDir, "occupant").createNewFile());
+
+ Assertions.assertThrows(
+ IOException.class,
+ () -> FileFetcher.get().fetchFileFromUri(srcFile.toURI().toString(),
destDir, 10, null));
+
+ File tmpSymlink = new File(tempDir, "dest_is_dir.symlink.tmp");
+ Assertions.assertFalse(
+ tmpSymlink.exists(), "temp symlink must be cleaned up after a failed
move");
+ }
+
+ @Test
+ public void testUpperCaseSchemeIsCaseInsensitive() {
+ // RFC 3986 schemes are case-insensitive: "HTTP" must route like "http"
(i.e. through the SSRF
+ // validator), not fall through to the unsupported-scheme branch.
+ File destFile = new File(tempDir, "uppercase");
+ FileFetcher.get().initialize(true);
+
+ IllegalArgumentException exception =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ FileFetcher.get().fetchFileFromUri("HTTP://127.0.0.1/keytab",
destFile, 10, null));
+ Assertions.assertTrue(exception.getMessage().contains("Gravitino server
side"));
+ }
+
+ @Test
+ public void testFtpSchemeRejectedWhenBlockingEnabled() {
+ // FTP's data channel cannot be pinned to the validated address (PASV/EPSV
SSRF), so it must be
+ // rejected on the blocking path rather than handed to the JDK FTP client.
+ File destFile = new File(tempDir, "ftp");
+ FileFetcher.get().initialize(true);
+
+ IllegalArgumentException exception =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ FileFetcher.get()
+ .fetchFileFromUri("ftp://files.example.com/keytab";,
destFile, 10, null));
+ Assertions.assertTrue(exception.getMessage().contains("FTP"));
+ Assertions.assertTrue(
+
exception.getMessage().contains(FileFetcher.BLOCK_UNSAFE_REMOTE_URI_CONFIG));
+ }
+
@Test
public void testHdfsSchemeWithoutConfShouldFail() {
File destFile = new File(tempDir, "dest_hdfs");
diff --git
a/common/src/test/java/org/apache/gravitino/utils/TestRemoteFileDownloader.java
b/common/src/test/java/org/apache/gravitino/utils/TestRemoteFileDownloader.java
new file mode 100644
index 0000000000..5ce07e6840
--- /dev/null
+++
b/common/src/test/java/org/apache/gravitino/utils/TestRemoteFileDownloader.java
@@ -0,0 +1,686 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.utils;
+
+import com.sun.net.httpserver.HttpServer;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Tests for {@link RemoteFileDownloader}.
+ *
+ * <p>The {@code https} scheme is exercised end-to-end with TLS fixtures by
the catalog Kerberos
+ * integration tests. Here the {@code http} path is covered directly; the only
{@code
+ * https}-specific production logic is the small {@code startTls} step (SNI
and {@code HTTPS}
+ * endpoint identification over an already-pinned socket), which then reuses
exactly the same
+ * status/header/body parsing verified by these {@code http} tests.
+ */
+public class TestRemoteFileDownloader {
+
+ @TempDir File tempDir;
+
+ @Test
+ public void testHttpDownloadUsesPinnedAddressNotHostnameResolution() throws
Exception {
+ HttpServer server = createLoopbackServer("keytab-content", new
AtomicReference<>());
+ try {
+ server.start();
+ int port = server.getAddress().getPort();
+ File dest = new File(tempDir, "pinned");
+ // The host is a guaranteed-unresolvable name (RFC 6761 reserves
.invalid). A successful fetch
+ // therefore proves the download connected to the pinned loopback
address rather than
+ // re-resolving the hostname (which is the DNS-rebinding TOCTOU this
guards against).
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + port + "/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ RemoteFileDownloader.download(uri, pinned, dest, 30000);
+
+ Assertions.assertEquals("keytab-content",
Files.readString(dest.toPath()));
+ // The temp file must be atomically moved into place, leaving no ".tmp"
sibling behind.
+ File[] leftovers = tempDir.listFiles((d, name) -> name.endsWith(".tmp"));
+ Assertions.assertNotNull(leftovers);
+ Assertions.assertEquals(0, leftovers.length, "no temp file should remain
after success");
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ public void testHttpDownloadSendsOriginalHostHeader() throws Exception {
+ AtomicReference<String> hostHeader = new AtomicReference<>();
+ HttpServer server = createLoopbackServer("data", hostHeader);
+ try {
+ server.start();
+ int port = server.getAddress().getPort();
+ File dest = new File(tempDir, "host-header");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + port + "/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ RemoteFileDownloader.download(uri, pinned, dest, 30000);
+
+ // The original hostname must still be sent as the Host header, so
virtual-hosted servers
+ // continue to route correctly even though we connected to a pinned IP.
+ Assertions.assertEquals("ssrf-rebind.invalid:" + port, hostHeader.get());
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ public void testHttpDownloadRejectsNonSuccessStatus() throws Exception {
+ HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1",
0), 0);
+ server.createContext(
+ "/keytab",
+ exchange -> {
+ exchange.sendResponseHeaders(404, -1);
+ exchange.close();
+ });
+ try {
+ server.start();
+ int port = server.getAddress().getPort();
+ File dest = new File(tempDir, "missing");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + port + "/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ Assertions.assertThrows(
+ IOException.class, () -> RemoteFileDownloader.download(uri, pinned,
dest, 30000));
+ Assertions.assertFalse(dest.exists(), "no destination file should be
left on failure");
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ public void testHttpDownloadRejectsRedirect() throws Exception {
+ HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1",
0), 0);
+ server.createContext(
+ "/keytab",
+ exchange -> {
+ // A redirect target would re-resolve to an unvalidated address, so
it must be rejected.
+ exchange.getResponseHeaders().add("Location",
"http://example.com/elsewhere";);
+ exchange.sendResponseHeaders(302, -1);
+ exchange.close();
+ });
+ try {
+ server.start();
+ int port = server.getAddress().getPort();
+ File dest = new File(tempDir, "redirect");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + port + "/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ Assertions.assertThrows(
+ IOException.class, () -> RemoteFileDownloader.download(uri, pinned,
dest, 30000));
+ Assertions.assertFalse(dest.exists());
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ public void testHttpDownloadDecodesChunkedResponse() throws Exception {
+ // An explicit chunked response ("Wiki" + "pedia") must be de-chunked to
"Wikipedia"; if the
+ // chunk framing were written verbatim the file would be corrupted.
+ byte[] response =
+ ("HTTP/1.1 200 OK\r\n"
+ + "Transfer-Encoding: chunked\r\n"
+ + "Connection: close\r\n"
+ + "\r\n"
+ + "4\r\nWiki\r\n"
+ + "5\r\npedia\r\n"
+ + "0\r\n"
+ + "\r\n")
+ .getBytes(StandardCharsets.US_ASCII);
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, "chunked");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ RemoteFileDownloader.download(uri, pinned, dest, 30000);
+
+ Assertions.assertEquals("Wikipedia", Files.readString(dest.toPath()));
+ }
+ }
+
+ @Test
+ public void testChunkedFramingTakesPrecedenceOverContentLength() throws
Exception {
+ // TE.CL ambiguity (both Transfer-Encoding: chunked and Content-Length
present): chunked must
+ // win (RFC 7230), so the body de-chunks to "Wikipedia", not 4 bytes.
+ byte[] response =
+ ("HTTP/1.1 200 OK\r\n"
+ + "Transfer-Encoding: chunked\r\n"
+ + "Content-Length: 4\r\n"
+ + "Connection: close\r\n"
+ + "\r\n"
+ + "4\r\nWiki\r\n"
+ + "5\r\npedia\r\n"
+ + "0\r\n\r\n")
+ .getBytes(StandardCharsets.US_ASCII);
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, "te-cl");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ RemoteFileDownloader.download(uri, pinned, dest, 30000);
+
+ Assertions.assertEquals("Wikipedia", Files.readString(dest.toPath()));
+ }
+ }
+
+ @Test
+ public void testHttpDownloadRejectsAccumulatedOversizedHeaders() throws
Exception {
+ // Many header lines, each well under the per-line cap, whose total
exceeds the 64 KiB header
+ // cap must be rejected (a distinct guard from the per-line cap).
+ StringBuilder sb = new StringBuilder("HTTP/1.0 200 OK\r\n");
+ String value = new String(new char[1000]).replace('\0', 'A');
+ for (int i = 0; i < 70; i++) {
+ sb.append("X-Pad-").append(i).append(": ").append(value).append("\r\n");
+ }
+ sb.append("Content-Length: 4\r\nConnection: close\r\n\r\nbody");
+ byte[] response = sb.toString().getBytes(StandardCharsets.US_ASCII);
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, "header-flood");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ IOException exception =
+ Assertions.assertThrows(
+ IOException.class, () -> RemoteFileDownloader.download(uri,
pinned, dest, 30000));
+ Assertions.assertTrue(
+ exception.getMessage().contains("headers exceed the allowed size"),
+ exception.getMessage());
+ Assertions.assertFalse(dest.exists());
+ }
+ }
+
+ @Test
+ public void testHttpDownloadRejectsUnboundedChunkedTrailers() throws
Exception {
+ // After the terminating 0-chunk a malicious server floods trailer
headers; their total size
+ // must be bounded rather than read in an unbounded loop.
+ StringBuilder sb =
+ new StringBuilder(
+ "HTTP/1.1 200 OK\r\nTransfer-Encoding:
chunked\r\n\r\n4\r\nWiki\r\n0\r\n");
+ // ~70 KiB of trailer lines, each well under the per-line cap, exceeding
the 64 KiB total cap.
+ String trailerValue = new String(new char[1000]).replace('\0', 'A');
+ for (int i = 0; i < 70; i++) {
+ sb.append("X-Trailer: ").append(trailerValue).append("\r\n");
+ }
+ byte[] response = sb.toString().getBytes(StandardCharsets.US_ASCII);
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, "trailer-flood");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ Assertions.assertThrows(
+ IOException.class, () -> RemoteFileDownloader.download(uri, pinned,
dest, 30000));
+ Assertions.assertFalse(dest.exists());
+ }
+ }
+
+ @Test
+ public void testHttpDownloadAcceptsMultiSpaceStatusLine() throws Exception {
+ // RFC 7230 allows one SP, but some servers/proxies emit extra whitespace;
a valid 200 must
+ // still be accepted.
+ byte[] response =
+ ("HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection:
close\r\n\r\nabcd")
+ .getBytes(StandardCharsets.US_ASCII);
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, "multispace");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ RemoteFileDownloader.download(uri, pinned, dest, 30000);
+
+ Assertions.assertEquals("abcd", Files.readString(dest.toPath()));
+ }
+ }
+
+ @Test
+ public void testHttpDownloadRejectsTruncatedContentLength() throws Exception
{
+ // Content-Length promises 100 bytes but only 10 are delivered before the
connection closes;
+ // the partial download must be rejected, not silently accepted.
+ byte[] response =
+ ("HTTP/1.0 200 OK\r\nContent-Length: 100\r\nConnection:
close\r\n\r\n0123456789")
+ .getBytes(StandardCharsets.US_ASCII);
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, "truncated");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ Assertions.assertThrows(
+ IOException.class, () -> RemoteFileDownloader.download(uri, pinned,
dest, 30000));
+ Assertions.assertFalse(dest.exists(), "truncated download must not leave
a destination file");
+ // The staging temp file must be cleaned up on a failed download, not
orphaned.
+ File[] leftovers = tempDir.listFiles((d, name) -> name.endsWith(".tmp"));
+ Assertions.assertNotNull(leftovers);
+ Assertions.assertEquals(0, leftovers.length, "no temp file should remain
after a failure");
+ }
+ }
+
+ @Test
+ public void testHttpDownloadRejectsOversizedStatusLine() throws Exception {
+ // The status line is bounded only by readLine's per-line cap (the
header-total cap does not
+ // apply here), so a >64 KiB status line must be rejected by that specific
guard.
+ String huge = new String(new char[70_000]).replace('\0', 'A');
+ byte[] response = ("HTTP/1.1 200 " +
huge).getBytes(StandardCharsets.US_ASCII);
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, "huge-status");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ IOException exception =
+ Assertions.assertThrows(
+ IOException.class, () -> RemoteFileDownloader.download(uri,
pinned, dest, 30000));
+ Assertions.assertTrue(
+ exception.getMessage().contains("HTTP line exceeds the allowed
size"),
+ exception.getMessage());
+ Assertions.assertFalse(dest.exists());
+ }
+ }
+
+ @Test
+ public void testHttpDownloadRejectsObsoleteLineFolding() throws Exception {
+ // A header continuation line (obsolete line folding, leading SP/TAB) is a
smuggling vector and
+ // must be rejected. The continuation carries a colon so the no-colon
guard cannot mask the
+ // folding guard, and the message is asserted so only the folding guard
can satisfy the test.
+ byte[] response =
+ ("HTTP/1.0 200 OK\r\nX-Folded: a\r\n\tfoo: bar\r\nConnection:
close\r\n\r\nbody")
+ .getBytes(StandardCharsets.US_ASCII);
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, "folded");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ IOException exception =
+ Assertions.assertThrows(
+ IOException.class, () -> RemoteFileDownloader.download(uri,
pinned, dest, 30000));
+ Assertions.assertTrue(
+ exception.getMessage().contains("Obsolete line folding"),
exception.getMessage());
+ Assertions.assertFalse(dest.exists());
+ }
+ }
+
+ @Test
+ public void testHttpDownloadRejectsHeaderWithoutColon() throws Exception {
+ byte[] response =
+ ("HTTP/1.0 200 OK\r\nBadHeaderNoColon\r\nConnection:
close\r\n\r\nbody")
+ .getBytes(StandardCharsets.US_ASCII);
+ assertDownloadRejected(response, "no-colon");
+ }
+
+ @Test
+ public void testHttpDownloadRejectsNegativeChunkSize() throws Exception {
+ byte[] response =
+ ("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n-1\r\n")
+ .getBytes(StandardCharsets.US_ASCII);
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, "neg-chunk");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ IOException exception =
+ Assertions.assertThrows(
+ IOException.class, () -> RemoteFileDownloader.download(uri,
pinned, dest, 30000));
+ // Assert on the specific guard so it is independently pinned (a removed
guard would otherwise
+ // still throw later for a different reason).
+ Assertions.assertTrue(
+ exception.getMessage().contains("Negative chunk size"),
exception.getMessage());
+ Assertions.assertFalse(dest.exists());
+ }
+ }
+
+ @Test
+ public void testHttpDownloadRejectsNonNumericChunkSize() throws Exception {
+ // A non-hexadecimal chunk-size line must be rejected as an IOException,
not propagate a raw
+ // unchecked NumberFormatException out of the declared `throws
IOException` contract.
+ byte[] response =
+ ("HTTP/1.1 200 OK\r\nTransfer-Encoding:
chunked\r\n\r\nGG\r\nWiki\r\n0\r\n\r\n")
+ .getBytes(StandardCharsets.US_ASCII);
+ assertDownloadRejected(response, "bad-hex-chunk");
+ }
+
+ @Test
+ public void testHttpDownloadRejectsMalformedChunkTerminator() throws
Exception {
+ // After the 4-byte chunk data "Wiki" the terminator must be CRLF; "XX"
must be rejected.
+ byte[] response =
+ ("HTTP/1.1 200 OK\r\nTransfer-Encoding:
chunked\r\n\r\n4\r\nWikiXX\r\n0\r\n\r\n")
+ .getBytes(StandardCharsets.US_ASCII);
+ assertDownloadRejected(response, "bad-terminator");
+ }
+
+ @Test
+ public void testHttpDownloadRejectsMultiCodingTransferEncoding() throws
Exception {
+ // "gzip, chunked" is de-chunkable but still gzip-compressed; since we do
not gunzip,
+ // de-chunking
+ // alone would write corrupt bytes. Anything other than a sole "chunked"
coding must be
+ // rejected.
+ byte[] response =
+ ("HTTP/1.1 200 OK\r\n"
+ + "Transfer-Encoding: gzip, chunked\r\n"
+ + "Connection: close\r\n"
+ + "\r\n"
+ + "4\r\nWiki\r\n0\r\n\r\n")
+ .getBytes(StandardCharsets.US_ASCII);
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, "multi-coding");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ IOException exception =
+ Assertions.assertThrows(
+ IOException.class, () -> RemoteFileDownloader.download(uri,
pinned, dest, 30000));
+ Assertions.assertTrue(
+ exception.getMessage().contains("Unsupported Transfer-Encoding"),
exception.getMessage());
+ Assertions.assertFalse(dest.exists());
+ }
+ }
+
+ @Test
+ public void testHttpDownloadRejectsChunkedSubstringTransferEncoding() throws
Exception {
+ // "xchunked" contains the substring "chunked" but is not the chunked
coding; it must not be
+ // mistaken for it. The message is asserted so the unsupported-coding
guard, not a later chunk
+ // parse error, is what satisfies the test.
+ byte[] response =
+ ("HTTP/1.1 200 OK\r\n"
+ + "Transfer-Encoding: xchunked\r\n"
+ + "Connection: close\r\n"
+ + "\r\n"
+ + "4\r\nWiki\r\n0\r\n\r\n")
+ .getBytes(StandardCharsets.US_ASCII);
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, "x-chunked");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ IOException exception =
+ Assertions.assertThrows(
+ IOException.class, () -> RemoteFileDownloader.download(uri,
pinned, dest, 30000));
+ Assertions.assertTrue(
+ exception.getMessage().contains("Unsupported Transfer-Encoding"),
exception.getMessage());
+ Assertions.assertFalse(dest.exists());
+ }
+ }
+
+ @Test
+ public void testHttpDownloadRejectsConflictingDuplicateContentLength()
throws Exception {
+ // Two conflicting Content-Length values (RFC 7230 §3.3.2) are a
response-splitting/smuggling
+ // vector; a silent last-wins would let parsers disagree, so the conflict
is rejected outright.
+ byte[] response =
+ ("HTTP/1.0 200 OK\r\n"
+ + "Content-Length: 4\r\n"
+ + "Content-Length: 5\r\n"
+ + "Connection: close\r\n"
+ + "\r\n"
+ + "abcd")
+ .getBytes(StandardCharsets.US_ASCII);
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, "dup-cl");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ IOException exception =
+ Assertions.assertThrows(
+ IOException.class, () -> RemoteFileDownloader.download(uri,
pinned, dest, 30000));
+ Assertions.assertTrue(
+ exception.getMessage().contains("Conflicting duplicate"),
exception.getMessage());
+ Assertions.assertFalse(dest.exists());
+ }
+ }
+
+ @Test
+ public void testHttpDownloadRejectsConflictingDuplicateTransferEncoding()
throws Exception {
+ // Conflicting duplicate Transfer-Encoding headers (a last-wins "chunked"
would otherwise pass
+ // the chunked check) must be rejected as a smuggling vector before any
body decode.
+ byte[] response =
+ ("HTTP/1.1 200 OK\r\n"
+ + "Transfer-Encoding: gzip\r\n"
+ + "Transfer-Encoding: chunked\r\n"
+ + "Connection: close\r\n"
+ + "\r\n"
+ + "4\r\nWiki\r\n0\r\n\r\n")
+ .getBytes(StandardCharsets.US_ASCII);
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, "dup-te");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ IOException exception =
+ Assertions.assertThrows(
+ IOException.class, () -> RemoteFileDownloader.download(uri,
pinned, dest, 30000));
+ Assertions.assertTrue(
+ exception.getMessage().contains("Conflicting duplicate"),
exception.getMessage());
+ Assertions.assertFalse(dest.exists());
+ }
+ }
+
+ @Test
+ public void testHttpDownloadSendsHttp11Request() throws Exception {
+ AtomicReference<String> protocol = new AtomicReference<>();
+ HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1",
0), 0);
+ server.createContext(
+ "/keytab",
+ exchange -> {
+ protocol.set(exchange.getProtocol());
+ byte[] bytes = "data".getBytes(StandardCharsets.UTF_8);
+ exchange.sendResponseHeaders(200, bytes.length);
+ try (OutputStream os = exchange.getResponseBody()) {
+ os.write(bytes);
+ }
+ });
+ try {
+ server.start();
+ int port = server.getAddress().getPort();
+ File dest = new File(tempDir, "http11");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + port + "/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ RemoteFileDownloader.download(uri, pinned, dest, 30000);
+
+ // The request must be HTTP/1.1 so origins frame the body
(Content-Length or chunked) rather
+ // than relying on connection close, which the downloader rejects.
+ Assertions.assertEquals("HTTP/1.1", protocol.get());
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ private void assertDownloadRejected(byte[] response, String name) throws
Exception {
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, name);
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ Assertions.assertThrows(
+ IOException.class, () -> RemoteFileDownloader.download(uri, pinned,
dest, 30000));
+ Assertions.assertFalse(dest.exists());
+ }
+ }
+
+ @Test
+ public void testHttpDownloadRejectsUnframedResponse() throws Exception {
+ // No Content-Length and no chunked encoding: a close-delimited body
cannot be verified for
+ // completeness (a premature close looks identical to a complete body), so
it is rejected rather
+ // than risk installing a silently-truncated keytab/jar.
+ byte[] response =
+ ("HTTP/1.0 200 OK\r\nConnection: close\r\n\r\nclose-delimited-body")
+ .getBytes(StandardCharsets.US_ASCII);
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, "unframed");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ Assertions.assertThrows(
+ IOException.class, () -> RemoteFileDownloader.download(uri, pinned,
dest, 30000));
+ Assertions.assertFalse(dest.exists());
+ }
+ }
+
+ @Test
+ public void testHttpDownloadRejectsOversizedContentLength() throws Exception
{
+ // A Content-Length beyond the 2 GiB body cap must be rejected without
streaming anything.
+ byte[] response =
+ ("HTTP/1.0 200 OK\r\nContent-Length: 3000000000\r\nConnection:
close\r\n\r\n")
+ .getBytes(StandardCharsets.US_ASCII);
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, "too-big");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ IOException exception =
+ Assertions.assertThrows(
+ IOException.class, () -> RemoteFileDownloader.download(uri,
pinned, dest, 30000));
+ // Assert the specific reason so the Content-Length cap is pinned and a
competing truncation
+ // failure (read == -1 on the empty body) cannot masquerade as it.
+ Assertions.assertTrue(
+ exception.getMessage().contains("exceeds the maximum allowed size"),
+ exception.getMessage());
+ Assertions.assertFalse(dest.exists());
+ }
+ }
+
+ @Test
+ public void testHttpDownloadRejectsOversizedHeader() throws Exception {
+ // A single header line larger than the 64 KiB cap must be rejected, not
buffered unbounded.
+ StringBuilder hugeHeader = new StringBuilder("X-Big: ");
+ for (int i = 0; i < 70_000; i++) {
+ hugeHeader.append('A');
+ }
+ byte[] response =
+ ("HTTP/1.0 200 OK\r\n" + hugeHeader + "\r\nConnection:
close\r\n\r\nbody")
+ .getBytes(StandardCharsets.US_ASCII);
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, "huge-header");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ Assertions.assertThrows(
+ IOException.class, () -> RemoteFileDownloader.download(uri, pinned,
dest, 30000));
+ Assertions.assertFalse(dest.exists());
+ }
+ }
+
+ @Test
+ public void testHttpDownloadRejectsLeadingPlusContentLength() throws
Exception {
+ // Long.parseLong would accept "+5"; RFC 7230 forbids it. The response
must be rejected.
+ byte[] response =
+ ("HTTP/1.0 200 OK\r\nContent-Length: +5\r\nConnection:
close\r\n\r\nhello")
+ .getBytes(StandardCharsets.US_ASCII);
+ try (RawServer server = new RawServer(response)) {
+ File dest = new File(tempDir, "plus-cl");
+ URI uri = new URI("http://ssrf-rebind.invalid:"; + server.port() +
"/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ Assertions.assertThrows(
+ IOException.class, () -> RemoteFileDownloader.download(uri, pinned,
dest, 30000));
+ Assertions.assertFalse(dest.exists());
+ }
+ }
+
+ @Test
+ public void testUnsupportedSchemeRejected() throws Exception {
+ File dest = new File(tempDir, "scp");
+ URI uri = new URI("scp://host:22/keytab");
+ InetAddress pinned = InetAddress.getByName("127.0.0.1");
+
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> RemoteFileDownloader.download(uri, pinned, dest, 30000));
+ }
+
+ @Test
+ public void testHelpersFormatHostHeaderAndTargets() throws Exception {
+ Assertions.assertEquals("h:8080", RemoteFileDownloader.hostHeader("h",
8080));
+ Assertions.assertEquals("h", RemoteFileDownloader.hostHeader("h", -1));
+ Assertions.assertEquals("/", RemoteFileDownloader.requestTarget(new
URI("http://h";)));
+ Assertions.assertEquals(
+ "/p?q=1", RemoteFileDownloader.requestTarget(new
URI("http://h/p?q=1";)));
+ }
+
+ private HttpServer createLoopbackServer(String response,
AtomicReference<String> hostHeader)
+ throws Exception {
+ HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1",
0), 0);
+ server.createContext(
+ "/keytab",
+ exchange -> {
+ hostHeader.set(exchange.getRequestHeaders().getFirst("Host"));
+ byte[] bytes = response.getBytes(StandardCharsets.UTF_8);
+ exchange.sendResponseHeaders(200, bytes.length);
+ try (OutputStream os = exchange.getResponseBody()) {
+ os.write(bytes);
+ }
+ });
+ return server;
+ }
+
+ /** A single-shot loopback server that writes a fixed raw response, for
byte-level control. */
+ private static final class RawServer implements AutoCloseable {
+ private final ServerSocket serverSocket;
+ private final Thread thread;
+
+ RawServer(byte[] response) throws IOException {
+ this.serverSocket = new ServerSocket(0, 1,
InetAddress.getByName("127.0.0.1"));
+ this.thread =
+ new Thread(
+ () -> {
+ try (Socket socket = serverSocket.accept()) {
+ drainRequest(socket.getInputStream());
+ socket.getOutputStream().write(response);
+ socket.getOutputStream().flush();
+ } catch (IOException ignored) {
+ // The client may close early; nothing to do.
+ }
+ });
+ this.thread.setDaemon(true);
+ this.thread.start();
+ }
+
+ int port() {
+ return serverSocket.getLocalPort();
+ }
+
+ private static void drainRequest(InputStream in) throws IOException {
+ int state = 0; // counts how far we are through the terminating \r\n\r\n
sequence
+ int b;
+ while (state < 4 && (b = in.read()) != -1) {
+ boolean expectCr = state == 0 || state == 2;
+ if (expectCr) {
+ state = (b == '\r') ? state + 1 : 0;
+ } else {
+ state = (b == '\n') ? state + 1 : 0;
+ }
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ serverSocket.close();
+ }
+ }
+}
diff --git
a/common/src/test/java/org/apache/gravitino/utils/TestRemoteUriValidator.java
b/common/src/test/java/org/apache/gravitino/utils/TestRemoteUriValidator.java
index 5afd96fe82..0a751210c8 100644
---
a/common/src/test/java/org/apache/gravitino/utils/TestRemoteUriValidator.java
+++
b/common/src/test/java/org/apache/gravitino/utils/TestRemoteUriValidator.java
@@ -18,6 +18,7 @@
*/
package org.apache.gravitino.utils;
+import java.net.InetAddress;
import java.net.URI;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -26,66 +27,81 @@ public class TestRemoteUriValidator {
private static final String BLOCK_UNSAFE_ADDRESS_CONFIG =
"test.block-unsafe-address";
@Test
- public void testRejectLocalAddressesByDefault() {
+ public void testResolveAndValidateReturnsAddressForSafeHost() throws
Exception {
+ // A public literal IP resolves to itself (no DNS) and is safe, so it must
be returned for the
+ // caller to pin the subsequent download to.
+ InetAddress address =
+ RemoteUriValidator.resolveAndValidate(
+ new URI("http://8.8.8.8/file";), BLOCK_UNSAFE_ADDRESS_CONFIG);
+ Assertions.assertEquals("8.8.8.8", address.getHostAddress());
+ }
+
+ @Test
+ public void testResolveAndValidateRejectsUnsafeHost() {
IllegalArgumentException exception =
Assertions.assertThrows(
IllegalArgumentException.class,
() ->
- RemoteUriValidator.validate(
- new URI("http://127.0.0.1/";), true,
BLOCK_UNSAFE_ADDRESS_CONFIG));
+ RemoteUriValidator.resolveAndValidate(
+ new URI("http://127.0.0.1/file";),
BLOCK_UNSAFE_ADDRESS_CONFIG));
Assertions.assertTrue(exception.getMessage().contains("Gravitino server
side"));
Assertions.assertTrue(exception.getMessage().contains(BLOCK_UNSAFE_ADDRESS_CONFIG));
+ }
- Assertions.assertThrows(
- IllegalArgumentException.class,
- () ->
- RemoteUriValidator.validate(
- new URI("http://localhost/";), true,
BLOCK_UNSAFE_ADDRESS_CONFIG));
- Assertions.assertThrows(
- IllegalArgumentException.class,
- () ->
- RemoteUriValidator.validate(
- new URI("http://169.254.169.254/";), true,
BLOCK_UNSAFE_ADDRESS_CONFIG));
- Assertions.assertThrows(
- IllegalArgumentException.class,
- () ->
- RemoteUriValidator.validate(
- new URI("http://10.0.0.1/";), true,
BLOCK_UNSAFE_ADDRESS_CONFIG));
- Assertions.assertThrows(
- IllegalArgumentException.class,
- () ->
- RemoteUriValidator.validate(
- new URI("http://172.16.0.1/";), true,
BLOCK_UNSAFE_ADDRESS_CONFIG));
- Assertions.assertThrows(
- IllegalArgumentException.class,
- () ->
- RemoteUriValidator.validate(
- new URI("http://192.168.0.1/";), true,
BLOCK_UNSAFE_ADDRESS_CONFIG));
- Assertions.assertThrows(
- IllegalArgumentException.class,
- () ->
- RemoteUriValidator.validate(
- new URI("http://100.100.100.200/";), true,
BLOCK_UNSAFE_ADDRESS_CONFIG));
- Assertions.assertThrows(
- IllegalArgumentException.class,
- () ->
- RemoteUriValidator.validate(
- new URI("http://[fd00::1]/";), true,
BLOCK_UNSAFE_ADDRESS_CONFIG));
+ @Test
+ public void testRejectLocalAddressesByDefault() {
+ // Loopback, link-local (incl. the AWS/cloud metadata 169.254.169.254),
RFC 1918 private ranges,
+ // Alibaba/Oracle metadata, CGNAT, broadcast, "this network", multicast,
and IPv6 unique-local
+ // must all be rejected.
+ String[] unsafeHosts = {
+ "127.0.0.1",
+ "localhost",
+ "169.254.169.254",
+ "10.0.0.1",
+ "172.16.0.1",
+ "192.168.0.1",
+ "100.100.100.200",
+ "100.64.0.1",
+ "192.0.0.192",
+ "255.255.255.255",
+ "0.0.0.1",
+ "224.0.0.1",
+ "[ff02::1]",
+ "[fd00::1]",
+ // IPv4-compatible, NAT64, 6to4 and ISATAP IPv6 forms that embed a
blocked IPv4 are rejected.
+ "[::127.0.0.1]",
+ "[::169.254.169.254]",
+ "[64:ff9b::169.254.169.254]",
+ "[2002:a9fe:a9fe::]",
+ "[2001:db8::5efe:169.254.169.254]"
+ };
+ for (String host : unsafeHosts) {
+ IllegalArgumentException exception =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ RemoteUriValidator.resolveAndValidate(
+ new URI("http://"; + host + "/"),
BLOCK_UNSAFE_ADDRESS_CONFIG),
+ "Expected " + host + " to be rejected");
+ Assertions.assertTrue(exception.getMessage().contains("Gravitino server
side"));
+
Assertions.assertTrue(exception.getMessage().contains(BLOCK_UNSAFE_ADDRESS_CONFIG));
+ }
+ }
+
+ @Test
+ public void testResolveAndValidateAllowsGlobalIpv6() throws Exception {
+ // A normal global IPv6 host (no embedded blocked IPv4) must not be
over-blocked by the
+ // embedded-IPv4 re-classification.
+ InetAddress address =
+ RemoteUriValidator.resolveAndValidate(
+ new URI("http://[2001:4860:4860::8888]/file";),
BLOCK_UNSAFE_ADDRESS_CONFIG);
+ Assertions.assertNotNull(address);
}
@Test
- public void testAllowUnsafeAddressesWhenBlockingDisabled() {
- Assertions.assertDoesNotThrow(
- () ->
- RemoteUriValidator.validate(
- new URI("http://127.0.0.1/";), false,
BLOCK_UNSAFE_ADDRESS_CONFIG));
- Assertions.assertDoesNotThrow(
- () ->
- RemoteUriValidator.validate(
- new URI("http://localhost/";), false,
BLOCK_UNSAFE_ADDRESS_CONFIG));
- Assertions.assertDoesNotThrow(
- () ->
- RemoteUriValidator.validate(
- new URI("http://192.168.0.1/";), false,
BLOCK_UNSAFE_ADDRESS_CONFIG));
+ public void testRejectMissingHost() {
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> RemoteUriValidator.resolveAndValidate(new URI("file:///tmp/x"),
"hint"));
}
}
diff --git a/common/src/test/java/org/apache/gravitino/utils/TestSafeUri.java
b/common/src/test/java/org/apache/gravitino/utils/TestSafeUri.java
new file mode 100644
index 0000000000..0db804fcbb
--- /dev/null
+++ b/common/src/test/java/org/apache/gravitino/utils/TestSafeUri.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.utils;
+
+import java.net.URI;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestSafeUri {
+
+ @Test
+ public void testDropsUserInfo() throws Exception {
+ String redacted = SafeUri.redact(new
URI("ftp://user:s3cr3t@host:21/keytab";));
+ Assertions.assertEquals("ftp://host:21/keytab";, redacted);
+ Assertions.assertFalse(redacted.contains("s3cr3t"));
+ Assertions.assertFalse(redacted.contains("user"));
+ }
+
+ @Test
+ public void testDropsQueryToken() throws Exception {
+ String redacted = SafeUri.redact(new
URI("https://host/path/file.jar?token=SECRET&x=1";));
+ Assertions.assertEquals("https://host/path/file.jar";, redacted);
+ Assertions.assertFalse(redacted.contains("SECRET"));
+ }
+
+ @Test
+ public void testKeepsSchemeHostPathForHostlessUri() throws Exception {
+ Assertions.assertEquals("file:///tmp/x", SafeUri.redact(new
URI("file:///tmp/x")));
+ }
+
+ @Test
+ public void testNullUri() {
+ Assertions.assertEquals("null", SafeUri.redact(null));
+ }
+}
