LuciferYang commented on code in PR #11785:
URL: https://github.com/apache/gravitino/pull/11785#discussion_r3506307866
##########
common/src/main/java/org/apache/gravitino/utils/RemoteUriValidator.java:
##########
@@ -71,19 +74,122 @@ private static boolean isUnsafeAddress(InetAddress
address) {
}
byte[] bytes = address.getAddress();
- if (isCloudMetadataAddress(bytes)) {
+ if (isUnsafeIpv4Address(bytes)) {
return true;
}
+ if (bytes.length == 16) {
+ if (isIpv6UniqueLocalAddress(bytes)) {
+ return true;
+ }
+ // Several IPv6 forms embed an IPv4 address the platform checks above
miss (IPv4-compatible,
+ // NAT64, 6to4, ISATAP; plus IPv4-mapped defensively). Re-classify the
embedded IPv4 so a
+ // blocked address such as cloud metadata cannot be reached via an IPv6
literal.
+ byte[] embeddedIpv4 = embeddedIpv4(bytes);
+ if (embeddedIpv4 != null) {
+ try {
+ return isUnsafeAddress(InetAddress.getByAddress(embeddedIpv4));
+ } catch (UnknownHostException e) {
+ // getByAddress only rejects a wrong-length array; a 4-byte array
never reaches here. Fail
+ // closed if it somehow does.
+ return true;
+ }
+ }
+ }
+ return false;
+ }
- return isIpv6UniqueLocalAddress(bytes);
+ /**
+ * Returns the IPv4 address embedded in a 16-byte IPv6 address whose
embedded IPv4 is the literal
+ * connection target, or {@code null} if there is none. Covers the
IPv4-compatible ({@code
+ * ::a.b.c.d}), NAT64 ({@code 64:ff9b::a.b.c.d}, RFC 6052), 6to4 ({@code
2002:a.b.c.d::}, RFC
+ * 3056) and ISATAP ({@code ::0:5efe:a.b.c.d}, RFC 5214) forms. The
IPv4-mapped ({@code
+ * ::ffff:a.b.c.d}) form is handled defensively: the JDK normally surfaces
it as a 4-byte {@link
+ * java.net.Inet4Address} validated by {@link #isUnsafeIpv4Address}, so this
branch is a fallback.
+ *
+ * <p>Teredo ({@code 2001:0::/32}) is intentionally excluded: its embedded
IPv4 is the
+ * client/relay identifier rather than the connection destination, and it
requires a non-default
+ * Teredo tunnel to route at all.
+ */
+ private static byte[] embeddedIpv4(byte[] bytes) {
+ boolean highTenZero = true;
+ for (int i = 0; i < 10; i++) {
+ if (bytes[i] != 0) {
+ highTenZero = false;
+ break;
+ }
+ }
+ if (highTenZero) {
+ boolean compatible = bytes[10] == 0 && bytes[11] == 0;
+ // Defensive: the JDK normally collapses IPv4-mapped addresses to a
4-byte Inet4Address.
+ boolean mapped = (bytes[10] & 0xFF) == 0xFF && (bytes[11] & 0xFF) ==
0xFF;
+ if (compatible || mapped) {
+ return lowestFourBytes(bytes);
+ }
+ }
+ // NAT64 well-known prefix 64:ff9b::/96 (RFC 6052): the IPv4 is the low 32
bits. Only the
+ // well-known prefix is matched; network-specific prefixes (RFC 6052 §2.2,
e.g. RFC 8215's
+ // 64:ff9b:1::/48) are site-defined and routable only where explicitly
deployed, so detecting
+ // them would require configuration this validator does not have.
+ if ((bytes[0] & 0xFF) == 0x00
+ && (bytes[1] & 0xFF) == 0x64
+ && (bytes[2] & 0xFF) == 0xFF
+ && (bytes[3] & 0xFF) == 0x9B
+ && isZero(bytes, 4, 12)) {
+ return lowestFourBytes(bytes);
+ }
+ // 6to4 (2002::/16, RFC 3056): the gateway IPv4 is the 32 bits after the
prefix. 2002::/16 is
+ // reserved exclusively for 6to4, so no legitimate non-6to4 host occupies
it.
+ if ((bytes[0] & 0xFF) == 0x20 && (bytes[1] & 0xFF) == 0x02) {
+ return new byte[] {bytes[2], bytes[3], bytes[4], bytes[5]};
+ }
+ // ISATAP interface identifier (RFC 5214): the low 64 bits are
00:00:5e:fe:a.b.c.d or
+ // 02:00:5e:fe:a.b.c.d, embedding the IPv4 in the low 32 bits regardless
of the /64 prefix.
+ if ((bytes[8] & 0xFD) == 0x00
+ && bytes[9] == 0x00
+ && (bytes[10] & 0xFF) == 0x5E
+ && (bytes[11] & 0xFF) == 0xFE) {
+ return lowestFourBytes(bytes);
+ }
+ return null;
+ }
+
+ private static byte[] lowestFourBytes(byte[] bytes) {
+ return new byte[] {bytes[12], bytes[13], bytes[14], bytes[15]};
+ }
+
+ private static boolean isZero(byte[] bytes, int fromInclusive, int
toExclusive) {
+ for (int i = fromInclusive; i < toExclusive; i++) {
+ if (bytes[i] != 0) {
+ return false;
+ }
+ }
+ return true;
}
- private static boolean isCloudMetadataAddress(byte[] bytes) {
- return bytes.length == 4
- && (bytes[0] & 0xFF) == 100
- && (bytes[1] & 0xFF) == 100
- && (bytes[2] & 0xFF) == 100
- && (bytes[3] & 0xFF) == 200;
+ private static boolean isUnsafeIpv4Address(byte[] bytes) {
+ if (bytes.length != 4) {
+ return false;
+ }
+ int b0 = bytes[0] & 0xFF;
+ int b1 = bytes[1] & 0xFF;
+ int b2 = bytes[2] & 0xFF;
+ int b3 = bytes[3] & 0xFF;
+
+ // 0.0.0.0/8 "this network" (RFC 1122). isAnyLocalAddress only covers the
single 0.0.0.0.
+ if (b0 == 0) {
+ return true;
+ }
+ // 100.64.0.0/10 carrier-grade NAT / shared address space (RFC 6598). This
range also covers
+ // the Alibaba Cloud metadata endpoint 100.100.100.200. Not caught by
isSiteLocalAddress.
+ if (b0 == 100 && (b1 & 0xC0) == 0x40) {
Review Comment:
`100.64.0.0/10` (RFC 6598) has a 10-bit prefix, so its range is
`100.64.0.0`–`100.127.255.255` — the second octet runs 64–127, not just 64.
`(b1 & 0xC0) == 0x40` tests the top two bits to cover that whole range; `b1 ==
0x40` would only match `100.64.0.0/16`.
The address that motivated this entry — the Alibaba Cloud ECS metadata IP
`100.100.100.200` ([Alibaba Cloud
docs](https://help.aliyun.com/en/ecs/user-guide/view-instance-metadata/)) — has
second octet 100, so `b1 == 0x40` would let it through. Added a comment making
the 10-bit prefix explicit.
##########
common/src/main/java/org/apache/gravitino/utils/RemoteFileDownloader.java:
##########
@@ -0,0 +1,396 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.utils;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import javax.net.ssl.SSLParameters;
+import javax.net.ssl.SSLSocket;
+import javax.net.ssl.SSLSocketFactory;
+
+/**
+ * Downloads an {@code http} or {@code https} file while pinning the TCP
connection to a
+ * pre-validated address.
+ *
+ * <p>This closes the DNS-rebinding TOCTOU window between SSRF validation and
the actual fetch: the
+ * host is resolved and validated once by {@link
RemoteUriValidator#resolveAndValidate}, and the
+ * resulting {@link InetAddress} is connected to directly here, so the
hostname is never
+ * re-resolved. A minimal HTTP/1.1 client is used so the connection can be
pinned at the socket
+ * level (the JDK exposes no per-connection address override for plain {@code
HttpURLConnection},
+ * and silently drops a {@code Host} header set via {@code
setRequestProperty}). The original
+ * hostname is still used for the HTTP {@code Host} header and, for {@code
https}, for TLS SNI and
+ * certificate-hostname verification, so virtual hosting and certificate
validation keep working.
+ * Redirects are not followed, because a redirect target would re-resolve to
an unvalidated address.
+ *
+ * <p>{@code ftp} is intentionally not handled here: the FTP data channel is
opened to an address
+ * the server chooses in its {@code PASV}/{@code EPSV} reply, which cannot be
pinned, so {@link
+ * FileFetcher} rejects {@code ftp} on the SSRF-blocking path instead.
+ *
+ * <p>Only responses framed by {@code Content-Length} or chunked
transfer-encoding are accepted; a
+ * connection-close-delimited body is rejected because a premature close
cannot be told apart from a
+ * complete one. Combined with streaming to a sibling temporary file that is
atomically moved into
+ * place only on success, this means a connection that drops mid-transfer
never leaves a truncated
+ * destination behind. Response and header sizes are bounded to prevent a
malicious pinned host from
+ * exhausting disk or memory.
+ */
+final class RemoteFileDownloader {
+
+ private static final int DEFAULT_HTTP_PORT = 80;
+ private static final int DEFAULT_HTTPS_PORT = 443;
+ private static final int MAX_HEADER_BYTES = 64 * 1024;
+ private static final long MAX_BODY_BYTES = 2L * 1024 * 1024 * 1024;
Review Comment:
Added a comment noting the cap can be raised for larger files.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
