yuqi1129 commented on code in PR #11785: URL: https://github.com/apache/gravitino/pull/11785#discussion_r3504898733
########## common/src/main/java/org/apache/gravitino/utils/RemoteFileDownloader.java: ########## @@ -0,0 +1,396 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.utils; + +import java.io.BufferedInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; + +/** + * Downloads an {@code http} or {@code https} file while pinning the TCP connection to a + * pre-validated address. + * + * <p>This closes the DNS-rebinding TOCTOU window between SSRF validation and the actual fetch: the + * host is resolved and validated once by {@link RemoteUriValidator#resolveAndValidate}, and the + * resulting {@link InetAddress} is connected to directly here, so the hostname is never + * re-resolved. A minimal HTTP/1.1 client is used so the connection can be pinned at the socket + * level (the JDK exposes no per-connection address override for plain {@code HttpURLConnection}, + * and silently drops a {@code Host} header set via {@code setRequestProperty}). The original + * hostname is still used for the HTTP {@code Host} header and, for {@code https}, for TLS SNI and + * certificate-hostname verification, so virtual hosting and certificate validation keep working. + * Redirects are not followed, because a redirect target would re-resolve to an unvalidated address. + * + * <p>{@code ftp} is intentionally not handled here: the FTP data channel is opened to an address + * the server chooses in its {@code PASV}/{@code EPSV} reply, which cannot be pinned, so {@link + * FileFetcher} rejects {@code ftp} on the SSRF-blocking path instead. + * + * <p>Only responses framed by {@code Content-Length} or chunked transfer-encoding are accepted; a + * connection-close-delimited body is rejected because a premature close cannot be told apart from a + * complete one. Combined with streaming to a sibling temporary file that is atomically moved into + * place only on success, this means a connection that drops mid-transfer never leaves a truncated + * destination behind. Response and header sizes are bounded to prevent a malicious pinned host from + * exhausting disk or memory. + */ +final class RemoteFileDownloader { + + private static final int DEFAULT_HTTP_PORT = 80; + private static final int DEFAULT_HTTPS_PORT = 443; + private static final int MAX_HEADER_BYTES = 64 * 1024; + private static final long MAX_BODY_BYTES = 2L * 1024 * 1024 * 1024; Review Comment: Better to add comments like `we can change this value if a larger file is needed` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
