OIiveirra commented on code in PR #69527:
URL: https://github.com/apache/doris/pull/69527#discussion_r4365918709
##########
fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java:
##########
@@ -89,6 +106,32 @@ public S3TableValuedFunction(Map<String, String>
properties, List<TBrokerFileSta
}
+ private static boolean isRemoteLanceUri(String path) {
+ try {
+ URI uri = new URI(path == null ? "" : path);
+ String scheme = uri.getScheme();
+ String authority = uri.getRawAuthority();
+ if (scheme == null ||
!LANCE_REMOTE_SCHEMES.contains(scheme.toLowerCase(Locale.ROOT))
+ || uri.isOpaque()) {
+ return false;
+ }
+ if (authority == null || authority.isEmpty()) {
+ // GooseFS can resolve its remote master from storage options
or configuration.
+ return "goosefs".equalsIgnoreCase(scheme) && uri.getRawPath()
!= null
+ && uri.getRawPath().startsWith("/");
+ }
+ // Lance falls back to a local path if Rust's URL parser rejects
the URI. Java's URI
+ // accepts registry authorities with invalid ports, which Rust
rejects. Validate the
+ // host/port separately without disallowing legitimate bucket
names with underscores.
+ String host =
HostAndPort.fromString(authority.substring(authority.lastIndexOf('@') + 1))
+ .requireBracketsForIPv6().getHost();
+ // Java accepts IPv6 zone identifiers, while Rust's URL parser
rejects them.
+ return !host.isEmpty() && !(host.contains(":") &&
host.contains("%"));
Review Comment:
Fixed in 90a3a93aae68c021024842b1f19e32965e730508. The final remote Lance
URI guard now validates the original port spelling as ASCII digits, rejecting
signed zero before metadata loading. IPv6 is also validated without DNS using
`InetAddresses`, rejecting the leading-zero IPv4-tail case.
`S3TableValuedFunctionTest` covers `:-0`, `:-00`, `:+0`, the malformed
mapped IPv6 address, and valid port/IPv6 cases. Rejected inputs do not reach
the mocked metadata loader. All 35 selected FE tests passed.
##########
be/src/format_v2/table/lance_reader.cpp:
##########
@@ -35,11 +35,36 @@
#include "format_v2/lance/lance_reader_helper.h"
#include "format_v2/lance/lance_runtime_filter_helper.h"
#include "format_v2/lance/lance_session_manager.h"
+#include "io/fs/local_file_system.h"
#include "runtime/file_scan_profile.h"
#include "runtime/runtime_state.h"
namespace doris::format::lance {
+namespace {
+Status resolve_dataset_uri(const TFileRangeDesc& range, const
TFileScanRangeParams& scan_params,
+ std::string* uri) {
+ const auto& requested_uri =
range.table_format_params.lance_params.dataset_uri;
+ // Modern plans tag each range; older plans put the type in the scan
parameters.
+ // Untagged catalog scans retain their original storage-provider selection.
+ const bool is_local = range.__isset.file_type
+ ? range.file_type == TFileType::FILE_LOCAL
+ : scan_params.__isset.file_type &&
+ scan_params.file_type ==
TFileType::FILE_LOCAL;
+ // LOCAL TVF uses version zero for both schema discovery and whole-dataset
scans.
+ // Catalog scans pin a positive snapshot version, including file://
catalogs whose
+ // operator-configured warehouse is independent of the LOCAL TVF secure
directory.
+ if (is_local && range.table_format_params.lance_params.version == 0) {
+ // LOCAL Lance bypasses glob because it opens a dataset directory.
Apply the same
+ // canonical secure-root check before either schema discovery or a
cached dataset open.
+ return io::global_local_filesystem()->canonicalize_local_file(
Review Comment:
Fixed in 90a3a93aae68c021024842b1f19e32965e730508. LOCAL path validation now
decodes actual `file:` URI paths once before canonicalization, after removing
raw query/fragment components. Bare paths retain literal percent characters and
`+`; decoded NUL and out-of-root targets are rejected.
`LanceTableReaderLocalPathTest.FileUrisPreserveEscapedDatasetIdentity`
checks the canonical dataset key, real schema discovery and scanning for
escaped-space/literal-percent collisions, `%2520`, plus/percent names and
query/fragment handling. All 11 selected BE filesystem/Lance tests passed under
Clang21/ASAN; scans returned row IDs {1,2,3,4}.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
