comphead commented on code in PR #5503:
URL: https://github.com/apache/datafusion-comet/pull/5503#discussion_r3887041871
##########
native/core/src/parquet/parquet_support.rs:
##########
@@ -581,28 +584,262 @@ pub(crate) fn prepare_object_store_with_configs(
(store, path)
};
- let object_store_url = ObjectStoreUrl::parse(url_key.clone())?;
- runtime_env.register_object_store(&url, object_store);
+ // A RuntimeEnv can plan multiple scans with different backends or
credentials
+ // for the same bucket. Use the same identity as the cache, even for the
first
+ // registration, so neither later registration nor planning order changes
the
+ // store used by an existing scan. Native s3/s3a share the normalized s3
scheme;
+ // a Hadoop-selected scheme retains its physical spelling.
+ //
+ // Native LocalFileSystem ignores these Hadoop options and keeps file://
for
+ // compatibility. An explicitly Hadoop-routed file scheme is still
isolated.
+ let object_store_url = if scheme == "file" && !is_hdfs_scheme {
+ ObjectStoreUrl::parse(url_key)?
+ } else {
+ let backend = if is_hdfs_scheme { "hdfs" } else { "native" };
+ ObjectStoreUrl::parse(format!(
+ "{scheme}+comet-{config_hash:016x}-{backend}://{}",
Review Comment:
why would comet be a hardcoded part of url?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
