tustvold commented on code in PR #348: URL: https://github.com/apache/arrow-rs-object-store/pull/348#discussion_r2080463347
########## src/registry.rs: ########## @@ -0,0 +1,243 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! ObjectStoreRegistry holds all the object stores at Runtime with a scheme for each store. +//! This allows the user to resolve a URL to an ObjectStore at runtime. Unlike +//! [`crate::parse_url`], this allows for custom logic to be executed +//! when a URL is resolved to an ObjectStore. It also serves as a cache for object stores +//! to avoid repeated creation. + +use crate::{parse_url, ObjectStore}; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; +use url::Url; + +/// [`ObjectStoreRegistry`] maps a URL to an [`ObjectStore`] instance, +/// and allows users to read from different [`ObjectStore`] +/// instances. For example the registry might be configured so that +/// +/// 1. `s3://my_bucket/lineitem/` mapped to the `/lineitem` path on an +/// AWS S3 object store bound to `my_bucket` +/// +/// 2. `s3://my_other_bucket/lineitem/` mapped to the (same) +/// `/lineitem` path on a *different* AWS S3 object store bound to +/// `my_other_bucket` +/// +/// In this particular case, the url `s3://my_bucket/lineitem/` will be provided to +/// [`ObjectStoreRegistry::get_store`] and one of three things will happen: +/// +/// - If an [`ObjectStore`] has been registered with [`ObjectStoreRegistry::register_store`] with +/// `s3://my_bucket`, that [`ObjectStore`] will be returned +/// +/// - If an AWS S3 object store can be ad-hoc discovered by the url `s3://my_bucket/lineitem/`, this +/// object store will be registered with key `s3://my_bucket` and returned. +/// +/// - Otherwise `None` will be returned, indicating that no suitable [`ObjectStore`] could +/// be found +/// +/// This allows for two different use-cases: +/// +/// 1. Systems where object store buckets are explicitly created using DDL, can register these +/// buckets using [`ObjectStoreRegistry::register_store`] +/// +/// 2. Systems relying on ad-hoc discovery, without corresponding DDL, can create [`ObjectStore`] +/// lazily by providing a custom implementation of [`ObjectStoreRegistry`] +pub trait ObjectStoreRegistry: Send + Sync + std::fmt::Debug + 'static { + /// If a store with the same key existed before, it is replaced and returned + fn register_store( + &self, + url: &Url, + store: Arc<dyn ObjectStore>, + ) -> Option<Arc<dyn ObjectStore>>; + + /// Get a suitable store for the provided URL. For example: + /// + /// - URL with scheme `file:///` or no scheme will return the default LocalFS store + /// - URL with scheme `s3://bucket/` will return the S3 store + /// - URL with scheme `hdfs://hostname:port/` will return the hdfs store + /// + /// If no [`ObjectStore`] found for the `url`, ad-hoc discovery may be executed depending on + /// the `url` and [`ObjectStoreRegistry`] implementation. An [`ObjectStore`] may be lazily + /// created and registered. + fn get_store(&self, url: &Url) -> Option<Arc<dyn ObjectStore>>; + + /// List all registered URLs + fn list_urls(&self) -> Vec<String>; Review Comment: This API doesn't really make sense to me, URLs of what? Stores aren't typically keyed by URL... -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
