goldmedal commented on code in PR #11035: URL: https://github.com/apache/datafusion/pull/11035#discussion_r1748371500
########## datafusion/catalog/src/dynamic_file/catalog.rs: ########## @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! dynamic_file contains [`DynamicFileCatalog`] that creates tables from file paths +//! if the wrapped [`CatalogProviderList`] doesn't have the table provider. + +use crate::{CatalogProvider, CatalogProviderList, SchemaProvider, TableProvider}; +use async_trait::async_trait; +use std::any::Any; +use std::sync::Arc; + +/// Wrap another catalog provider list +pub struct DynamicFileCatalog { + /// The inner catalog provider list + inner: Arc<dyn CatalogProviderList>, + /// The factory that can create a table provider from the file path + factory: Arc<dyn UrlTableFactory>, +} + +impl DynamicFileCatalog { + pub fn new( + inner: Arc<dyn CatalogProviderList>, + factory: Arc<dyn UrlTableFactory>, + ) -> Self { + Self { inner, factory } + } +} + +impl CatalogProviderList for DynamicFileCatalog { + fn as_any(&self) -> &dyn Any { + self + } + + fn register_catalog( + &self, + name: String, + catalog: Arc<dyn CatalogProvider>, + ) -> Option<Arc<dyn CatalogProvider>> { + self.inner.register_catalog(name, catalog) + } + + fn catalog_names(&self) -> Vec<String> { + self.inner.catalog_names() + } + + fn catalog(&self, name: &str) -> Option<Arc<dyn CatalogProvider>> { + self.inner.catalog(name).map(|catalog| { + Arc::new(DynamicFileCatalogProvider::new( + catalog, + Arc::clone(&self.factory), + )) as _ + }) + } +} + +/// Wraps another catalog provider +struct DynamicFileCatalogProvider { + /// The inner catalog provider + inner: Arc<dyn CatalogProvider>, + /// The factory that can create a table provider from the file path + factory: Arc<dyn UrlTableFactory>, +} + +impl DynamicFileCatalogProvider { + pub fn new( + inner: Arc<dyn CatalogProvider>, + factory: Arc<dyn UrlTableFactory>, + ) -> Self { + Self { inner, factory } + } +} + +impl CatalogProvider for DynamicFileCatalogProvider { + fn as_any(&self) -> &dyn Any { + self + } + + fn schema_names(&self) -> Vec<String> { + self.inner.schema_names() + } + + fn schema(&self, name: &str) -> Option<Arc<dyn SchemaProvider>> { + self.inner.schema(name).map(|schema| { + Arc::new(DynamicFileSchemaProvider::new( + schema, + Arc::clone(&self.factory), + )) as _ + }) + } + + fn register_schema( + &self, + name: &str, + schema: Arc<dyn SchemaProvider>, + ) -> datafusion_common::Result<Option<Arc<dyn SchemaProvider>>> { + self.inner.register_schema(name, schema) + } +} + +/// Implements the [DynamicFileSchemaProvider] that can create tables provider from the file path. +/// +/// The provider will try to create a table provider from the file path if the table provider +/// isn't exist in the inner schema provider. The required object store must be registered in the session context. Review Comment: Yes, for example, how `DynamicListTableFactory` resolves the URL depends on what the session context registers as the object store. Indeed, it's a legacy comment (before moving to `datafusion-catalog`). We don't need to mention the object store here. 🤔 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
