This is an automated email from the ASF dual-hosted git repository.
weijun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 8973f1b602 Move catalog_common out of core (#15193)
8973f1b602 is described below
commit 8973f1b602cf6ce8ed4d7e852a8eac9acc2a6a60
Author: logan-keede <[email protected]>
AuthorDate: Fri Mar 14 18:52:51 2025 +0530
Move catalog_common out of core (#15193)
* Move Listing Schema Provider out of core
* remove catlog_common completely
* fix: cargo fmt
* forgotten comment
---
Cargo.lock | 5 +----
datafusion/catalog/Cargo.toml | 1 +
datafusion/catalog/src/lib.rs | 12 ++---------
.../src}/listing_schema.rs | 6 +++---
datafusion/core/src/catalog_common/mod.rs | 24 ----------------------
datafusion/core/src/execution/context/mod.rs | 2 +-
datafusion/core/src/execution/session_state.rs | 6 +++---
.../core/src/execution/session_state_defaults.rs | 2 +-
datafusion/core/src/lib.rs | 1 -
datafusion/datasource-csv/Cargo.toml | 4 ----
10 files changed, 12 insertions(+), 51 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index e772b2466c..44b3c5fbfc 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1842,6 +1842,7 @@ dependencies = [
"futures",
"itertools 0.14.0",
"log",
+ "object_store",
"parking_lot",
"tokio",
]
@@ -2005,13 +2006,9 @@ dependencies = [
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"futures",
- "itertools 0.14.0",
- "log",
"object_store",
- "rand 0.8.5",
"regex",
"tokio",
- "url",
]
[[package]]
diff --git a/datafusion/catalog/Cargo.toml b/datafusion/catalog/Cargo.toml
index 8647494111..4e1555b782 100644
--- a/datafusion/catalog/Cargo.toml
+++ b/datafusion/catalog/Cargo.toml
@@ -42,6 +42,7 @@ datafusion-sql = { workspace = true }
futures = { workspace = true }
itertools = { workspace = true }
log = { workspace = true }
+object_store = { workspace = true }
parking_lot = { workspace = true }
[dev-dependencies]
diff --git a/datafusion/catalog/src/lib.rs b/datafusion/catalog/src/lib.rs
index a339d4916b..cf7dd4b067 100644
--- a/datafusion/catalog/src/lib.rs
+++ b/datafusion/catalog/src/lib.rs
@@ -26,18 +26,9 @@
//! Implementations
//! * Information schema: [`information_schema`]
//! * Simple memory based catalog: [`MemoryCatalogProviderList`],
[`MemoryCatalogProvider`], [`MemorySchemaProvider`]
+//! * Listing schema: [`listing_schema`]
pub mod memory;
-#[deprecated(
- since = "46.0.0",
- note = "use datafusion_sql::resolve::resolve_table_references"
-)]
-pub use datafusion_sql::resolve::resolve_table_references;
-#[deprecated(
- since = "46.0.0",
- note = "use datafusion_common::{ResolvedTableReference, TableReference}"
-)]
-pub use datafusion_sql::{ResolvedTableReference, TableReference};
pub use memory::{
MemoryCatalogProvider, MemoryCatalogProviderList, MemorySchemaProvider,
};
@@ -45,6 +36,7 @@ mod r#async;
mod catalog;
mod dynamic_file;
pub mod information_schema;
+pub mod listing_schema;
mod schema;
mod session;
mod table;
diff --git a/datafusion/core/src/catalog_common/listing_schema.rs
b/datafusion/catalog/src/listing_schema.rs
similarity index 97%
rename from datafusion/core/src/catalog_common/listing_schema.rs
rename to datafusion/catalog/src/listing_schema.rs
index dc55a07ef8..cc2c2ee606 100644
--- a/datafusion/core/src/catalog_common/listing_schema.rs
+++ b/datafusion/catalog/src/listing_schema.rs
@@ -22,9 +22,9 @@ use std::collections::HashSet;
use std::path::Path;
use std::sync::{Arc, Mutex};
-use crate::catalog::{SchemaProvider, TableProvider, TableProviderFactory};
-use crate::execution::context::SessionState;
+use crate::{SchemaProvider, TableProvider, TableProviderFactory};
+use crate::Session;
use datafusion_common::{
Constraints, DFSchema, DataFusionError, HashMap, TableReference,
};
@@ -88,7 +88,7 @@ impl ListingSchemaProvider {
}
/// Reload table information from ObjectStore
- pub async fn refresh(&self, state: &SessionState) ->
datafusion_common::Result<()> {
+ pub async fn refresh(&self, state: &dyn Session) ->
datafusion_common::Result<()> {
let entries: Vec<_> =
self.store.list(Some(&self.path)).try_collect().await?;
let base = Path::new(self.path.as_ref());
let mut tables = HashSet::new();
diff --git a/datafusion/core/src/catalog_common/mod.rs
b/datafusion/core/src/catalog_common/mod.rs
deleted file mode 100644
index 213afb3240..0000000000
--- a/datafusion/core/src/catalog_common/mod.rs
+++ /dev/null
@@ -1,24 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Interfaces and default implementations of catalogs and schemas.
-//!
-//! Implementations
-//! * Listing schema: [`listing_schema`]
-
-pub mod listing_schema;
-pub use crate::catalog::{CatalogProvider, CatalogProviderList, SchemaProvider};
diff --git a/datafusion/core/src/execution/context/mod.rs
b/datafusion/core/src/execution/context/mod.rs
index ad0993ed43..beefca6d57 100644
--- a/datafusion/core/src/execution/context/mod.rs
+++ b/datafusion/core/src/execution/context/mod.rs
@@ -25,10 +25,10 @@ use std::sync::{Arc, Weak};
use super::options::ReadOptions;
use crate::{
+ catalog::listing_schema::ListingSchemaProvider,
catalog::{
CatalogProvider, CatalogProviderList, TableProvider,
TableProviderFactory,
},
- catalog_common::listing_schema::ListingSchemaProvider,
dataframe::DataFrame,
datasource::listing::{
ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,
diff --git a/datafusion/core/src/execution/session_state.rs
b/datafusion/core/src/execution/session_state.rs
index f4b0fd0c12..002220f93e 100644
--- a/datafusion/core/src/execution/session_state.rs
+++ b/datafusion/core/src/execution/session_state.rs
@@ -436,16 +436,16 @@ impl SessionState {
/// Resolve all table references in the SQL statement. Does not include
CTE references.
///
- /// See [`datafusion_catalog::resolve_table_references`] for more
information.
+ /// See [`datafusion_sql::resolve::resolve_table_references`] for more
information.
///
- /// [`datafusion_catalog::resolve_table_references`]:
datafusion_catalog::resolve_table_references
+ /// [`datafusion_sql::resolve::resolve_table_references`]:
datafusion_sql::resolve::resolve_table_references
pub fn resolve_table_references(
&self,
statement: &Statement,
) -> datafusion_common::Result<Vec<TableReference>> {
let enable_ident_normalization =
self.config.options().sql_parser.enable_ident_normalization;
- let (table_refs, _) = datafusion_catalog::resolve_table_references(
+ let (table_refs, _) =
datafusion_sql::resolve::resolve_table_references(
statement,
enable_ident_normalization,
)?;
diff --git a/datafusion/core/src/execution/session_state_defaults.rs
b/datafusion/core/src/execution/session_state_defaults.rs
index b48ef90f2b..a241738bd3 100644
--- a/datafusion/core/src/execution/session_state_defaults.rs
+++ b/datafusion/core/src/execution/session_state_defaults.rs
@@ -15,8 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+use crate::catalog::listing_schema::ListingSchemaProvider;
use crate::catalog::{CatalogProvider, TableProviderFactory};
-use crate::catalog_common::listing_schema::ListingSchemaProvider;
use crate::datasource::file_format::arrow::ArrowFormatFactory;
#[cfg(feature = "avro")]
use crate::datasource::file_format::avro::AvroFormatFactory;
diff --git a/datafusion/core/src/lib.rs b/datafusion/core/src/lib.rs
index b4d5f9740b..c2ac2f7601 100644
--- a/datafusion/core/src/lib.rs
+++ b/datafusion/core/src/lib.rs
@@ -699,7 +699,6 @@ pub const DATAFUSION_VERSION: &str =
env!("CARGO_PKG_VERSION");
extern crate core;
extern crate sqlparser;
-pub mod catalog_common;
pub mod dataframe;
pub mod datasource;
pub mod error;
diff --git a/datafusion/datasource-csv/Cargo.toml
b/datafusion/datasource-csv/Cargo.toml
index 689531758c..b95c51cbbe 100644
--- a/datafusion/datasource-csv/Cargo.toml
+++ b/datafusion/datasource-csv/Cargo.toml
@@ -44,13 +44,9 @@ datafusion-physical-expr = { workspace = true }
datafusion-physical-expr-common = { workspace = true }
datafusion-physical-plan = { workspace = true }
futures = { workspace = true }
-itertools = { workspace = true }
-log = { workspace = true }
object_store = { workspace = true }
-rand = { workspace = true }
regex = { workspace = true }
tokio = { workspace = true }
-url = { workspace = true }
[lints]
workspace = true
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
