This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch merge-catalog-memory in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
commit 06abf83ab04b77d3cb0b14eeff8ae2146fc31361 Author: Xuanwo <[email protected]> AuthorDate: Mon Jun 30 21:21:03 2025 +0800 refactor: Move memory catalog into core Signed-off-by: Xuanwo <[email protected]> --- Cargo.lock | 17 +- Cargo.toml | 1 - crates/catalog/memory/Cargo.toml | 42 --- crates/catalog/memory/DEPENDENCIES.rust.tsv | 325 --------------------- crates/catalog/memory/README.md | 27 -- crates/iceberg/Cargo.toml | 1 + .../src => iceberg/src/catalog/memory}/catalog.rs | 14 +- .../lib.rs => iceberg/src/catalog/memory/mod.rs} | 6 +- .../src/catalog/memory}/namespace_state.rs | 2 +- crates/iceberg/src/catalog/mod.rs | 2 + crates/iceberg/src/lib.rs | 4 +- crates/iceberg/src/writer/mod.rs | 4 +- crates/integrations/datafusion/Cargo.toml | 1 - .../tests/integration_datafusion_test.rs | 2 +- 14 files changed, 18 insertions(+), 430 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ca88923c..5279bd32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3520,6 +3520,7 @@ dependencies = [ "parquet", "pretty_assertions", "rand 0.8.5", + "regex", "reqwest", "roaring", "rust_decimal", @@ -3592,21 +3593,6 @@ dependencies = [ "volo-thrift", ] -[[package]] -name = "iceberg-catalog-memory" -version = "0.5.1" -dependencies = [ - "async-trait", - "futures", - "iceberg", - "itertools 0.13.0", - "regex", - "serde_json", - "tempfile", - "tokio", - "uuid", -] - [[package]] name = "iceberg-catalog-rest" version = "0.5.1" @@ -3673,7 +3659,6 @@ dependencies = [ "datafusion", "futures", "iceberg", - "iceberg-catalog-memory", "parquet", "tempfile", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 43d03520..60b04529 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,7 +75,6 @@ futures = "0.3" hive_metastore = "0.1" http = "1.2" iceberg = { version = "0.5.1", path = "./crates/iceberg" } -iceberg-catalog-memory = { version = "0.5.1", path = "./crates/catalog/memory" } iceberg-catalog-rest = { version = "0.5.1", path = "./crates/catalog/rest" } iceberg-datafusion = { version = "0.5.1", path = "./crates/integrations/datafusion" } indicatif = "0.17" diff --git a/crates/catalog/memory/Cargo.toml b/crates/catalog/memory/Cargo.toml deleted file mode 100644 index dfd3625a..00000000 --- a/crates/catalog/memory/Cargo.toml +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -edition = { workspace = true } -homepage = { workspace = true } -name = "iceberg-catalog-memory" -rust-version = { workspace = true } -version = { workspace = true } - -categories = ["database"] -description = "Apache Iceberg Rust Memory Catalog API" -keywords = ["iceberg", "memory", "catalog"] -license = { workspace = true } -repository = { workspace = true } - -[dependencies] -async-trait = { workspace = true } -futures = { workspace = true } -iceberg = { workspace = true } -itertools = { workspace = true } -serde_json = { workspace = true } -uuid = { workspace = true, features = ["v4"] } - -[dev-dependencies] -regex = { workspace = true } -tempfile = { workspace = true } -tokio = { workspace = true } diff --git a/crates/catalog/memory/DEPENDENCIES.rust.tsv b/crates/catalog/memory/DEPENDENCIES.rust.tsv deleted file mode 100644 index 16691243..00000000 --- a/crates/catalog/memory/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,325 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC MIT MPL-2.0 Unicode-3.0 Unlicense Zlib [email protected] X X [email protected] X X X [email protected] X [email protected] X X [email protected] X X [email protected] X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected] X X [email protected] X X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X X [email protected] X X [email protected] X [email protected] X X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X X [email protected] X X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected] X X [email protected] X [email protected] X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected] X [email protected] X X [email protected] X [email protected] X X X [email protected] X [email protected] X X [email protected] X X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected] X [email protected] X X [email protected] X [email protected] X [email protected] X X [email protected] X X [email protected] X [email protected] X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X X [email protected] X [email protected] X X [email protected] X X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected] X X [email protected] X X [email protected] X X X [email protected] X X X [email protected] X X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected] X X [email protected] X X [email protected] X [email protected] X X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X X [email protected] X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected]+wasi-snapshot-preview1 X X X [email protected]+wasi-0.2.2 X X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X [email protected] X X X [email protected] X X [email protected] X [email protected] X [email protected] X [email protected] X X X [email protected] X X X [email protected] X [email protected] X [email protected] X X [email protected] X [email protected] X [email protected] X [email protected] X [email protected] X X [email protected]+zstd.1.5.6 X X diff --git a/crates/catalog/memory/README.md b/crates/catalog/memory/README.md deleted file mode 100644 index 5b04f78a..00000000 --- a/crates/catalog/memory/README.md +++ /dev/null @@ -1,27 +0,0 @@ -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, - ~ software distributed under the License is distributed on an - ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - ~ KIND, either express or implied. See the License for the - ~ specific language governing permissions and limitations - ~ under the License. ---> - -# Apache Iceberg Memory Catalog Official Native Rust Implementation - -[](https://crates.io/crates/iceberg-catalog-memory) -[](https://docs.rs/iceberg/latest/iceberg-catalog-memory/) - -This crate contains the official Native Rust implementation of Apache Iceberg Memory Catalog. - -See the [API documentation](https://docs.rs/iceberg-catalog-memory/latest) for examples and the full API. diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index 9f5d4901..33d30eab 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -95,6 +95,7 @@ expect-test = { workspace = true } iceberg_test_utils = { path = "../test_utils", features = ["tests"] } pretty_assertions = { workspace = true } rand = { workspace = true } +regex = { workspace = true } tempfile = { workspace = true } tera = { workspace = true } diff --git a/crates/catalog/memory/src/catalog.rs b/crates/iceberg/src/catalog/memory/catalog.rs similarity index 99% rename from crates/catalog/memory/src/catalog.rs rename to crates/iceberg/src/catalog/memory/catalog.rs index 7454de12..c3be536c 100644 --- a/crates/catalog/memory/src/catalog.rs +++ b/crates/iceberg/src/catalog/memory/catalog.rs @@ -21,17 +21,17 @@ use std::collections::HashMap; use async_trait::async_trait; use futures::lock::Mutex; -use iceberg::io::FileIO; -use iceberg::spec::{TableMetadata, TableMetadataBuilder}; -use iceberg::table::Table; -use iceberg::{ +use crate::io::FileIO; +use crate::spec::{TableMetadata, TableMetadataBuilder}; +use crate::table::Table; +use crate::{ Catalog, Error, ErrorKind, Namespace, NamespaceIdent, Result, TableCommit, TableCreation, TableIdent, }; use itertools::Itertools; use uuid::Uuid; -use crate::namespace_state::NamespaceState; +use super::namespace_state::NamespaceState; /// namespace `location` property const LOCATION: &str = "location"; @@ -291,8 +291,8 @@ mod tests { use std::hash::Hash; use std::iter::FromIterator; - use iceberg::io::FileIOBuilder; - use iceberg::spec::{NestedField, PartitionSpec, PrimitiveType, Schema, SortOrder, Type}; + use crate::io::FileIOBuilder; + use crate::spec::{NestedField, PartitionSpec, PrimitiveType, Schema, SortOrder, Type}; use regex::Regex; use tempfile::TempDir; diff --git a/crates/catalog/memory/src/lib.rs b/crates/iceberg/src/catalog/memory/mod.rs similarity index 90% rename from crates/catalog/memory/src/lib.rs rename to crates/iceberg/src/catalog/memory/mod.rs index 8988ac7b..ff18bf1a 100644 --- a/crates/catalog/memory/src/lib.rs +++ b/crates/iceberg/src/catalog/memory/mod.rs @@ -15,11 +15,9 @@ // specific language governing permissions and limitations // under the License. -//! Iceberg memory Catalog API implementation. - -#![deny(missing_docs)] +//! Memory catalog implementation. mod catalog; mod namespace_state; -pub use catalog::*; +pub use catalog::MemoryCatalog; \ No newline at end of file diff --git a/crates/catalog/memory/src/namespace_state.rs b/crates/iceberg/src/catalog/memory/namespace_state.rs similarity index 99% rename from crates/catalog/memory/src/namespace_state.rs rename to crates/iceberg/src/catalog/memory/namespace_state.rs index e324e7a3..be2e877a 100644 --- a/crates/catalog/memory/src/namespace_state.rs +++ b/crates/iceberg/src/catalog/memory/namespace_state.rs @@ -17,7 +17,7 @@ use std::collections::{HashMap, hash_map}; -use iceberg::{Error, ErrorKind, NamespaceIdent, Result, TableIdent}; +use crate::{Error, ErrorKind, NamespaceIdent, Result, TableIdent}; use itertools::Itertools; // Represents the state of a namespace diff --git a/crates/iceberg/src/catalog/mod.rs b/crates/iceberg/src/catalog/mod.rs index ebb9a66c..96bab931 100644 --- a/crates/iceberg/src/catalog/mod.rs +++ b/crates/iceberg/src/catalog/mod.rs @@ -17,6 +17,8 @@ //! Catalog API for Apache Iceberg +pub mod memory; + use std::collections::HashMap; use std::fmt::{Debug, Display}; use std::future::Future; diff --git a/crates/iceberg/src/lib.rs b/crates/iceberg/src/lib.rs index 06e39156..47473fc6 100644 --- a/crates/iceberg/src/lib.rs +++ b/crates/iceberg/src/lib.rs @@ -22,12 +22,10 @@ //! ## Scan A Table //! //! ```rust, ignore -//! // This example uses `iceberg_catalog_memory`, which isn't enabled by default. -//! // To run this, add `iceberg-catalog-memory` as a dependency in your Cargo.toml. //! use futures::TryStreamExt; //! use iceberg::io::{FileIO, FileIOBuilder}; +//! use iceberg::memory::MemoryCatalog; //! use iceberg::{Catalog, Result, TableIdent}; -//! use iceberg_catalog_memory::MemoryCatalog; //! //! #[tokio::main] //! async fn main() -> Result<()> { diff --git a/crates/iceberg/src/writer/mod.rs b/crates/iceberg/src/writer/mod.rs index 26e19919..ae6847ea 100644 --- a/crates/iceberg/src/writer/mod.rs +++ b/crates/iceberg/src/writer/mod.rs @@ -55,7 +55,7 @@ //! }; //! use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; //! use iceberg::{Catalog, Result, TableIdent}; -//! use iceberg_catalog_memory::MemoryCatalog; +//! use crate::memory::MemoryCatalog; //! use parquet::file::properties::WriterProperties; //! #[tokio::main] //! async fn main() -> Result<()> { @@ -112,7 +112,7 @@ //! }; //! use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; //! use iceberg::{Catalog, Result, TableIdent}; -//! use iceberg_catalog_memory::MemoryCatalog; +//! use crate::memory::MemoryCatalog; //! use parquet::file::properties::WriterProperties; //! //! #[derive(Clone)] diff --git a/crates/integrations/datafusion/Cargo.toml b/crates/integrations/datafusion/Cargo.toml index a152b9d2..5adaed45 100644 --- a/crates/integrations/datafusion/Cargo.toml +++ b/crates/integrations/datafusion/Cargo.toml @@ -37,6 +37,5 @@ iceberg = { workspace = true } tokio = { workspace = true } [dev-dependencies] -iceberg-catalog-memory = { workspace = true } parquet = { workspace = true } tempfile = { workspace = true } diff --git a/crates/integrations/datafusion/tests/integration_datafusion_test.rs b/crates/integrations/datafusion/tests/integration_datafusion_test.rs index c21b72b3..78cf7a2f 100644 --- a/crates/integrations/datafusion/tests/integration_datafusion_test.rs +++ b/crates/integrations/datafusion/tests/integration_datafusion_test.rs @@ -28,7 +28,7 @@ use datafusion::parquet::arrow::PARQUET_FIELD_ID_META_KEY; use iceberg::io::FileIOBuilder; use iceberg::spec::{NestedField, PrimitiveType, Schema, StructType, Type}; use iceberg::{Catalog, NamespaceIdent, Result, TableCreation}; -use iceberg_catalog_memory::MemoryCatalog; +use iceberg::memory::MemoryCatalog; use iceberg_datafusion::IcebergCatalogProvider; use tempfile::TempDir;
