This is an automated email from the ASF dual-hosted git repository.
xudong963 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new b2c210abd9 Improve documentation for `FileSource`, `DataSource` and
`DataSourceExec` (#15766)
b2c210abd9 is described below
commit b2c210abd99b20669f4fddd9c53fd659b531ea82
Author: Andrew Lamb <[email protected]>
AuthorDate: Tue Apr 29 01:07:37 2025 -0400
Improve documentation for `FileSource`, `DataSource` and `DataSourceExec`
(#15766)
* Improve documentation for FileSource
* more
* Update datafusion/datasource/src/file.rs
Co-authored-by: Adrian Garcia Badaracco
<[email protected]>
* Clippy
* fmt
---------
Co-authored-by: Adrian Garcia Badaracco
<[email protected]>
---
datafusion/datasource/src/file.rs | 19 ++++++++++++++-----
datafusion/datasource/src/source.rs | 31 ++++++++++++++++++++++---------
2 files changed, 36 insertions(+), 14 deletions(-)
diff --git a/datafusion/datasource/src/file.rs
b/datafusion/datasource/src/file.rs
index 835285b21e..6bbe9e50d3 100644
--- a/datafusion/datasource/src/file.rs
+++ b/datafusion/datasource/src/file.rs
@@ -37,9 +37,16 @@ use datafusion_physical_plan::DisplayFormatType;
use object_store::ObjectStore;
-/// Common file format behaviors needs to implement.
+/// file format specific behaviors for elements in [`DataSource`]
///
-/// See implementation examples such as `ParquetSource`, `CsvSource`
+/// See more details on specific implementations:
+/// *
[`ArrowSource`](https://docs.rs/datafusion/latest/datafusion/datasource/physical_plan/struct.ArrowSource.html)
+/// *
[`AvroSource`](https://docs.rs/datafusion/latest/datafusion/datasource/physical_plan/struct.AvroSource.html)
+/// *
[`CsvSource`](https://docs.rs/datafusion/latest/datafusion/datasource/physical_plan/struct.CsvSource.html)
+/// *
[`JsonSource`](https://docs.rs/datafusion/latest/datafusion/datasource/physical_plan/struct.JsonSource.html)
+/// *
[`ParquetSource`](https://docs.rs/datafusion/latest/datafusion/datasource/physical_plan/struct.ParquetSource.html)
+///
+/// [`DataSource`]: crate::source::DataSource
pub trait FileSource: Send + Sync {
/// Creates a `dyn FileOpener` based on given parameters
fn create_file_opener(
@@ -69,10 +76,12 @@ pub trait FileSource: Send + Sync {
Ok(())
}
- /// If supported by the [`FileSource`], redistribute files across
partitions according to their size.
- /// Allows custom file formats to implement their own repartitioning logic.
+ /// If supported by the [`FileSource`], redistribute files across
partitions
+ /// according to their size. Allows custom file formats to implement their
+ /// own repartitioning logic.
///
- /// Provides a default repartitioning behavior, see comments on
[`FileGroupPartitioner`] for more detail.
+ /// The default implementation uses [`FileGroupPartitioner`]. See that
+ /// struct for more details.
fn repartitioned(
&self,
target_partitions: usize,
diff --git a/datafusion/datasource/src/source.rs
b/datafusion/datasource/src/source.rs
index 2d6ea1a8b3..6abe3c3298 100644
--- a/datafusion/datasource/src/source.rs
+++ b/datafusion/datasource/src/source.rs
@@ -40,15 +40,26 @@ use datafusion_physical_plan::filter_pushdown::{
FilterPushdownSupport,
};
-/// Common behaviors in Data Sources for both from Files and Memory.
+/// A source of data, typically a list of files or memory
+///
+/// This trait provides common behaviors for abstract sources of data. It has
+/// two common implementations:
+///
+/// 1. [`FileScanConfig`]: lists of files
+/// 2. [`MemorySourceConfig`]: in memory list of `RecordBatch`
+///
+/// File format specific behaviors are defined by [`FileSource`]
///
/// # See Also
-/// * [`DataSourceExec`] for physical plan implementation
-/// * [`FileSource`] for file format implementations (Parquet, Json, etc)
+/// * [`FileSource`] for file format specific implementations (Parquet, Json,
etc)
+/// * [`DataSourceExec`]: The [`ExecutionPlan`] that reads from a `DataSource`
///
/// # Notes
+///
/// Requires `Debug` to assist debugging
///
+/// [`FileScanConfig`]:
https://docs.rs/datafusion/latest/datafusion/datasource/physical_plan/struct.FileScanConfig.html
+/// [`MemorySourceConfig`]:
https://docs.rs/datafusion/latest//datafusion/datasource/memory/struct.MemorySourceConfig.html
/// [`FileSource`]: crate::file::FileSource
pub trait DataSource: Send + Sync + Debug {
fn open(
@@ -94,14 +105,16 @@ pub trait DataSource: Send + Sync + Debug {
}
}
-/// [`ExecutionPlan`] handles different file formats like JSON, CSV, AVRO,
ARROW, PARQUET
+/// [`ExecutionPlan`] that reads one or more files
+///
+/// `DataSourceExec` implements common functionality such as applying
+/// projections, and caching plan properties.
///
-/// `DataSourceExec` implements common functionality such as applying
projections,
-/// and caching plan properties.
+/// The [`DataSource`] describes where to find the data for this data source
+/// (for example in files or what in memory partitions).
///
-/// The [`DataSource`] trait describes where to find the data for this data
-/// source (for example what files or what in memory partitions). Format
-/// specifics are implemented with the [`FileSource`] trait.
+/// For file based [`DataSource`]s, format specific behavior is implemented in
+/// the [`FileSource`] trait.
///
/// [`FileSource`]: crate::file::FileSource
#[derive(Clone, Debug)]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
