gabotechs commented on code in PR #67:
URL: https://github.com/apache/datafusion-iceberg/pull/67#discussion_r4585163157


##########
crates/datafusion/src/read/scan.rs:
##########
@@ -98,20 +117,134 @@ impl IcebergDataSource {
 /// Builds an [`IcebergDataSource`]. Created by [`IcebergDataSource::builder`].
 ///
 /// A table provider passes the table's schema and selects columns with
-/// [`Self::with_projection`]. To rebuild a source, such as in another process,
+/// [`Self::with_projection`]. Call [`Self::build`] for lazy discovery or
+/// [`Self::build_eager`] to return a source of already-assigned files.
+///
+/// To rebuild a source, such as in another process,
 /// pass its [`IcebergDataSource::schema`], which is already projected, and no
-/// projection.
-#[derive(Debug, Clone)]
+/// projection. For already-planned files, reconstruct an eager source with
+/// [`Self::from_planned_files`] instead of discovering files again.
+#[derive(Clone)]
 pub struct IcebergDataSourceBuilder {
-    table: Table,
+    input: ScanInput,
     schema: ArrowSchemaRef,
     projection: Option<Vec<usize>>,
     snapshot_id: Option<i64>,
     predicate: Option<Predicate>,
     fetch: Option<usize>,
+    reader: Option<ArrowReader>,
+}
+
+#[derive(Clone)]
+enum ScanInput {
+    Table(Table),
+    Planned {
+        groups: Vec<Vec<FileScanTask>>,
+        file_io: FileIO,
+    },
+}
+
+impl std::fmt::Debug for IcebergDataSourceBuilder {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("IcebergDataSourceBuilder")
+            .field("schema", &self.schema)
+            .field("projection", &self.projection)
+            .field("snapshot_id", &self.snapshot_id)
+            .field("predicate", &self.predicate)
+            .field("fetch", &self.fetch)
+            .finish_non_exhaustive()
+    }
 }
 
 impl IcebergDataSourceBuilder {
+    /// Build from already-planned files without a table, catalog or discovery 
I/O.
+    /// Each group is one local output partition; empty groups and supplied 
order
+    /// are preserved. `schema` must already be projected to match the tasks.
+    /// `reader` must use the supplied FileIO and process-local 
runtime/settings.
+    /// Reader clones share their delete loader; their settings are not 
overwritten.
+    ///
+    /// Finish with [`Self::build_planned`], which reserves descriptor memory 
in the
+    /// supplied query context. Discovery-only projection, snapshot and 
predicate
+    /// settings are rejected rather than silently ignored. Statistics are 
unknown.
+    /// Use [`EagerIcebergDataSource::with_planned_files`] to replace the 
assignment
+    /// of an existing source instead.
+    ///
+    /// ```
+    /// use datafusion::arrow::datatypes::SchemaRef;
+    /// use datafusion::common::Result;
+    /// use datafusion::execution::TaskContext;
+    /// use iceberg::arrow::ArrowReader;
+    /// use iceberg::io::FileIO;
+    /// use iceberg::scan::FileScanTask;
+    /// use iceberg_datafusion::read::{EagerIcebergDataSource, 
IcebergDataSourceBuilder};
+    ///
+    /// fn restore(
+    ///     schema: SchemaRef,
+    ///     groups: Vec<Vec<FileScanTask>>,
+    ///     file_io: FileIO,
+    ///     local_reader: ArrowReader,
+    ///     fetch: Option<usize>,
+    ///     context: &TaskContext,
+    /// ) -> Result<EagerIcebergDataSource> {
+    ///     IcebergDataSourceBuilder::from_planned_files(schema, groups, 
file_io, local_reader)
+    ///         .with_fetch(fetch)
+    ///         .build_planned(context)
+    /// }
+    /// ```
+    pub fn from_planned_files(
+        schema: ArrowSchemaRef,
+        groups: Vec<Vec<FileScanTask>>,
+        file_io: FileIO,
+        reader: ArrowReader,
+    ) -> Self {
+        Self {
+            input: ScanInput::Planned { groups, file_io },
+            schema,
+            projection: None,
+            snapshot_id: None,
+            predicate: None,
+            fetch: None,
+            reader: Some(reader),
+        }
+    }
+
+    /// Finish already-planned input synchronously, including in a worker 
codec.
+    /// Does not discover files or perform I/O. Supplied groups are preserved 
exactly.
+    ///
+    /// # Errors
+    /// Returns an error for table input, discovery-only settings, zero groups 
or
+    /// insufficient query memory. Empty individual groups are valid.
+    pub fn build_planned(self, context: &TaskContext) -> 
Result<EagerIcebergDataSource> {
+        let ScanInput::Planned { groups, file_io } = self.input else {
+            return exec_err!("build_planned requires already-planned Iceberg 
files");

Review Comment:
   I see the introduction of this `enum ScanInput` is leading to a bunch of 
runtime checks in the builders as it's not capable of demonstrating a 
consistent state at compile time.
   
   If there was any way of restructuring it so that these errors are 
irrepresentable that'd be ideal.



##########
crates/datafusion/src/read/scan.rs:
##########
@@ -145,30 +278,112 @@ impl IcebergDataSourceBuilder {
         self
     }
 
-    /// Builds the source, which reads the columns of its projected schema by 
name.
+    /// Discover files and assign them round-robin to local output partitions.
+    /// Already-planned input delegates to [`Self::build_planned`] without 
discovery.
+    /// For table input, partition count is bounded by selected files (at 
least one).
+    /// Table discovery uses the caller's I/O-capable runtime and the query 
memory pool.
+    /// Dropping this future releases the retained tasks and reservation.
+    /// Without a reader override, multiple local partitions use one in-flight
+    /// file per partition; reassignment preserves the reader's settings.
     ///
     /// # Errors
+    /// Returns an error for invalid projection, failed discovery, or 
insufficient
+    /// query memory for the retained descriptors.
+    pub async fn build_eager(
+        self,
+        context: Arc<TaskContext>,
+    ) -> Result<EagerIcebergDataSource> {
+        if matches!(self.input, ScanInput::Planned { .. }) {
+            return self.build_planned(&context);
+        }
+        let schema = self.projected_schema()?;
+        let ScanInput::Table(table) = self.input else {
+            unreachable!()
+        };
+        let projection: Vec<_> = schema

Review Comment:
   The necessity of this runtime panic here suggests me that there might be a 
better way of structuring the builder. Ideally these things should be 
irrepresentable at compile time.



##########
crates/datafusion/tests/eager_file_planning.rs:
##########
@@ -0,0 +1,616 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::error::Error;
+use std::result::Result as StdResult;
+use std::sync::Arc;
+
+use datafusion::arrow::array::Int64Array;
+use datafusion::common::test_util::batches_to_sort_string;
+use datafusion::common::{Result, stats::Precision};
+use datafusion::datasource::TableProvider;
+use datafusion::datasource::source::{DataSource, DataSourceExec};
+use datafusion::execution::memory_pool::GreedyMemoryPool;
+use datafusion::execution::runtime_env::RuntimeEnvBuilder;
+use datafusion::physical_plan::common::collect;
+use datafusion::physical_plan::{ExecutionPlanProperties, collect as 
collect_plan};
+use datafusion::prelude::{SessionConfig, SessionContext, col, lit};
+use datafusion::scalar::ScalarValue;
+use iceberg_datafusion::read::{
+    EagerIcebergDataSource, IcebergDataSource, IcebergDataSourceBuilder,
+};
+use iceberg_datafusion::test_utils::IcebergTestHarness;
+use iceberg_datafusion::{
+    IcebergCatalogTableProvider, IcebergDataFusionConfig, 
IcebergStaticTableProvider,
+};
+
+mod common;
+
+use common::position_deletes::table_with_position_delete;
+
+fn eager_config() -> SessionConfig {
+    SessionConfig::new()
+        .with_target_partitions(4)
+        .with_option_extension(IcebergDataFusionConfig::default())
+        .set_bool("iceberg.execution.eager_file_planning", true)
+}
+
+fn iceberg_source(
+    plan: &dyn datafusion::physical_plan::ExecutionPlan,
+) -> &EagerIcebergDataSource {
+    plan.downcast_ref::<DataSourceExec>()
+        .expect("the Iceberg provider creates a DataSourceExec")
+        .data_source()
+        .downcast_ref::<EagerIcebergDataSource>()
+        .expect("the plan contains the Iceberg data source")
+}
+
+#[tokio::test]
+async fn plans_file_tasks_before_execution_without_changing_statistics() -> 
Result<()> {

Review Comment:
   These tests are very hard to read. Integration tests are not only here for 
verifying the integrity of the project through the public API, but they also 
serve as live runnable documentation of the features of this project.
   
   Ideally, a human should be able to skim through them and immediately 
understand the intent being tested. My recommendation is to restructure this 
tests so that:
   - All of them follow a consistent structure
   - `insta` is used for asserting over a human-readable string
   - inline `insta` snapshots in the tests themselves. As these are live 
documentation, people should be able to just review the asserted plan or 
results without jumping through files.
   - Test as close as possible to how someone would use the project in another 
codebase. For example, nobody is going to manually call 
`IcebergStaticTableProvider::scan` in their codebases, so we should also not be 
calling it here.
   
   Unfortunately, LLMs love to create these sloppy integration tests, but these 
are still key for humans to understand the project and reviewing future 
changes, so it's one of the best places for humans to come for verification.



##########
crates/datafusion/src/read/scan.rs:
##########
@@ -145,30 +278,112 @@ impl IcebergDataSourceBuilder {
         self
     }
 
-    /// Builds the source, which reads the columns of its projected schema by 
name.
+    /// Discover files and assign them round-robin to local output partitions.
+    /// Already-planned input delegates to [`Self::build_planned`] without 
discovery.
+    /// For table input, partition count is bounded by selected files (at 
least one).
+    /// Table discovery uses the caller's I/O-capable runtime and the query 
memory pool.
+    /// Dropping this future releases the retained tasks and reservation.
+    /// Without a reader override, multiple local partitions use one in-flight
+    /// file per partition; reassignment preserves the reader's settings.
     ///
     /// # Errors
+    /// Returns an error for invalid projection, failed discovery, or 
insufficient
+    /// query memory for the retained descriptors.
+    pub async fn build_eager(
+        self,
+        context: Arc<TaskContext>,
+    ) -> Result<EagerIcebergDataSource> {
+        if matches!(self.input, ScanInput::Planned { .. }) {
+            return self.build_planned(&context);
+        }
+        let schema = self.projected_schema()?;
+        let ScanInput::Table(table) = self.input else {
+            unreachable!()
+        };
+        let projection: Vec<_> = schema
+            .fields()
+            .iter()
+            .map(|field| field.name().clone())
+            .collect();
+        let scan = build_table_scan(
+            &table,
+            self.snapshot_id,
+            Some(&projection),
+            self.predicate.as_ref(),
+        )?;
+        let files = PlannedFileScan::collect(scan, &context).await?;
+        let partitions = files
+            .tasks()
+            .len()
+            .max(1)
+            .min(context.session_config().target_partitions().max(1));
+        let reader = self.reader.unwrap_or_else(|| {
+            let reader = table
+                .reader_builder()
+                .with_batch_size(context.session_config().batch_size())
+                .with_row_selection_enabled(
+                    context
+                        .session_config()
+                        .options()
+                        .execution
+                        .parquet
+                        .enable_page_index,
+                );
+            let reader = if partitions > 1 {
+                reader.with_data_file_concurrency_limit(1)
+            } else {
+                reader
+            };
+            reader.build()
+        });
+        let mut groups = vec![Vec::new(); partitions];
+        for index in 0..files.tasks().len() {
+            groups[index % partitions].push(index);
+        }
+        Ok(EagerIcebergDataSource::from_planned_files(
+            schema,
+            files,
+            groups,
+            table.file_io().clone(),
+            reader,
+            self.fetch,
+        ))
+    }
+
+    /// Build a lazy source and its shared reader without performing 
file-discovery I/O.
     ///
-    /// Returns an error if the projection is out of bounds for the schema.
+    /// # Errors
+    /// Returns an error if the projection is invalid or the input is already 
planned.
     pub fn build(self) -> Result<IcebergDataSource> {

Review Comment:
   Now that we have different DataSource impls, a more future-proof shape here 
would be to return directly an `Arc<dyn ExecutionPlan>`, a `DataSourceExec` or 
an `Arc<dyn DataSource>` rather than a concrete type. 
   
   This will allow to:
   1. Evolve the builder without breaking the API.
   2. Have a single consistent "finalizer" method for the builder.



##########
crates/datafusion/src/read/scan.rs:
##########
@@ -98,20 +117,134 @@ impl IcebergDataSource {
 /// Builds an [`IcebergDataSource`]. Created by [`IcebergDataSource::builder`].
 ///
 /// A table provider passes the table's schema and selects columns with
-/// [`Self::with_projection`]. To rebuild a source, such as in another process,
+/// [`Self::with_projection`]. Call [`Self::build`] for lazy discovery or
+/// [`Self::build_eager`] to return a source of already-assigned files.
+///
+/// To rebuild a source, such as in another process,
 /// pass its [`IcebergDataSource::schema`], which is already projected, and no
-/// projection.
-#[derive(Debug, Clone)]
+/// projection. For already-planned files, reconstruct an eager source with
+/// [`Self::from_planned_files`] instead of discovering files again.
+#[derive(Clone)]
 pub struct IcebergDataSourceBuilder {
-    table: Table,
+    input: ScanInput,
     schema: ArrowSchemaRef,
     projection: Option<Vec<usize>>,
     snapshot_id: Option<i64>,
     predicate: Option<Predicate>,
     fetch: Option<usize>,
+    reader: Option<ArrowReader>,
+}
+
+#[derive(Clone)]
+enum ScanInput {
+    Table(Table),
+    Planned {
+        groups: Vec<Vec<FileScanTask>>,
+        file_io: FileIO,
+    },
+}
+
+impl std::fmt::Debug for IcebergDataSourceBuilder {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("IcebergDataSourceBuilder")
+            .field("schema", &self.schema)
+            .field("projection", &self.projection)
+            .field("snapshot_id", &self.snapshot_id)
+            .field("predicate", &self.predicate)
+            .field("fetch", &self.fetch)
+            .finish_non_exhaustive()
+    }
 }
 
 impl IcebergDataSourceBuilder {
+    /// Build from already-planned files without a table, catalog or discovery 
I/O.
+    /// Each group is one local output partition; empty groups and supplied 
order
+    /// are preserved. `schema` must already be projected to match the tasks.
+    /// `reader` must use the supplied FileIO and process-local 
runtime/settings.
+    /// Reader clones share their delete loader; their settings are not 
overwritten.
+    ///
+    /// Finish with [`Self::build_planned`], which reserves descriptor memory 
in the
+    /// supplied query context. Discovery-only projection, snapshot and 
predicate
+    /// settings are rejected rather than silently ignored. Statistics are 
unknown.
+    /// Use [`EagerIcebergDataSource::with_planned_files`] to replace the 
assignment
+    /// of an existing source instead.
+    ///
+    /// ```
+    /// use datafusion::arrow::datatypes::SchemaRef;
+    /// use datafusion::common::Result;
+    /// use datafusion::execution::TaskContext;
+    /// use iceberg::arrow::ArrowReader;
+    /// use iceberg::io::FileIO;
+    /// use iceberg::scan::FileScanTask;
+    /// use iceberg_datafusion::read::{EagerIcebergDataSource, 
IcebergDataSourceBuilder};
+    ///
+    /// fn restore(
+    ///     schema: SchemaRef,
+    ///     groups: Vec<Vec<FileScanTask>>,
+    ///     file_io: FileIO,
+    ///     local_reader: ArrowReader,
+    ///     fetch: Option<usize>,
+    ///     context: &TaskContext,
+    /// ) -> Result<EagerIcebergDataSource> {
+    ///     IcebergDataSourceBuilder::from_planned_files(schema, groups, 
file_io, local_reader)
+    ///         .with_fetch(fetch)
+    ///         .build_planned(context)
+    /// }
+    /// ```
+    pub fn from_planned_files(

Review Comment:
   I see this only exists for the sake of the tests, my suggestion would be 
remove it and instead have exactly one way in this project of constructing an 
`IcebergDataSourceBuilder`. 
   
   Right now the only way of constructing it is through `IcebergDataSource`, 
which seems wrong, as the `IcebergDataSourceBuilder` might not even construct 
an `IcebergDataSource` at all, now it might construct an 
`EagerIcebergDataSource`.



##########
crates/datafusion/src/read/scan.rs:
##########
@@ -98,20 +117,134 @@ impl IcebergDataSource {
 /// Builds an [`IcebergDataSource`]. Created by [`IcebergDataSource::builder`].
 ///
 /// A table provider passes the table's schema and selects columns with
-/// [`Self::with_projection`]. To rebuild a source, such as in another process,
+/// [`Self::with_projection`]. Call [`Self::build`] for lazy discovery or
+/// [`Self::build_eager`] to return a source of already-assigned files.
+///
+/// To rebuild a source, such as in another process,
 /// pass its [`IcebergDataSource::schema`], which is already projected, and no
-/// projection.
-#[derive(Debug, Clone)]
+/// projection. For already-planned files, reconstruct an eager source with
+/// [`Self::from_planned_files`] instead of discovering files again.
+#[derive(Clone)]
 pub struct IcebergDataSourceBuilder {
-    table: Table,
+    input: ScanInput,
     schema: ArrowSchemaRef,
     projection: Option<Vec<usize>>,
     snapshot_id: Option<i64>,
     predicate: Option<Predicate>,
     fetch: Option<usize>,
+    reader: Option<ArrowReader>,
+}
+
+#[derive(Clone)]
+enum ScanInput {
+    Table(Table),
+    Planned {
+        groups: Vec<Vec<FileScanTask>>,
+        file_io: FileIO,
+    },
+}
+
+impl std::fmt::Debug for IcebergDataSourceBuilder {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("IcebergDataSourceBuilder")
+            .field("schema", &self.schema)
+            .field("projection", &self.projection)
+            .field("snapshot_id", &self.snapshot_id)
+            .field("predicate", &self.predicate)
+            .field("fetch", &self.fetch)
+            .finish_non_exhaustive()
+    }
 }
 
 impl IcebergDataSourceBuilder {
+    /// Build from already-planned files without a table, catalog or discovery 
I/O.
+    /// Each group is one local output partition; empty groups and supplied 
order
+    /// are preserved. `schema` must already be projected to match the tasks.
+    /// `reader` must use the supplied FileIO and process-local 
runtime/settings.
+    /// Reader clones share their delete loader; their settings are not 
overwritten.
+    ///
+    /// Finish with [`Self::build_planned`], which reserves descriptor memory 
in the
+    /// supplied query context. Discovery-only projection, snapshot and 
predicate
+    /// settings are rejected rather than silently ignored. Statistics are 
unknown.
+    /// Use [`EagerIcebergDataSource::with_planned_files`] to replace the 
assignment
+    /// of an existing source instead.
+    ///
+    /// ```
+    /// use datafusion::arrow::datatypes::SchemaRef;
+    /// use datafusion::common::Result;
+    /// use datafusion::execution::TaskContext;
+    /// use iceberg::arrow::ArrowReader;
+    /// use iceberg::io::FileIO;
+    /// use iceberg::scan::FileScanTask;
+    /// use iceberg_datafusion::read::{EagerIcebergDataSource, 
IcebergDataSourceBuilder};
+    ///
+    /// fn restore(
+    ///     schema: SchemaRef,
+    ///     groups: Vec<Vec<FileScanTask>>,
+    ///     file_io: FileIO,
+    ///     local_reader: ArrowReader,
+    ///     fetch: Option<usize>,
+    ///     context: &TaskContext,
+    /// ) -> Result<EagerIcebergDataSource> {
+    ///     IcebergDataSourceBuilder::from_planned_files(schema, groups, 
file_io, local_reader)
+    ///         .with_fetch(fetch)
+    ///         .build_planned(context)
+    /// }
+    /// ```
+    pub fn from_planned_files(

Review Comment:
   This method does not seem right. The whole point of having a builder pattern 
is so that we avoid constructing through methods that rely on positional 
arguments, and therefore, are subject to breaking changes.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to