gabotechs commented on code in PR #87:
URL: https://github.com/apache/datafusion-iceberg/pull/87#discussion_r4610358195


##########
crates/test-tools/src/append.rs:
##########
@@ -0,0 +1,463 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use datafusion::arrow::array::{RecordBatch, new_null_array};
+use datafusion::arrow::compute::cast;
+use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef;
+use datafusion::common::{exec_err, not_impl_err};
+use datafusion::error::Result;
+use datafusion::parquet::file::properties::WriterProperties;
+use iceberg::arrow::{RecordBatchPartitionSplitter, schema_to_arrow_schema};
+use iceberg::spec::{
+    DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestListWriter,
+    ManifestWriterBuilder, Operation, Snapshot, SnapshotSummaryCollector, 
Summary,
+};
+use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder;
+use iceberg::writer::file_writer::ParquetWriterBuilder;
+use iceberg::writer::file_writer::location_generator::{
+    DefaultFileNameGenerator, DefaultLocationGenerator,
+};
+use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder;
+use iceberg::writer::{IcebergWriter, IcebergWriterBuilder};
+
+use crate::FixtureTable;
+use crate::error::df_err;
+
+/// One commit that adds data files, started by [`FixtureTable::append`].
+///
+/// Each [`Append::file`] becomes exactly one Parquet data file, so tests
+/// control how rows are laid out across files. [`Append::commit`] adds them
+/// all in one snapshot on top of the current one. Snapshots are numbered 1,
+/// 2, 3, ..., unlike iceberg-rust's `fast_append`, which picks random ids, so
+/// that tests can show snapshot ids. Only adds data files: no deletes, and
+/// format version 2 only.
+pub struct Append<'a> {
+    table: &'a mut FixtureTable,
+    writer_properties: WriterProperties,
+    files: Vec<RecordBatch>,
+}
+
+impl<'a> Append<'a> {
+    pub(crate) fn new(table: &'a mut FixtureTable) -> Self {
+        Self {
+            table,
+            writer_properties: WriterProperties::default(),
+            files: Vec::new(),
+        }
+    }
+
+    /// Sets the properties every data file of this commit is written with.
+    pub fn writer_properties(mut self, writer_properties: WriterProperties) -> 
Self {
+        self.writer_properties = writer_properties;
+        self
+    }
+
+    /// Adds one data file containing `batch`.
+    ///
+    /// Columns are matched to the table's by name and cast to its types.
+    /// Columns of the table that `batch` lacks are null. On a partitioned
+    /// table every row must belong to the same partition.
+    pub fn file(mut self, batch: RecordBatch) -> Self {
+        self.files.push(batch);
+        self
+    }
+
+    /// Writes the data files and commits them as a new snapshot of the main
+    /// branch. Returns the data files in the order they were added.
+    pub async fn commit(self) -> Result<Vec<DataFile>> {
+        let table = self.table.table();
+        let metadata = table.metadata();
+        if metadata.format_version() != FormatVersion::V2 {
+            return not_impl_err!("fixtures only append to format version 2 
tables");
+        }
+        let file_io = table.file_io();
+        let schema = metadata.current_schema().clone();
+        let partition_spec = metadata.default_partition_spec().clone();
+        let arrow_schema = 
Arc::new(schema_to_arrow_schema(&schema).map_err(df_err)?);
+        let parent_snapshot_id = metadata.current_snapshot_id();
+        let snapshot_id = metadata
+            .snapshots()
+            .map(|s| s.snapshot_id())
+            .max()
+            .unwrap_or(0)
+            + 1;
+        let sequence_number = metadata.next_sequence_number();
+
+        let splitter = if partition_spec.is_unpartitioned() {
+            None
+        } else {
+            Some(
+                RecordBatchPartitionSplitter::try_new_with_computed_values(
+                    schema.clone(),
+                    partition_spec.clone(),
+                )
+                .map_err(df_err)?,
+            )
+        };
+        let writer_builder = 
DataFileWriterBuilder::new(RollingFileWriterBuilder::new(
+            ParquetWriterBuilder::new(self.writer_properties, schema.clone()),
+            // Never roll over, so that each batch is written to one file.
+            usize::MAX,
+            file_io.clone(),
+            DefaultLocationGenerator::new(metadata).map_err(df_err)?,
+            DefaultFileNameGenerator::new(
+                format!("{snapshot_id}"),
+                None,
+                DataFileFormat::Parquet,
+            ),
+        ));
+        let mut data_files = Vec::with_capacity(self.files.len());
+        for batch in self.files {
+            let batch = conform(&batch, &arrow_schema)?;

Review Comment:
   Good catch, I hadn't thought about empty batches at all. Went with your 
suggestion, so `commit` now rejects them up front, and I added a test that a 
commit with an empty batch fails without creating a snapshot. Fixed in bcdd706.



##########
crates/test-tools/src/append.rs:
##########
@@ -0,0 +1,463 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use datafusion::arrow::array::{RecordBatch, new_null_array};
+use datafusion::arrow::compute::cast;
+use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef;
+use datafusion::common::{exec_err, not_impl_err};
+use datafusion::error::Result;
+use datafusion::parquet::file::properties::WriterProperties;
+use iceberg::arrow::{RecordBatchPartitionSplitter, schema_to_arrow_schema};
+use iceberg::spec::{
+    DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestListWriter,
+    ManifestWriterBuilder, Operation, Snapshot, SnapshotSummaryCollector, 
Summary,
+};
+use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder;
+use iceberg::writer::file_writer::ParquetWriterBuilder;
+use iceberg::writer::file_writer::location_generator::{
+    DefaultFileNameGenerator, DefaultLocationGenerator,
+};
+use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder;
+use iceberg::writer::{IcebergWriter, IcebergWriterBuilder};
+
+use crate::FixtureTable;
+use crate::error::df_err;
+
+/// One commit that adds data files, started by [`FixtureTable::append`].
+///
+/// Each [`Append::file`] becomes exactly one Parquet data file, so tests
+/// control how rows are laid out across files. [`Append::commit`] adds them
+/// all in one snapshot on top of the current one. Snapshots are numbered 1,
+/// 2, 3, ..., unlike iceberg-rust's `fast_append`, which picks random ids, so
+/// that tests can show snapshot ids. Only adds data files: no deletes, and
+/// format version 2 only.
+pub struct Append<'a> {
+    table: &'a mut FixtureTable,
+    writer_properties: WriterProperties,
+    files: Vec<RecordBatch>,
+}
+
+impl<'a> Append<'a> {
+    pub(crate) fn new(table: &'a mut FixtureTable) -> Self {
+        Self {
+            table,
+            writer_properties: WriterProperties::default(),
+            files: Vec::new(),
+        }
+    }
+
+    /// Sets the properties every data file of this commit is written with.
+    pub fn writer_properties(mut self, writer_properties: WriterProperties) -> 
Self {
+        self.writer_properties = writer_properties;
+        self
+    }
+
+    /// Adds one data file containing `batch`.
+    ///
+    /// Columns are matched to the table's by name and cast to its types.
+    /// Columns of the table that `batch` lacks are null. On a partitioned
+    /// table every row must belong to the same partition.
+    pub fn file(mut self, batch: RecordBatch) -> Self {
+        self.files.push(batch);
+        self
+    }
+
+    /// Writes the data files and commits them as a new snapshot of the main
+    /// branch. Returns the data files in the order they were added.
+    pub async fn commit(self) -> Result<Vec<DataFile>> {
+        let table = self.table.table();
+        let metadata = table.metadata();
+        if metadata.format_version() != FormatVersion::V2 {
+            return not_impl_err!("fixtures only append to format version 2 
tables");
+        }
+        let file_io = table.file_io();
+        let schema = metadata.current_schema().clone();
+        let partition_spec = metadata.default_partition_spec().clone();
+        let arrow_schema = 
Arc::new(schema_to_arrow_schema(&schema).map_err(df_err)?);
+        let parent_snapshot_id = metadata.current_snapshot_id();
+        let snapshot_id = metadata
+            .snapshots()
+            .map(|s| s.snapshot_id())
+            .max()
+            .unwrap_or(0)
+            + 1;
+        let sequence_number = metadata.next_sequence_number();
+
+        let splitter = if partition_spec.is_unpartitioned() {
+            None
+        } else {
+            Some(
+                RecordBatchPartitionSplitter::try_new_with_computed_values(
+                    schema.clone(),
+                    partition_spec.clone(),
+                )
+                .map_err(df_err)?,
+            )
+        };
+        let writer_builder = 
DataFileWriterBuilder::new(RollingFileWriterBuilder::new(
+            ParquetWriterBuilder::new(self.writer_properties, schema.clone()),
+            // Never roll over, so that each batch is written to one file.
+            usize::MAX,
+            file_io.clone(),
+            DefaultLocationGenerator::new(metadata).map_err(df_err)?,
+            DefaultFileNameGenerator::new(
+                format!("{snapshot_id}"),
+                None,
+                DataFileFormat::Parquet,
+            ),
+        ));
+        let mut data_files = Vec::with_capacity(self.files.len());
+        for batch in self.files {
+            let batch = conform(&batch, &arrow_schema)?;
+            let partition_key = match &splitter {
+                None => None,
+                Some(splitter) => {
+                    let mut partitions = 
splitter.split(&batch).map_err(df_err)?;
+                    if partitions.len() != 1 {
+                        return exec_err!(
+                            "the rows of a data file must belong to one 
partition, \
+                             but they belong to {}",
+                            partitions.len()
+                        );
+                    }
+                    Some(partitions.remove(0).0)
+                }
+            };
+            let mut writer = 
writer_builder.build(partition_key).await.map_err(df_err)?;
+            writer.write(batch).await.map_err(df_err)?;
+            data_files.extend(writer.close().await.map_err(df_err)?);
+        }
+
+        let mut manifest_writer = ManifestWriterBuilder::new(
+            file_io
+                .new_output(format!(
+                    "{}/metadata/snap-{snapshot_id}-m0.avro",
+                    metadata.location()
+                ))
+                .map_err(df_err)?,
+            Some(snapshot_id),
+            schema.clone(),
+            partition_spec.as_ref().clone(),
+        )
+        .build_v2_data();
+        let mut summary = SnapshotSummaryCollector::default();
+        for data_file in &data_files {
+            manifest_writer
+                .add_file(data_file.clone(), sequence_number)

Review Comment:
   You're right, and it's exactly the kind of thing that would have quietly 
bitten us once fixtures start adding delete files. Switched to 
`UNASSIGNED_SEQUENCE_NUMBER`, and there's now a test that reads the raw 
manifest and checks the entries come out null, since going through the manifest 
list would hide it by filling them in. Fixed in bcdd706.



##########
crates/test-tools/src/table.rs:
##########
@@ -0,0 +1,224 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::str::FromStr;
+use std::sync::Arc;
+
+use datafusion::arrow::datatypes::SchemaRef;
+use datafusion::common::exec_datafusion_err;
+use datafusion::error::Result;
+use iceberg::arrow::schema_to_arrow_schema;
+use iceberg::memory::MemoryCatalog;
+use iceberg::spec::{
+    Schema, TableMetadata, Transform, UnboundPartitionField, 
UnboundPartitionSpec,
+};
+use iceberg::table::Table;
+use iceberg::{Catalog, MetadataLocation, TableCreation, TableIdent};
+use tempfile::TempDir;
+
+use crate::Append;
+use crate::error::df_err;
+
+/// Describes a new table, which [`TableBuilder::create`] then creates.
+///
+/// It covers what is fixed at creation: schema, partition spec and
+/// properties. The table starts with no snapshot; add data with
+/// [`FixtureTable::append`].
+pub struct TableBuilder {
+    catalog: Arc<MemoryCatalog>,
+    directory: Arc<TempDir>,
+    ident: TableIdent,
+    schema: Schema,
+    partition_fields: Vec<(String, Transform)>,
+    properties: HashMap<String, String>,
+}
+
+impl TableBuilder {
+    pub(crate) fn new(
+        catalog: Arc<MemoryCatalog>,
+        directory: Arc<TempDir>,
+        ident: TableIdent,
+        schema: Schema,
+    ) -> Self {
+        Self {
+            catalog,
+            directory,
+            ident,
+            schema,
+            partition_fields: Vec::new(),
+            properties: HashMap::new(),
+        }
+    }
+
+    /// Partitions the table by `transform` of top-level `column`. The
+    /// partition field is named after the column, suffixed with the transform
+    /// unless it's the identity, as Iceberg Java names them.
+    pub fn partition_by(
+        mut self,
+        column: impl Into<String>,
+        transform: Transform,
+    ) -> Self {
+        self.partition_fields.push((column.into(), transform));
+        self
+    }
+
+    /// Sets table property `key`, e.g. a `write.parquet.*` setting.
+    pub fn property(mut self, key: impl Into<String>, value: impl 
Into<String>) -> Self {
+        self.properties.insert(key.into(), value.into());
+        self
+    }
+
+    /// Creates the table in the catalog, with format version 2, creating its
+    /// namespace too if needed.
+    pub async fn create(self) -> Result<FixtureTable> {
+        let mut partition_spec = UnboundPartitionSpec::builder();
+        for (column, transform) in self.partition_fields {
+            let source = self.schema.field_by_name(&column).ok_or_else(|| {
+                exec_datafusion_err!("cannot partition by {column}: no such 
column")
+            })?;
+            partition_spec = partition_spec
+                .add_partition_field(
+                    UnboundPartitionField::builder()
+                        .source_ids(vec![source.id])
+                        .name(partition_field_name(&column, transform))
+                        .transform(transform)
+                        .build()
+                        .map_err(df_err)?,
+                )
+                .map_err(df_err)?;
+        }
+
+        let namespace = self.ident.namespace();
+        if !self
+            .catalog
+            .namespace_exists(namespace)
+            .await
+            .map_err(df_err)?
+        {
+            self.catalog
+                .create_namespace(namespace, HashMap::new())
+                .await
+                .map_err(df_err)?;
+        }
+        let table = self
+            .catalog
+            .create_table(
+                namespace,
+                TableCreation::builder()
+                    .name(self.ident.name().to_string())
+                    .schema(self.schema)
+                    .partition_spec(partition_spec.build())
+                    .properties(self.properties)
+                    .build(),
+            )
+            .await
+            .map_err(df_err)?;
+        Ok(FixtureTable {
+            catalog: self.catalog,
+            table,
+            _directory: self.directory,
+        })
+    }
+}
+
+fn partition_field_name(column: &str, transform: Transform) -> String {
+    let suffix = match transform {
+        Transform::Identity => return column.to_string(),
+        Transform::Bucket(_) => "bucket",
+        Transform::Truncate(_) => "trunc",
+        Transform::Year => "year",
+        Transform::Month => "month",
+        Transform::Day => "day",
+        Transform::Hour => "hour",
+        Transform::Void => "null",
+        Transform::Unknown => "unknown",
+    };
+    format!("{column}_{suffix}")
+}
+
+/// A table created by a [`TableBuilder`], and the handle for changing it.
+///
+/// It tracks the table as of its latest commit, so [`FixtureTable::table`]
+/// reflects every change made through it. Changes made in other ways, such as
+/// SQL `INSERT`s, aren't seen until the table is loaded from the catalog 
again.
+/// It keeps its files, which live in its catalog's directory, until dropped.
+pub struct FixtureTable {
+    catalog: Arc<MemoryCatalog>,
+    table: Table,
+    _directory: Arc<TempDir>,
+}
+
+impl FixtureTable {
+    /// The table as of its latest commit through this handle.
+    pub fn table(&self) -> &Table {
+        &self.table
+    }
+
+    /// The table's name in the catalog.
+    pub fn ident(&self) -> &TableIdent {
+        self.table.identifier()
+    }
+
+    /// The catalog holding the table, e.g. to register it with an Iceberg 
catalog
+    /// provider.
+    pub fn catalog(&self) -> Arc<dyn Catalog> {
+        self.catalog.clone()
+    }
+
+    /// The table's current schema, as a DataFusion schema.
+    pub fn schema(&self) -> Result<SchemaRef> {
+        let schema = 
schema_to_arrow_schema(self.table.metadata().current_schema())
+            .map_err(df_err)?;
+        Ok(Arc::new(schema))
+    }
+
+    /// The table's current metadata file.
+    pub fn metadata_location(&self) -> Result<&str> {
+        self.table.metadata_location_result().map_err(df_err)
+    }
+
+    /// Starts a commit that appends data files.
+    pub fn append(&mut self) -> Append<'_> {
+        Append::new(self)
+    }
+
+    /// Makes `metadata` the table's current metadata.
+    ///
+    /// iceberg-rust's catalogs only accept commits built from transaction
+    /// actions, and there is none for some of the changes fixtures need, such
+    /// as choosing snapshot ids. This writes the next metadata file and then
+    /// re-registers the table under it, which the in-memory catalog does
+    /// without touching the table's files.
+    pub(crate) async fn commit(&mut self, metadata: TableMetadata) -> 
Result<()> {
+        let location = MetadataLocation::from_str(self.metadata_location()?)
+            .map_err(df_err)?
+            .with_next_version();
+        metadata
+            .write_to(self.table.file_io(), &location)
+            .await
+            .map_err(df_err)?;
+        let ident = self.ident().clone();
+        self.catalog.drop_table(&ident).await.map_err(df_err)?;

Review Comment:
   Thanks for the repro, that's a nasty one to hit silently. Added the check, 
so committing now fails if the catalog no longer points at the metadata this 
handle last saw, plus a test that commits a change through the catalog and 
checks the next fixture append fails and leaves that change in place. Fixed in 
bcdd706.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to