gabotechs commented on code in PR #87:
URL: https://github.com/apache/datafusion-iceberg/pull/87#discussion_r4610358427


##########
crates/test-tools/src/append.rs:
##########
@@ -0,0 +1,463 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use datafusion::arrow::array::{RecordBatch, new_null_array};
+use datafusion::arrow::compute::cast;
+use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef;
+use datafusion::common::{exec_err, not_impl_err};
+use datafusion::error::Result;
+use datafusion::parquet::file::properties::WriterProperties;
+use iceberg::arrow::{RecordBatchPartitionSplitter, schema_to_arrow_schema};
+use iceberg::spec::{
+    DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestListWriter,
+    ManifestWriterBuilder, Operation, Snapshot, SnapshotSummaryCollector, 
Summary,
+};
+use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder;
+use iceberg::writer::file_writer::ParquetWriterBuilder;
+use iceberg::writer::file_writer::location_generator::{
+    DefaultFileNameGenerator, DefaultLocationGenerator,
+};
+use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder;
+use iceberg::writer::{IcebergWriter, IcebergWriterBuilder};
+
+use crate::FixtureTable;
+use crate::error::df_err;
+
+/// One commit that adds data files, started by [`FixtureTable::append`].
+///
+/// Each [`Append::file`] becomes exactly one Parquet data file, so tests
+/// control how rows are laid out across files. [`Append::commit`] adds them
+/// all in one snapshot on top of the current one. Snapshots are numbered 1,
+/// 2, 3, ..., unlike iceberg-rust's `fast_append`, which picks random ids, so
+/// that tests can show snapshot ids. Only adds data files: no deletes, and
+/// format version 2 only.
+pub struct Append<'a> {
+    table: &'a mut FixtureTable,
+    writer_properties: WriterProperties,
+    files: Vec<RecordBatch>,
+}
+
+impl<'a> Append<'a> {
+    pub(crate) fn new(table: &'a mut FixtureTable) -> Self {
+        Self {
+            table,
+            writer_properties: WriterProperties::default(),
+            files: Vec::new(),
+        }
+    }
+
+    /// Sets the properties every data file of this commit is written with.
+    pub fn writer_properties(mut self, writer_properties: WriterProperties) -> 
Self {
+        self.writer_properties = writer_properties;
+        self
+    }
+
+    /// Adds one data file containing `batch`.
+    ///
+    /// Columns are matched to the table's by name and cast to its types.
+    /// Columns of the table that `batch` lacks are null. On a partitioned
+    /// table every row must belong to the same partition.
+    pub fn file(mut self, batch: RecordBatch) -> Self {
+        self.files.push(batch);
+        self
+    }
+
+    /// Writes the data files and commits them as a new snapshot of the main
+    /// branch. Returns the data files in the order they were added.
+    pub async fn commit(self) -> Result<Vec<DataFile>> {
+        let table = self.table.table();
+        let metadata = table.metadata();
+        if metadata.format_version() != FormatVersion::V2 {
+            return not_impl_err!("fixtures only append to format version 2 
tables");
+        }
+        let file_io = table.file_io();
+        let schema = metadata.current_schema().clone();
+        let partition_spec = metadata.default_partition_spec().clone();
+        let arrow_schema = 
Arc::new(schema_to_arrow_schema(&schema).map_err(df_err)?);
+        let parent_snapshot_id = metadata.current_snapshot_id();
+        let snapshot_id = metadata
+            .snapshots()
+            .map(|s| s.snapshot_id())
+            .max()
+            .unwrap_or(0)
+            + 1;
+        let sequence_number = metadata.next_sequence_number();
+
+        let splitter = if partition_spec.is_unpartitioned() {
+            None
+        } else {
+            Some(
+                RecordBatchPartitionSplitter::try_new_with_computed_values(
+                    schema.clone(),
+                    partition_spec.clone(),
+                )
+                .map_err(df_err)?,
+            )
+        };
+        let writer_builder = 
DataFileWriterBuilder::new(RollingFileWriterBuilder::new(
+            ParquetWriterBuilder::new(self.writer_properties, schema.clone()),
+            // Never roll over, so that each batch is written to one file.
+            usize::MAX,
+            file_io.clone(),
+            DefaultLocationGenerator::new(metadata).map_err(df_err)?,
+            DefaultFileNameGenerator::new(
+                format!("{snapshot_id}"),
+                None,
+                DataFileFormat::Parquet,
+            ),
+        ));
+        let mut data_files = Vec::with_capacity(self.files.len());
+        for batch in self.files {
+            let batch = conform(&batch, &arrow_schema)?;
+            let partition_key = match &splitter {
+                None => None,
+                Some(splitter) => {
+                    let mut partitions = 
splitter.split(&batch).map_err(df_err)?;
+                    if partitions.len() != 1 {
+                        return exec_err!(
+                            "the rows of a data file must belong to one 
partition, \
+                             but they belong to {}",
+                            partitions.len()
+                        );
+                    }
+                    Some(partitions.remove(0).0)
+                }
+            };
+            let mut writer = 
writer_builder.build(partition_key).await.map_err(df_err)?;
+            writer.write(batch).await.map_err(df_err)?;
+            data_files.extend(writer.close().await.map_err(df_err)?);
+        }
+
+        let mut manifest_writer = ManifestWriterBuilder::new(
+            file_io
+                .new_output(format!(
+                    "{}/metadata/snap-{snapshot_id}-m0.avro",
+                    metadata.location()
+                ))
+                .map_err(df_err)?,
+            Some(snapshot_id),
+            schema.clone(),
+            partition_spec.as_ref().clone(),
+        )
+        .build_v2_data();
+        let mut summary = SnapshotSummaryCollector::default();
+        for data_file in &data_files {
+            manifest_writer
+                .add_file(data_file.clone(), sequence_number)

Review Comment:
   👍 sounds good. Switched to `UNASSIGNED_SEQUENCE_NUMBER`, and there's now a 
test that reads the raw manifest and checks the entries come out null, since 
going through the manifest list would hide it by filling them in. Fixed in 
bcdd706.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to