szehon-ho commented on code in PR #3662:
URL: https://github.com/apache/iceberg-rust/pull/3662#discussion_r4545910197


##########
crates/iceberg/src/transaction/snapshot/simple.rs:
##########
@@ -128,24 +126,24 @@ impl SimpleSnapshotProducer {
         .await
     }
 
-    /// Checks that every file is written under the table's default partition
-    /// spec with a compatible partition value.
+    /// Checks that every file uses a known partition spec with a compatible
+    /// partition value.
     ///
     /// Runs against the table the manifests are written for; a later rebase
     /// does not invalidate manifests already written under a spec the table
     /// still has.
     fn validate_added_data_files(&self, table: &Table) -> Result<()> {
         let metadata = table.metadata();
         for data_file in &self.intent.data_files {
-            // TODO(#3500): accept any spec known to the table and group the
-            // files by spec when writing manifests, instead of requiring the
-            // default spec.
-            if metadata.default_partition_spec_id() != 
data_file.partition_spec_id {
-                return Err(invalid_data!(
-                    "Data file partition spec id does not match table default 
partition spec id"
-                ));
-            }
-            validate_partition_value(data_file.partition(), 
metadata.default_partition_type())?;
+            let spec = metadata
+                .partition_spec_by_id(data_file.partition_spec_id)
+                .ok_or_else(|| {
+                    invalid_data!("Unknown partition spec id {}", 
data_file.partition_spec_id)

Review Comment:
   Added file paths to both errors. Grouping now resolves the spec and 
partition type once per group and passes the resolved spec to the manifest 
writer.



##########
crates/iceberg/src/transaction/snapshot/simple.rs:
##########
@@ -180,28 +178,25 @@ impl SimpleSnapshotProducer {
 
         self.validate_added_data_files(table)?;

Review Comment:
   Added an explicit spec-existence check before reusing cached manifests, with 
a regression test for removal of a cached spec.



##########
crates/iceberg/src/transaction/snapshot/simple.rs:
##########
@@ -411,14 +406,96 @@ mod tests {
         assert_eq!(producer.manifest_counter, 1);
     }
 
+    #[tokio::test]
+    async fn test_append_uses_file_spec_after_default_spec_changes() {
+        let table = make_v2_minimal_table();
+        let file = data_file(&table, "test/1.parquet");
+        let mut state = producer(vec![file.clone()]);
+        let first_snapshot = added_snapshot(&mut 
state.apply(&table).await.unwrap());
+        let first_list = table
+            .manifest_list_reader(&first_snapshot)
+            .load()
+            .await
+            .unwrap();
+
+        let metadata = table
+            .metadata()
+            .clone()
+            .into_builder(None)
+            
.add_default_partition_spec(UnboundPartitionSpec::builder().build())

Review Comment:
   The retry now switches to a spec with more fields and compares the 
summaries. Removed the fresh-producer half.



##########
crates/iceberg/src/transaction/snapshot/simple.rs:
##########
@@ -180,28 +178,25 @@ impl SimpleSnapshotProducer {
 
         self.validate_added_data_files(table)?;
 
-        // Allocate every path before writing, so each is recorded as owned
-        // even if the write fails part-way: the path is write-once and a
-        // partial file is an orphan to clean up like any other.
-        //
-        // TODO(#3500): group the files by partition spec and write one
-        // manifest per group.
+        // BTreeMap keeps added manifests in deterministic partition spec 
order.
+        let mut files_by_spec: BTreeMap<i32, Vec<&DataFile>> = BTreeMap::new();
+        for file in &self.intent.data_files {

Review Comment:
   Changed `validated_added_data_files` to return the validated groups with 
their resolved specs, which are passed directly to manifest writing.



##########
crates/iceberg/src/transaction/snapshot/simple.rs:
##########
@@ -128,24 +126,24 @@ impl SimpleSnapshotProducer {
         .await
     }
 
-    /// Checks that every file is written under the table's default partition
-    /// spec with a compatible partition value.
+    /// Checks that every file uses a known partition spec with a compatible
+    /// partition value.
     ///
     /// Runs against the table the manifests are written for; a later rebase
     /// does not invalidate manifests already written under a spec the table
     /// still has.
     fn validate_added_data_files(&self, table: &Table) -> Result<()> {
         let metadata = table.metadata();
         for data_file in &self.intent.data_files {
-            // TODO(#3500): accept any spec known to the table and group the
-            // files by spec when writing manifests, instead of requiring the
-            // default spec.
-            if metadata.default_partition_spec_id() != 
data_file.partition_spec_id {
-                return Err(invalid_data!(
-                    "Data file partition spec id does not match table default 
partition spec id"
-                ));
-            }
-            validate_partition_value(data_file.partition(), 
metadata.default_partition_type())?;
+            let spec = metadata
+                .partition_spec_by_id(data_file.partition_spec_id)
+                .ok_or_else(|| {
+                    invalid_data!("Unknown partition spec id {}", 
data_file.partition_spec_id)
+                })?;
+            validate_partition_value(
+                data_file.partition(),
+                &spec.partition_type(metadata.current_schema())?,

Review Comment:
   Keeping dropped-source-column support for a follow-up, as agreed in the 
review.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to