szehon-ho commented on code in PR #3662:
URL: https://github.com/apache/iceberg-rust/pull/3662#discussion_r4545908937
##########
crates/iceberg/src/transaction/append.rs:
##########
@@ -1001,6 +1001,196 @@ mod tests {
assert_eq!(data_file, *manifest.entries()[0].data_file());
}
+ fn with_bucket_partition_spec(table: Table) -> Table {
+ let spec =
PartitionSpec::builder(table.metadata().current_schema().clone())
+ .add_partition_field("x", "x", Transform::Identity)
+ .unwrap()
+ .add_partition_field("y", "y_bucket", Transform::Bucket(16))
+ .unwrap()
+ .build()
+ .unwrap()
+ .into_unbound();
+ let metadata = table
+ .metadata()
+ .clone()
+ .into_builder(None)
+ .add_default_partition_spec(spec)
+ .unwrap()
+ .build()
+ .unwrap()
+ .metadata;
+ table.with_metadata(Arc::new(metadata))
+ }
+
+ #[tokio::test]
+ async fn test_fast_append_groups_files_by_partition_spec() {
+ for table in [
+ make_v1_table(),
+ make_v2_minimal_table(),
+ make_v3_minimal_table(),
+ ] {
+ let metadata = table
+ .metadata()
+ .clone()
+ .into_builder(None)
+ .set_properties(HashMap::from([(
+
TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT.to_string(),
+ "2".to_string(),
+ )]))
+ .unwrap()
+ .build()
+ .unwrap()
+ .metadata;
+ let table = table.with_metadata(Arc::new(metadata));
+ let table =
with_bucket_partition_spec(table).with_metadata_location(
+
"s3://bucket/test/location/metadata/00000-9c12d441-03fe-4693-9a96-a0705ddf69c1.metadata.json".to_string(),
+ );
+ let old_file = DataFileBuilder::default()
+ .content(DataContentType::Data)
+ .file_path("test/old.parquet".to_string())
+ .file_format(DataFileFormat::Parquet)
+ .file_size_in_bytes(100)
+ .record_count(1)
+ .partition_spec_id(0)
+ .partition(Struct::from_iter([Some(Literal::long(300))]))
+ .build()
+ .unwrap();
+ let mut other_old_file = old_file.clone();
+ other_old_file.file_path = "test/other-old.parquet".to_string();
+ let mut new_file = old_file.clone();
+ new_file.file_path = "test/new.parquet".to_string();
+ new_file.partition_spec_id =
table.metadata().default_partition_spec_id();
+ new_file.partition =
+ Struct::from_iter([Some(Literal::long(300)),
Some(Literal::int(3))]);
+
+ let action =
Transaction::new(&table).fast_append().add_data_files(vec![
+ new_file.clone(),
+ old_file.clone(),
+ other_old_file.clone(),
+ old_file.clone(),
Review Comment:
Removed the duplicate inputs and already-live check so this test focuses on
grouping and summaries.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]