anoopj commented on code in PR #2985:
URL: https://github.com/apache/iceberg-rust/pull/2985#discussion_r3763831892
##########
crates/iceberg/src/arrow/record_batch_transformer.rs:
##########
@@ -2293,6 +2395,80 @@ mod test {
assert!((0..3).all(|i| seq_col.is_null(i)));
}
+ /// Builds a transformer + a file batch for the coalesce case: `id` plus a
physical
+ /// `_last_updated_sequence_number` column carrying `seq_values`.
+ fn coalesce_transformer_and_batch(
+ seq_values: Vec<Option<i64>>,
+ id_values: Vec<i32>,
+ fallback: i64,
+ ) -> (RecordBatchTransformer, RecordBatch) {
+ use crate::metadata_columns::{
+ RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER,
+ RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
+ };
+ use crate::spec::Datum;
+
+ let snapshot_schema = Arc::new(
+ Schema::builder()
+ .with_schema_id(0)
+ .with_fields(vec![
+ NestedField::required(1, "id",
Type::Primitive(PrimitiveType::Int)).into(),
+ ])
+ .build()
+ .unwrap(),
+ );
+ let parquet_schema = Arc::new(ArrowSchema::new(vec![
+ field_with_id("id", DataType::Int32, false, 1),
+ field_with_id(
+ RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER,
+ DataType::Int64,
+ true,
+ RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
+ ),
+ ]));
+ let projected_field_ids = [1,
RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER];
+ let transformer = RecordBatchTransformerBuilder::new(snapshot_schema,
&projected_field_ids)
+ .with_coalesced_metadata_column(
+ RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
+ Datum::long(fallback),
+ )
+ .build();
+ let batch = RecordBatch::try_new(parquet_schema, vec![
+ Arc::new(Int32Array::from(id_values)),
+ Arc::new(Int64Array::from(seq_values)),
+ ])
+ .unwrap();
+ (transformer, batch)
+ }
+
+ #[test]
+ fn last_updated_sequence_number_coalesce_column() {
+ let (mut transformer, batch) =
+ coalesce_transformer_and_batch(vec![Some(5), None, Some(8)],
vec![10, 20, 30], 9);
+ let result = transformer.process_record_batch(batch).unwrap();
+
+ // Per-row value where non-null; the fallback (9) where null.
+ let seq_col = cast(result.column(1), &DataType::Int64).unwrap();
Review Comment:
Switched 6 tests to a shared helper that looks up column by name.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
