c-thiel commented on code in PR #3623:
URL: https://github.com/apache/iceberg-rust/pull/3623#discussion_r4579044310
##########
crates/iceberg/src/arrow/reader/row_filter.rs:
##########
@@ -2037,4 +2038,127 @@ mod tests {
.await;
assert_eq!(rows(&on), 1);
}
+
+ async fn read_id_and_ts(
+ reader: ArrowReader,
+ path: &str,
+ schema: SchemaRef,
+ predicate: &Predicate,
+ project_ts: bool,
+ ) -> Vec<(i32, Option<i64>)> {
+ let task = FileScanTask::builder()
+ .with_file_size_in_bytes(std::fs::metadata(path).unwrap().len())
+ .with_start(0)
+ .with_length(0)
+ .with_data_file_path(path.to_string())
+ .with_data_file_format(DataFileFormat::Parquet)
+ .with_schema(schema.clone())
+ .with_project_field_ids(if project_ts { vec![1, 2] } else {
vec![1] })
+ .with_predicate(Some(predicate.clone().bind(schema,
true).unwrap()))
+ .with_case_sensitive(true)
+ .build()
+ .unwrap();
+ let batches: Vec<RecordBatch> = reader
+ .read(Box::pin(futures::stream::iter(vec![Ok(task)])))
+ .unwrap()
+ .stream()
+ .try_collect()
+ .await
+ .unwrap();
+
+ let mut rows = vec![];
+ for batch in &batches {
+ let ids = batch
+ .column(0)
+ .as_primitive::<arrow_array::types::Int32Type>();
+ let ts = project_ts.then(|| {
+ batch
+ .column(1)
+
.as_primitive::<arrow_array::types::TimestampMicrosecondType>()
+ });
+ for row in 0..batch.num_rows() {
+ rows.push((ids.value(row), ts.map(|ts| ts.value(row))));
+ }
+ }
+ rows
+ }
+
+ #[tokio::test]
+ async fn test_millisecond_timestamp_column_in_every_pruning_mode() {
+ let tmp = TempDir::new().unwrap();
+ let path = format!("{}/ts_millis.parquet",
tmp.path().to_str().unwrap());
+
+ for (primitive_type, tz) in [
+ (PrimitiveType::Timestamp, None),
+ (PrimitiveType::Timestamptz, Some("UTC")),
+ ] {
+ let ts =
+ TimestampMillisecondArray::from(vec![-1_500, 1_000,
2_000]).with_timezone_opt(tz);
+ let arrow_schema = Arc::new(ArrowSchema::new(vec![
+ field_with_id("id", DataType::Int32, 1),
+ field_with_id("ts", ts.data_type().clone(), 2),
+ ]));
+ let batch = RecordBatch::try_new(arrow_schema.clone(), vec![
+ Arc::new(Int32Array::from(vec![0, 1, 2])),
+ Arc::new(ts),
+ ])
+ .unwrap();
+ write_row_groups(&path, arrow_schema, vec![batch], true);
+
+ let schema = Arc::new(
+ Schema::builder()
+ .with_fields(vec![
+ NestedField::required(1, "id",
Type::Primitive(PrimitiveType::Int)).into(),
+ NestedField::required(2, "ts",
Type::Primitive(primitive_type.clone()))
+ .into(),
+ ])
+ .build()
+ .unwrap(),
+ );
+ let micros = |v| Datum::new(primitive_type.clone(),
PrimitiveLiteral::Long(v));
+ let cases = [
+ (Reference::new("ts").equal_to(micros(2_000_000)), vec![(
+ 2, 2_000_000,
+ )]),
+ (Reference::new("ts").less_than(micros(1_000_500)), vec![
+ (0, -1_500_000),
+ (1, 1_000_000),
+ ]),
+ (Reference::new("ts").equal_to(micros(1_000_500)), vec![]),
+ ];
+
+ for (mode, row_group_filtering, row_selection, bloom_filter) in [
+ ("default", true, false, false),
+ ("page index only", false, true, false),
+ ("bloom filter only", false, false, true),
+ ("all off", false, false, false),
+ ] {
+ let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(),
Runtime::current())
+ .with_row_group_filtering_enabled(row_group_filtering)
+ .with_row_selection_enabled(row_selection)
+ .with_bloom_filter_enabled(bloom_filter)
+ .build();
Review Comment:
#3514 removed these builder methods, so after merging main this test fails
with `no method named with_row_group_filtering_enabled`. Main's tests now pass
them through
`.with_options(ReaderOptions::builder().with_row_group_filtering_enabled(..).build())`.
##########
crates/iceberg/src/arrow/reader/projection.rs:
##########
@@ -214,6 +214,22 @@ impl ArrowReader {
}),
arrow_schema.metadata().clone(),
);
+ // A millisecond column under a nanosecond field would otherwise read
as nulls.
+ for field in projected_arrow_schema.flattened_fields() {
+ if let DataType::Timestamp(TimeUnit::Millisecond, _) =
field.data_type()
+ && let Some(field_id) = parquet_field_id(field)
+ && let Some(Type::Primitive(
+ PrimitiveType::TimestampNs | PrimitiveType::TimestamptzNs,
+ )) = iceberg_schema_of_task
+ .field_by_id(field_id)
+ .map(|field| field.field_type.as_ref())
+ {
+ return Err(invalid_data!(
+ "Unsupported Arrow data type: {}",
+ field.data_type()
+ ));
+ }
+ }
Review Comment:
This covers milliseconds under the nanosecond types, but the same drop
happens when the UTC flag differs: a naive `TIMESTAMP(MILLIS,false)` column
under `timestamptz`, or `TIMESTAMP(MILLIS,true)` under `timestamp`. That
includes the legacy `TIMESTAMP_MILLIS` converted type, which parquet-rs maps to
`Timestamp(ms, "UTC")`. `arrow_schema_to_schema` now maps the leaf to the other
timestamp type, `type_promotion_is_valid` rejects the pair, and the leaf is
dropped. Read together with another column, the value comes back as null, or
fails with "Missing required field" if the column is required. On main the same
read fails with `Unsupported Arrow data type: Timestamp(ms)`.
This returns `[true, false]` on this branch and an error on main:
```rust
#[test]
fn test_arrow_projection_mask_millisecond_column_with_other_utc_flag() {
let parquet_schema = SchemaDescriptor::new(Arc::new(
parse_message_type(
"
message schema {
required int32 id = 1;
optional int64 ts (TIMESTAMP(MILLIS,false)) = 2;
}",
)
.unwrap(),
));
let schema = Schema::builder()
.with_fields(vec![
NestedField::required(1, "id",
Type::Primitive(PrimitiveType::Int)).into(),
NestedField::optional(2, "ts",
Type::Primitive(PrimitiveType::Timestamptz)).into(),
])
.build()
.unwrap();
let arrow_schema = Arc::new(ArrowSchema::new(vec![
Field::new("id", DataType::Int32,
false).with_metadata(field_id_meta(1)),
Field::new("ts", DataType::Timestamp(TimeUnit::Millisecond, None),
true)
.with_metadata(field_id_meta(2)),
]));
let mask = ArrowReader::get_arrow_projection_mask(
&[1, 2],
&schema,
&parquet_schema,
&arrow_schema,
false,
);
println!("{mask:?}");
}
```
Could this check reject any projected millisecond leaf whose type from
`arrow_type_to_type` is not the table type? That covers the nanosecond case
too. The message could name the column and both types, since "Unsupported Arrow
data type: Timestamp(ms)" is no longer accurate now that milliseconds are
supported. (MICROS with a mismatched flag is already dropped on main, so I'd
leave that one out of this PR.)
##########
crates/iceberg/src/arrow/record_batch_transformer.rs:
##########
@@ -1590,6 +1598,14 @@ mod test {
Ok(transformer.process_record_batch(batch)?.column(1).clone())
}
+ #[test]
+ fn promote_millisecond_timestamp_overflow_errors() {
+ let column = NestedField::optional(2, "ts",
Type::Primitive(PrimitiveType::Timestamp));
+ let file_column =
Arc::new(TimestampMillisecondArray::from(vec![i64::MAX])) as ArrayRef;
+
+ assert!(transform_top_level(column, file_column).is_err());
+ }
Review Comment:
I think it would be good to assert the error kind here, not only `is_err()`.
A `date` case would also cover the spec requirement. This one fails with
`Unexpected` on this branch, so it goes together with the comment above:
```rust
#[test]
fn promote_date_beyond_timestamp_ns_range_errors() {
// Day 200000 is in the year 2517, after the last `timestamp_ns`.
let column = NestedField::optional(2, "d",
Type::Primitive(PrimitiveType::TimestampNs));
let file_column = Arc::new(Date32Array::from(vec![18000, 200000])) as
ArrayRef;
let err = transform_top_level(column, file_column).unwrap_err();
assert_eq!(err.kind(), ErrorKind::DataInvalid);
}
```
##########
crates/iceberg/src/arrow/record_batch_transformer.rs:
##########
@@ -454,7 +454,14 @@ impl PromotePlan {
fn apply(&self, array: &ArrayRef) -> Result<ArrayRef> {
match self {
PromotePlan::PassThrough => Ok(Arc::clone(array)),
- PromotePlan::Cast(target_type) => Ok(cast(array.as_ref(),
target_type)?),
+ PromotePlan::Cast(target_type) => Ok(cast_with_options(
+ array.as_ref(),
+ target_type,
+ &CastOptions {
+ safe: false,
+ ..Default::default()
+ },
+ )?),
Review Comment:
I think making this cast strict is the right call. I think it also does more
than handle the millisecond overflow: a v3 `date` promoted to `timestamp_ns`
overflows for dates after 2262-04-11, and the spec says such values "must
result in a runtime failure". So this would also fix the out-of-range half of
#3404 (#3485 left that part open). If you agree, maybe the description could
say "Part of #3404", and drop the sentence that promotions "widen and cannot
fail this way".
One small thing: through `?` the error ends up as `ErrorKind::Unexpected`
with the message "Arrow Schema Error". I think `DataInvalid` fits better, since
the value doesn't fit the target type. Something like:
```rust
.map_err(|e| {
invalid_data!("cannot promote a value from {} to {target_type}",
array.data_type())
.with_source(e)
})
```
##########
crates/iceberg/src/arrow/schema.rs:
##########
@@ -2776,16 +2807,95 @@ mod tests {
PrimitiveType::Geography(GeographyType::default()),
] {
assert_eq!(
- get_parquet_stat_min_as_datum(&primitive_type,
&stats).unwrap(),
+ get_parquet_stat_min_as_datum(&primitive_type, &stats,
None).unwrap(),
None
);
assert_eq!(
- get_parquet_stat_max_as_datum(&primitive_type,
&stats).unwrap(),
+ get_parquet_stat_max_as_datum(&primitive_type, &stats,
None).unwrap(),
None
);
}
}
+ #[test]
+ fn test_parquet_timestamp_unit() {
Review Comment:
nit: I think MICROS and NANOS columns (and the legacy `TIMESTAMP_MICROS`)
would be nice here too. They would also cover the NANOS mapping that the "no
bounds" behaviour depends on.
##########
crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs:
##########
@@ -1612,4 +1633,39 @@ mod tests {
let result = BloomFilterEvaluator::eval(&predicate,
&bloom_filters).unwrap();
assert!(result, "non-representable double must not prune");
}
+
+ #[test]
+ fn test_eq_probes_timestamp_in_column_unit() {
+ let schema = Arc::new(single_field_schema("ts",
PrimitiveType::Timestamp));
+ let filter = |unit, values: &[i64]| {
+ let mut sbbf = Sbbf::new_with_ndv_fpp(values.len() as u64,
0.01).unwrap();
+ for v in values {
+ sbbf.insert(v);
+ }
+ HashMap::from([(
+ 1,
+ ColumnBloomFilter::new(sbbf, PhysicalType::INT64, 0)
+ .with_timestamp_unit(Some(unit)),
+ )])
+ };
+ let millis = filter(TimeUnit::Millisecond, &[-1_500, 2_000]);
+ let nanos = filter(TimeUnit::Nanosecond, &[2_000_000_000]);
+
+ for (filters, micros, expected) in [
+ (&millis, 2_000_000, true),
+ (&millis, 2_000_500, false),
+ (&millis, -1_500_000, true),
+ (&nanos, 2_000_000, true),
+ ] {
Review Comment:
nit: I think a whole-millisecond value that is not in the filter would be a
good addition, for example `(&millis, 3_000_000, false)`. Right now, an
implementation that returns true for every whole millisecond would still pass.
##########
crates/iceberg/src/arrow/reader/predicate_visitor.rs:
##########
@@ -278,6 +278,27 @@ fn project_column(
}
}
+/// Like [`project_column`], but widens millisecond timestamps to microseconds
so
+/// comparisons with microsecond literals keep their precision.
+fn project_comparison_column(
+ batch: &RecordBatch,
+ column_idx: usize,
+) -> std::result::Result<ArrayRef, ArrowError> {
+ let column = project_column(batch, column_idx)?;
+
+ match column.data_type() {
+ DataType::Timestamp(TimeUnit::Millisecond, tz) => cast_with_options(
+ &column,
+ &DataType::Timestamp(TimeUnit::Microsecond, tz.clone()),
+ &CastOptions {
+ safe: false,
+ ..Default::default()
+ },
+ ),
+ _ => Ok(column),
+ }
+}
Review Comment:
Just a heads-up: I think this overlaps with #3540, which is approved. It
converts the filter columns to the table schema with a `RecordBatchTransformer`
in `get_row_filter`, so a millisecond column already reaches the predicate as
`Timestamp(µs)`. With that, I think this widening is no longer needed. #3540
also compares milliseconds under the nanosecond types exactly, while this path
widens the column to µs and then truncates the ns literal. The two PRs conflict
in `predicate_visitor.rs` and `row_filter.rs`. If #3540 lands first, I think
this commit and its two unit tests could go, and the end-to-end test could
stay. What do you think?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]