JingsongLi commented on code in PR #1358:
URL: https://github.com/apache/paimon-rust/pull/1358#discussion_r4519947974
##########
crates/integrations/datafusion/src/sql_context.rs:
##########
@@ -3765,17 +3770,45 @@ fn parse_static_partitions(
let field = field_map.get(col_name.as_str()).ok_or_else(|| {
DataFusionError::Plan(format!("Column '{col_name}' not found in
table schema"))
})?;
- let datum = sql_expr_to_datum(val_expr, field.data_type())?;
+ let datum = sql_expr_to_datum(val_expr, field.data_type(),
legacy_partition_name)?;
result.insert(col_name, Some(datum));
}
Ok(result)
}
/// Convert a SQL literal expression to a Paimon Datum.
-fn sql_expr_to_datum(expr: &SqlExpr, data_type: &PaimonDataType) ->
DFResult<Datum> {
+fn sql_expr_to_datum(
+ expr: &SqlExpr,
+ data_type: &PaimonDataType,
+ legacy_partition_name: bool,
+) -> DFResult<Datum> {
+ let parse_text = |text: &str| {
+ crate::procedures::parse_partition_value(text, data_type,
legacy_partition_name).ok_or_else(
+ || {
+ DataFusionError::Plan(format!(
+ "Invalid partition value '{text}' for type {data_type:?}"
+ ))
+ },
+ )
+ };
let (value, negate) = match expr {
SqlExpr::Value(v) => (&v.value, false),
+ SqlExpr::TypedString(typed)
+ if matches!(
+ (&typed.data_type, data_type),
+ (SqlDataType::Date, PaimonDataType::Date(_))
+ | (
+ SqlDataType::Timestamp(..),
+ PaimonDataType::Timestamp(_) |
PaimonDataType::LocalZonedTimestamp(_)
+ )
+ ) =>
+ {
+ let text = typed.value.value.clone().into_string().ok_or_else(|| {
+ DataFusionError::Plan(format!("Unsupported partition value
expression: {expr}"))
+ })?;
+ return parse_text(&text);
Review Comment:
**[P1] Use the session time zone when resolving typed TIMESTAMP WITH TIME
ZONE literals.** I set `datafusion.execution.time_zone` to `'+08:00'`, created
a table partitioned by `k TIMESTAMP WITH TIME ZONE`, and inserted `(1,
TIMESTAMP WITH TIME ZONE '2024-03-01 10:00:00')` and `(2, TIMESTAMP WITH TIME
ZONE '2024-03-01 18:00:00')`. `ALTER TABLE ... DROP PARTITION (k = TIMESTAMP
WITH TIME ZONE '2024-03-01 10:00:00')` left row 1 and deleted row 2. The INSERT
interprets the typed literal in the session zone, while this forwards only its
text to a physical partition-name parser, which uses UTC with the default
`partition.legacy-name`. Please evaluate typed literals with the same SQL
session semantics as INSERT and test two partitions where the wrong instant
also exists.
##########
crates/integrations/datafusion/src/sql_context.rs:
##########
@@ -3942,14 +3985,20 @@ fn datum_to_constant_array(
Datum::Double(v) => Ok(Arc::new(Float64Array::from(vec![*v;
num_rows]))),
Datum::String(v) => Ok(Arc::new(StringArray::from(vec![v.as_str();
num_rows]))),
Datum::Date(v) => Ok(Arc::new(Date32Array::from(vec![*v;
num_rows]))),
- Datum::Time(_)
- | Datum::Timestamp { .. }
- | Datum::LocalZonedTimestamp { .. }
- | Datum::Decimal { .. }
- | Datum::Bytes(_)
- | Datum::Variant { .. } => Err(DataFusionError::Plan(format!(
- "Unsupported datum type for partition column: {d}"
- ))),
+ Datum::Timestamp { .. } | Datum::LocalZonedTimestamp { .. } |
Datum::Decimal { .. } => {
+ let arrow_type =
+
paimon::arrow::paimon_type_to_arrow(data_type).map_err(to_datafusion_error)?;
+ crate::physical_plan::scan::datum_to_scalar(d.clone(),
&arrow_type)
Review Comment:
**[P2] Use the same timestamp precision for the written column and the
partition predicate.** On a table partitioned by `k TIMESTAMP(3)`, a normal
INSERT with `TIMESTAMP '2024-03-01 10:11:12.123456'` succeeds and stores
`.123`. Using that same literal in `INSERT OVERWRITE ... PARTITION (k =
TIMESTAMP '2024-03-01 10:11:12.123456') VALUES (9)` fails with `file ... does
not belong to this partition`: the static column conversion drops
sub-millisecond digits, but the predicate retains them. Please use one
canonical value for both the written column and the commit filter, or reject
nonrepresentable values during planning. Cover excess precision alongside the
exact-millisecond case.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]