JingsongLi commented on code in PR #1358:
URL: https://github.com/apache/paimon-rust/pull/1358#discussion_r4519947974


##########
crates/integrations/datafusion/src/sql_context.rs:
##########
@@ -3765,17 +3770,45 @@ fn parse_static_partitions(
         let field = field_map.get(col_name.as_str()).ok_or_else(|| {
             DataFusionError::Plan(format!("Column '{col_name}' not found in 
table schema"))
         })?;
-        let datum = sql_expr_to_datum(val_expr, field.data_type())?;
+        let datum = sql_expr_to_datum(val_expr, field.data_type(), 
legacy_partition_name)?;
         result.insert(col_name, Some(datum));
     }
 
     Ok(result)
 }
 
 /// Convert a SQL literal expression to a Paimon Datum.
-fn sql_expr_to_datum(expr: &SqlExpr, data_type: &PaimonDataType) -> 
DFResult<Datum> {
+fn sql_expr_to_datum(
+    expr: &SqlExpr,
+    data_type: &PaimonDataType,
+    legacy_partition_name: bool,
+) -> DFResult<Datum> {
+    let parse_text = |text: &str| {
+        crate::procedures::parse_partition_value(text, data_type, 
legacy_partition_name).ok_or_else(
+            || {
+                DataFusionError::Plan(format!(
+                    "Invalid partition value '{text}' for type {data_type:?}"
+                ))
+            },
+        )
+    };
     let (value, negate) = match expr {
         SqlExpr::Value(v) => (&v.value, false),
+        SqlExpr::TypedString(typed)
+            if matches!(
+                (&typed.data_type, data_type),
+                (SqlDataType::Date, PaimonDataType::Date(_))
+                    | (
+                        SqlDataType::Timestamp(..),
+                        PaimonDataType::Timestamp(_) | 
PaimonDataType::LocalZonedTimestamp(_)
+                    )
+            ) =>
+        {
+            let text = typed.value.value.clone().into_string().ok_or_else(|| {
+                DataFusionError::Plan(format!("Unsupported partition value 
expression: {expr}"))
+            })?;
+            return parse_text(&text);

Review Comment:
   **[P1] Use the session time zone when resolving typed TIMESTAMP WITH TIME 
ZONE literals.** I set `datafusion.execution.time_zone` to `'+08:00'`, created 
a table partitioned by `k TIMESTAMP WITH TIME ZONE`, and inserted `(1, 
TIMESTAMP WITH TIME ZONE '2024-03-01 10:00:00')` and `(2, TIMESTAMP WITH TIME 
ZONE '2024-03-01 18:00:00')`. `ALTER TABLE ... DROP PARTITION (k = TIMESTAMP 
WITH TIME ZONE '2024-03-01 10:00:00')` left row 1 and deleted row 2. The INSERT 
interprets the typed literal in the session zone, while this forwards only its 
text to a physical partition-name parser, which uses UTC with the default 
`partition.legacy-name`. Please evaluate typed literals with the same SQL 
session semantics as INSERT and test two partitions where the wrong instant 
also exists.



##########
crates/integrations/datafusion/src/sql_context.rs:
##########
@@ -3942,14 +3985,20 @@ fn datum_to_constant_array(
             Datum::Double(v) => Ok(Arc::new(Float64Array::from(vec![*v; 
num_rows]))),
             Datum::String(v) => Ok(Arc::new(StringArray::from(vec![v.as_str(); 
num_rows]))),
             Datum::Date(v) => Ok(Arc::new(Date32Array::from(vec![*v; 
num_rows]))),
-            Datum::Time(_)
-            | Datum::Timestamp { .. }
-            | Datum::LocalZonedTimestamp { .. }
-            | Datum::Decimal { .. }
-            | Datum::Bytes(_)
-            | Datum::Variant { .. } => Err(DataFusionError::Plan(format!(
-                "Unsupported datum type for partition column: {d}"
-            ))),
+            Datum::Timestamp { .. } | Datum::LocalZonedTimestamp { .. } | 
Datum::Decimal { .. } => {
+                let arrow_type =
+                    
paimon::arrow::paimon_type_to_arrow(data_type).map_err(to_datafusion_error)?;
+                crate::physical_plan::scan::datum_to_scalar(d.clone(), 
&arrow_type)

Review Comment:
   **[P2] Use the same timestamp precision for the written column and the 
partition predicate.** On a table partitioned by `k TIMESTAMP(3)`, a normal 
INSERT with `TIMESTAMP '2024-03-01 10:11:12.123456'` succeeds and stores 
`.123`. Using that same literal in `INSERT OVERWRITE ... PARTITION (k = 
TIMESTAMP '2024-03-01 10:11:12.123456') VALUES (9)` fails with `file ... does 
not belong to this partition`: the static column conversion drops 
sub-millisecond digits, but the predicate retains them. Please use one 
canonical value for both the written column and the commit filter, or reject 
nonrepresentable values during planning. Cover excess precision alongside the 
exact-millisecond case.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to