SteNicholas commented on code in PR #1308:
URL: https://github.com/apache/paimon-rust/pull/1308#discussion_r4439608295


##########
crates/integrations/datafusion/src/spark_functions.rs:
##########
@@ -0,0 +1,300 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use datafusion::arrow::array::{new_empty_array, ArrayRef, BooleanArray};
+use datafusion::arrow::compute::filter;
+use datafusion::arrow::datatypes::{DataType, Field, FieldRef, TimeUnit};
+use datafusion::common::config::ConfigOptions;
+use datafusion::common::utils::SingleRowListArrayBuilder;
+use datafusion::common::{DataFusionError, Result, ScalarValue};
+use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs};
+use datafusion::logical_expr::simplify::{ExprSimplifyResult, SimplifyContext};
+use datafusion::logical_expr::type_coercion::functions::fields_with_udf;
+use datafusion::logical_expr::{
+    Accumulator, AggregateUDF, AggregateUDFImpl, ColumnarValue, EmitTo, Expr, 
GroupsAccumulator,
+    ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,
+};
+use datafusion::prelude::SessionContext;
+use std::sync::Arc;
+
+// Normalize integration input types; all evaluation and simplification remain 
upstream.
+pub(crate) fn register(ctx: &SessionContext) {
+    for name in [
+        "base64",
+        "sha2",
+        "date_add",
+        "date_sub",
+        "factorial",
+        "mod",
+        "pmod",
+        "hex",
+        "unhex",
+        "hour",
+        "size",
+    ] {
+        let inner = Arc::clone(&ctx.state().scalar_functions()[name]);
+        ctx.register_udf(ScalarUDF::from(Inputs::new(inner)));
+    }
+    for name in ["collect_list", "collect_set", "try_sum", "avg"] {
+        let inner = Arc::clone(&ctx.state().aggregate_functions()[name]);
+        ctx.register_udaf(AggregateUDF::from(AggregateInputs { inner }));
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct AggregateInputs {
+    inner: Arc<AggregateUDF>,
+}
+impl AggregateUDFImpl for AggregateInputs {
+    fn name(&self) -> &str {
+        self.inner.name()
+    }
+    fn signature(&self) -> &Signature {
+        self.inner.signature()
+    }
+    fn coerce_types(&self, types: &[DataType]) -> Result<Vec<DataType>> {
+        self.inner.inner().coerce_types(types)
+    }
+    fn return_type(&self, types: &[DataType]) -> Result<DataType> {
+        self.inner.inner().return_type(types)
+    }
+    fn return_field(&self, fields: &[FieldRef]) -> Result<FieldRef> {
+        self.inner.return_field(fields)
+    }
+    fn accumulator(&self, args: AccumulatorArgs) -> Result<Box<dyn 
Accumulator>> {
+        match self.name() {
+            "collect_list" if !args.order_bys.is_empty() => 
Err(DataFusionError::NotImplemented(
+                "collect_list ORDER BY is not supported by DataFusion Spark 
54".into(),
+            )),
+            "collect_list" if args.is_distinct => {
+                
datafusion_spark::function::aggregate::collect_set().accumulator(args)
+            }
+            "try_sum" if args.is_distinct => 
Err(DataFusionError::NotImplemented(
+                "try_sum DISTINCT is not supported by DataFusion Spark 
54".into(),
+            )),
+            _ => self.inner.accumulator(args),
+        }
+    }
+    fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> {
+        if self.name() == "collect_list" && args.is_distinct {
+            
datafusion_spark::function::aggregate::collect_set().state_fields(args)
+        } else {
+            self.inner.state_fields(args)
+        }
+    }
+    fn default_value(&self, ty: &DataType) -> Result<ScalarValue> {
+        if matches!(self.name(), "collect_list" | "collect_set") {
+            let DataType::List(field) = ty else {
+                return Err(DataFusionError::Internal(
+                    "Spark collection return type must be a list".into(),
+                ));
+            };
+            Ok(
+                
SingleRowListArrayBuilder::new(new_empty_array(field.data_type()))
+                    .build_list_scalar(),
+            )
+        } else {
+            let value = self.inner.inner().default_value(ty)?;
+            if value == ScalarValue::Null {
+                ScalarValue::try_new_null(ty)
+            } else {
+                Ok(value)
+            }
+        }
+    }
+    fn reverse_expr(&self) -> datafusion::logical_expr::ReversedUDAF {
+        self.inner.inner().reverse_expr()
+    }
+    fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {
+        self.inner.groups_accumulator_supported(args)
+    }
+    fn create_groups_accumulator(
+        &self,
+        args: AccumulatorArgs,
+    ) -> Result<Box<dyn GroupsAccumulator>> {
+        Ok(Box::new(FilteredGroups(
+            self.inner.create_groups_accumulator(args)?,
+        )))
+    }
+}
+
+// Spark 54 AVG ignores its grouped FILTER argument. Pre-filter its 
already-evaluated
+// arrays and group indices, keeping the upstream vectorized accumulator and 
state.
+struct FilteredGroups(Box<dyn GroupsAccumulator>);
+impl FilteredGroups {
+    fn filter(
+        values: &[ArrayRef],
+        groups: &[usize],
+        predicate: &BooleanArray,
+    ) -> Result<(Vec<ArrayRef>, Vec<usize>)> {
+        let arrays = values
+            .iter()
+            .map(|value| filter(value.as_ref(), predicate).map_err(Into::into))
+            .collect::<Result<Vec<_>>>()?;
+        let groups = groups
+            .iter()
+            .zip(predicate.iter())
+            .filter_map(|(group, pass)| (pass == Some(true)).then_some(*group))
+            .collect();
+        Ok((arrays, groups))
+    }
+}
+impl GroupsAccumulator for FilteredGroups {
+    fn supports_convert_to_state(&self) -> bool {
+        self.0.supports_convert_to_state()
+    }
+    fn convert_to_state(
+        &self,
+        values: &[ArrayRef],
+        predicate: Option<&BooleanArray>,
+    ) -> Result<Vec<ArrayRef>> {
+        self.0.convert_to_state(values, predicate)
+    }
+    fn update_batch(
+        &mut self,
+        values: &[ArrayRef],
+        groups: &[usize],
+        predicate: Option<&BooleanArray>,
+        total: usize,
+    ) -> Result<()> {
+        if let Some(predicate) = predicate {
+            let (values, groups) = Self::filter(values, groups, predicate)?;
+            self.0.update_batch(&values, &groups, None, total)
+        } else {
+            self.0.update_batch(values, groups, None, total)
+        }
+    }
+    fn merge_batch(
+        &mut self,
+        values: &[ArrayRef],
+        groups: &[usize],
+        predicate: Option<&BooleanArray>,
+        total: usize,
+    ) -> Result<()> {
+        if let Some(predicate) = predicate {
+            let (values, groups) = Self::filter(values, groups, predicate)?;
+            self.0.merge_batch(&values, &groups, None, total)
+        } else {
+            self.0.merge_batch(values, groups, None, total)
+        }
+    }
+    fn evaluate(&mut self, emit: EmitTo) -> Result<ArrayRef> {
+        self.0.evaluate(emit)
+    }
+    fn state(&mut self, emit: EmitTo) -> Result<Vec<ArrayRef>> {
+        self.0.state(emit)
+    }
+    fn size(&self) -> usize {
+        self.0.size() + std::mem::size_of::<Self>()
+    }
+}
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct Inputs {
+    inner: Arc<ScalarUDF>,
+    signature: Signature,
+}
+impl Inputs {
+    fn new(inner: Arc<ScalarUDF>) -> Self {
+        Self {
+            signature: Signature::user_defined(inner.signature().volatility),
+            inner,
+        }
+    }
+}
+fn decoded(mut ty: &DataType) -> &DataType {
+    while let DataType::Dictionary(_, value) = ty {
+        ty = value;
+    }
+    ty
+}
+impl ScalarUDFImpl for Inputs {
+    fn name(&self) -> &str {
+        self.inner.name()
+    }
+    fn aliases(&self) -> &[String] {
+        self.inner.aliases()
+    }
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+    fn coerce_types(&self, types: &[DataType]) -> Result<Vec<DataType>> {
+        let mut types = types
+            .iter()
+            .map(|ty| decoded(ty).clone())
+            .collect::<Vec<_>>();
+        match (self.name(), types.as_mut_slice()) {
+            ("hex" | "unhex", [value]) if *value == DataType::Null => *value = 
DataType::Utf8,

Review Comment:
   [P2] Normalize BinaryView before delegating hex
   
   Please normalize `BinaryView` to `Binary` here, including when it is the 
decoded dictionary value type. For example, `SELECT 
hex(arrow_cast(decode('00ff', 'hex'), 'BinaryView'))` previously returned 
`00FF`, but the new adapter preserves `BinaryView`: upstream's binary signature 
accepts it unchanged, while Spark 54's `compute_hex` has no `BinaryView` 
execution branch and returns an unexpected-argument-type error. The removed 
implementation explicitly converted binary inputs to `Binary`. A regression 
test for this Arrow representation would cover the gap.



##########
crates/integrations/datafusion/src/spark_functions.rs:
##########
@@ -0,0 +1,300 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use datafusion::arrow::array::{new_empty_array, ArrayRef, BooleanArray};
+use datafusion::arrow::compute::filter;
+use datafusion::arrow::datatypes::{DataType, Field, FieldRef, TimeUnit};
+use datafusion::common::config::ConfigOptions;
+use datafusion::common::utils::SingleRowListArrayBuilder;
+use datafusion::common::{DataFusionError, Result, ScalarValue};
+use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs};
+use datafusion::logical_expr::simplify::{ExprSimplifyResult, SimplifyContext};
+use datafusion::logical_expr::type_coercion::functions::fields_with_udf;
+use datafusion::logical_expr::{
+    Accumulator, AggregateUDF, AggregateUDFImpl, ColumnarValue, EmitTo, Expr, 
GroupsAccumulator,
+    ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,
+};
+use datafusion::prelude::SessionContext;
+use std::sync::Arc;
+
+// Normalize integration input types; all evaluation and simplification remain 
upstream.
+pub(crate) fn register(ctx: &SessionContext) {
+    for name in [
+        "base64",
+        "sha2",
+        "date_add",
+        "date_sub",
+        "factorial",
+        "mod",
+        "pmod",
+        "hex",
+        "unhex",
+        "hour",
+        "size",
+    ] {
+        let inner = Arc::clone(&ctx.state().scalar_functions()[name]);
+        ctx.register_udf(ScalarUDF::from(Inputs::new(inner)));
+    }
+    for name in ["collect_list", "collect_set", "try_sum", "avg"] {
+        let inner = Arc::clone(&ctx.state().aggregate_functions()[name]);
+        ctx.register_udaf(AggregateUDF::from(AggregateInputs { inner }));
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct AggregateInputs {
+    inner: Arc<AggregateUDF>,
+}
+impl AggregateUDFImpl for AggregateInputs {
+    fn name(&self) -> &str {
+        self.inner.name()
+    }
+    fn signature(&self) -> &Signature {
+        self.inner.signature()
+    }
+    fn coerce_types(&self, types: &[DataType]) -> Result<Vec<DataType>> {
+        self.inner.inner().coerce_types(types)
+    }
+    fn return_type(&self, types: &[DataType]) -> Result<DataType> {
+        self.inner.inner().return_type(types)
+    }
+    fn return_field(&self, fields: &[FieldRef]) -> Result<FieldRef> {
+        self.inner.return_field(fields)
+    }
+    fn accumulator(&self, args: AccumulatorArgs) -> Result<Box<dyn 
Accumulator>> {
+        match self.name() {
+            "collect_list" if !args.order_bys.is_empty() => 
Err(DataFusionError::NotImplemented(
+                "collect_list ORDER BY is not supported by DataFusion Spark 
54".into(),
+            )),
+            "collect_list" if args.is_distinct => {
+                
datafusion_spark::function::aggregate::collect_set().accumulator(args)
+            }
+            "try_sum" if args.is_distinct => 
Err(DataFusionError::NotImplemented(
+                "try_sum DISTINCT is not supported by DataFusion Spark 
54".into(),
+            )),
+            _ => self.inner.accumulator(args),
+        }
+    }
+    fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> {
+        if self.name() == "collect_list" && args.is_distinct {
+            
datafusion_spark::function::aggregate::collect_set().state_fields(args)
+        } else {
+            self.inner.state_fields(args)
+        }
+    }
+    fn default_value(&self, ty: &DataType) -> Result<ScalarValue> {
+        if matches!(self.name(), "collect_list" | "collect_set") {
+            let DataType::List(field) = ty else {
+                return Err(DataFusionError::Internal(
+                    "Spark collection return type must be a list".into(),
+                ));
+            };
+            Ok(
+                
SingleRowListArrayBuilder::new(new_empty_array(field.data_type()))
+                    .build_list_scalar(),
+            )
+        } else {
+            let value = self.inner.inner().default_value(ty)?;
+            if value == ScalarValue::Null {
+                ScalarValue::try_new_null(ty)
+            } else {
+                Ok(value)
+            }
+        }
+    }
+    fn reverse_expr(&self) -> datafusion::logical_expr::ReversedUDAF {
+        self.inner.inner().reverse_expr()
+    }
+    fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {
+        self.inner.groups_accumulator_supported(args)
+    }
+    fn create_groups_accumulator(
+        &self,
+        args: AccumulatorArgs,
+    ) -> Result<Box<dyn GroupsAccumulator>> {
+        Ok(Box::new(FilteredGroups(
+            self.inner.create_groups_accumulator(args)?,
+        )))
+    }
+}
+
+// Spark 54 AVG ignores its grouped FILTER argument. Pre-filter its 
already-evaluated
+// arrays and group indices, keeping the upstream vectorized accumulator and 
state.
+struct FilteredGroups(Box<dyn GroupsAccumulator>);
+impl FilteredGroups {
+    fn filter(
+        values: &[ArrayRef],
+        groups: &[usize],
+        predicate: &BooleanArray,
+    ) -> Result<(Vec<ArrayRef>, Vec<usize>)> {
+        let arrays = values
+            .iter()
+            .map(|value| filter(value.as_ref(), predicate).map_err(Into::into))
+            .collect::<Result<Vec<_>>>()?;
+        let groups = groups
+            .iter()
+            .zip(predicate.iter())
+            .filter_map(|(group, pass)| (pass == Some(true)).then_some(*group))
+            .collect();
+        Ok((arrays, groups))
+    }
+}
+impl GroupsAccumulator for FilteredGroups {
+    fn supports_convert_to_state(&self) -> bool {
+        self.0.supports_convert_to_state()
+    }
+    fn convert_to_state(
+        &self,
+        values: &[ArrayRef],
+        predicate: Option<&BooleanArray>,
+    ) -> Result<Vec<ArrayRef>> {
+        self.0.convert_to_state(values, predicate)
+    }
+    fn update_batch(
+        &mut self,
+        values: &[ArrayRef],
+        groups: &[usize],
+        predicate: Option<&BooleanArray>,
+        total: usize,
+    ) -> Result<()> {
+        if let Some(predicate) = predicate {
+            let (values, groups) = Self::filter(values, groups, predicate)?;
+            self.0.update_batch(&values, &groups, None, total)
+        } else {
+            self.0.update_batch(values, groups, None, total)
+        }
+    }
+    fn merge_batch(
+        &mut self,
+        values: &[ArrayRef],
+        groups: &[usize],
+        predicate: Option<&BooleanArray>,
+        total: usize,
+    ) -> Result<()> {
+        if let Some(predicate) = predicate {
+            let (values, groups) = Self::filter(values, groups, predicate)?;
+            self.0.merge_batch(&values, &groups, None, total)
+        } else {
+            self.0.merge_batch(values, groups, None, total)
+        }
+    }
+    fn evaluate(&mut self, emit: EmitTo) -> Result<ArrayRef> {
+        self.0.evaluate(emit)
+    }
+    fn state(&mut self, emit: EmitTo) -> Result<Vec<ArrayRef>> {
+        self.0.state(emit)
+    }
+    fn size(&self) -> usize {
+        self.0.size() + std::mem::size_of::<Self>()
+    }
+}
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct Inputs {
+    inner: Arc<ScalarUDF>,
+    signature: Signature,
+}
+impl Inputs {
+    fn new(inner: Arc<ScalarUDF>) -> Self {
+        Self {
+            signature: Signature::user_defined(inner.signature().volatility),
+            inner,
+        }
+    }
+}
+fn decoded(mut ty: &DataType) -> &DataType {
+    while let DataType::Dictionary(_, value) = ty {
+        ty = value;
+    }
+    ty
+}
+impl ScalarUDFImpl for Inputs {
+    fn name(&self) -> &str {
+        self.inner.name()
+    }
+    fn aliases(&self) -> &[String] {
+        self.inner.aliases()
+    }
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+    fn coerce_types(&self, types: &[DataType]) -> Result<Vec<DataType>> {
+        let mut types = types
+            .iter()
+            .map(|ty| decoded(ty).clone())
+            .collect::<Vec<_>>();
+        match (self.name(), types.as_mut_slice()) {
+            ("hex" | "unhex", [value]) if *value == DataType::Null => *value = 
DataType::Utf8,
+            ("hour", [value]) if *value == DataType::Null => {
+                *value = DataType::Timestamp(TimeUnit::Microsecond, None)
+            }
+            ("size", [value]) if *value == DataType::Null => {
+                *value = DataType::List(Arc::new(Field::new("item", 
DataType::Null, true)))
+            }
+            ("date_add" | "date_sub", [date, days]) => {
+                if matches!(
+                    date,
+                    DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8 
| DataType::Null
+                ) {
+                    *date = DataType::Date32;
+                }
+                if matches!(days, DataType::Int64 | DataType::Null) {
+                    *days = DataType::Int32;
+                }
+            }
+            ("factorial", [value]) if *value == DataType::Int64 => *value = 
DataType::Int32,
+            ("mod" | "pmod", [left, right]) => {
+                if *left == DataType::Null {
+                    *left = if *right == DataType::Null {
+                        DataType::Int64
+                    } else {
+                        right.clone()
+                    };
+                }
+                if *right == DataType::Null {
+                    *right = left.clone();
+                }
+            }
+            _ => {}
+        }
+        let fields = types
+            .iter()
+            .map(|ty| Arc::new(Field::new("arg", ty.clone(), true)))
+            .collect::<Vec<_>>();
+        Ok(fields_with_udf(&fields, self.inner.as_ref())?

Review Comment:
   [P2] Preserve sha2 support for fixed-size binary
   
   Please normalize `FixedSizeBinary` inputs to `Binary` before calling the 
upstream coercion helper for `sha2`. For example, `SELECT 
sha2(arrow_cast(decode('00ff', 'hex'), 'FixedSizeBinary(2)'), 256)` now fails 
during coercion: Spark 54 requests `Native(logical_binary())`, which excludes 
the separate `FixedSizeBinary` native type, and its only allowed implicit 
source is string. The previous implementation accepted the full binary family 
and converted it to `Binary` before hashing. This also affects dictionaries 
whose decoded values are fixed-size binary.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to