SteNicholas commented on code in PR #1311:
URL: https://github.com/apache/paimon-rust/pull/1311#discussion_r4442088797
##########
crates/integrations/datafusion/src/spark_functions.rs:
##########
@@ -65,14 +74,95 @@ fn date_part() -> Arc<ScalarUDF> {
}
#[derive(Debug)]
-pub(crate) struct DatePartPlanner;
+pub(crate) struct FunctionPlanner;
-impl ExprPlanner for DatePartPlanner {
+impl ExprPlanner for FunctionPlanner {
fn plan_extract(&self, args: Vec<Expr>) ->
Result<PlannerResult<Vec<Expr>>> {
Ok(PlannerResult::Planned(Expr::ScalarFunction(
ScalarFunction::new_udf(date_part(), args),
)))
}
+ fn plan_substring(&self, args: Vec<Expr>) ->
Result<PlannerResult<Vec<Expr>>> {
+ Ok(PlannerResult::Planned(Expr::ScalarFunction(
+ ScalarFunction::new_udf(
+ Arc::new(ScalarUDF::from(Inputs::new(
+ datafusion_spark::function::string::substring(),
+ ))),
+ args,
+ ),
+ )))
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct ByteLength {
+ bits: bool,
+ signature: Signature,
+}
+
+impl ScalarUDFImpl for ByteLength {
+ fn name(&self) -> &str {
+ if self.bits {
+ "bit_length"
+ } else {
+ "octet_length"
+ }
+ }
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+ fn coerce_types(&self, types: &[DataType]) -> Result<Vec<DataType>> {
+ let [ty] = types else {
+ return Err(DataFusionError::Plan(format!(
+ "{} expects one argument",
+ self.name()
+ )));
+ };
+ Ok(vec![match decoded(ty) {
Review Comment:
**[P2] Preserve run-end-encoded string inputs**
The replaced DataFusion `octet_length` and `bit_length` functions use
native-string coercion, which automatically decodes `RunEndEncoded` inputs.
Here, `decoded()` unwraps only dictionaries, so a string column represented as
`RunEndEncoded(Int32, Utf8)` reaches the unsupported-type branch. Existing
queries such as `SELECT octet_length(s), bit_length(s) FROM t` therefore fail
during planning for those columns.
Please unwrap `RunEndEncoded` value types as well, or retain the upstream
string-coercion path, and add a `RunArray` regression test for both functions.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
