alamb commented on code in PR #9583: URL: https://github.com/apache/arrow-datafusion/pull/9583#discussion_r1523453847
########## datafusion/optimizer/src/analyzer/function_rewrite.rs: ########## @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! [`ApplyFunctionRewrites`] to replace `Expr`s with function calls (e.g `||` to array_concat`) + +use super::AnalyzerRule; +use datafusion_common::config::ConfigOptions; +use datafusion_common::tree_node::{Transformed, TreeNodeRewriter}; +use datafusion_common::{DFSchema, Result}; +use datafusion_expr::expr_rewriter::{rewrite_preserving_name, FunctionRewrite}; +use datafusion_expr::utils::merge_schema; +use datafusion_expr::{Expr, LogicalPlan}; +use std::sync::Arc; + +/// Analyzer rule that invokes [`FunctionRewrite`]s on expressions +#[derive(Default)] +pub struct ApplyFunctionRewrites { + /// Expr --> Function writes to apply + function_rewrites: Vec<Arc<dyn FunctionRewrite + Send + Sync>>, +} + +impl ApplyFunctionRewrites { + pub fn new(function_rewrites: Vec<Arc<dyn FunctionRewrite + Send + Sync>>) -> Self { + Self { function_rewrites } + } +} + +impl AnalyzerRule for ApplyFunctionRewrites { + fn name(&self) -> &str { + "apply_function_rewrites" + } + + fn analyze(&self, plan: LogicalPlan, options: &ConfigOptions) -> Result<LogicalPlan> { + self.analyze_internal(&plan, options) + } +} + +impl ApplyFunctionRewrites { + fn analyze_internal( + &self, + plan: &LogicalPlan, + options: &ConfigOptions, + ) -> Result<LogicalPlan> { + // optimize child plans first + let new_inputs = plan + .inputs() + .iter() + .map(|p| self.analyze_internal(p, options)) + .collect::<Result<Vec<_>>>()?; + + // get schema representing all available input fields. This is used for data type + // resolution only, so order does not matter here + let mut schema = merge_schema(new_inputs.iter().collect()); + + if let LogicalPlan::TableScan(ts) = plan { + let source_schema = + DFSchema::try_from_qualified_schema(&ts.table_name, &ts.source.schema())?; + schema.merge(&source_schema); + } + + let mut expr_rewrite = OperatorToFunctionRewriter { + function_rewrites: &self.function_rewrites, + options, + schema: &schema, + }; + + let new_expr = plan + .expressions() + .into_iter() + .map(|expr| { + // ensure names don't change: + // https://github.com/apache/arrow-datafusion/issues/3555 + rewrite_preserving_name(expr, &mut expr_rewrite) + }) + .collect::<Result<Vec<_>>>()?; + + plan.with_new_exprs(new_expr, new_inputs) + } +} +struct OperatorToFunctionRewriter<'a> { Review Comment: Yes I think so. In order to pass down the ConfigOptions without clone'ing the them specifically I think in general we could improve planning efficiency in DataFusion by avoiding `clone` but that is a project for another day ########## datafusion/functions-array/src/rewrite.rs: ########## @@ -0,0 +1,211 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Rewrites for using Array Functions + +use crate::concat::{array_append, array_concat}; +use crate::expr_fn::{array_has_all, array_prepend}; +use datafusion_common::config::ConfigOptions; +use datafusion_common::tree_node::Transformed; +use datafusion_common::utils::list_ndims; +use datafusion_common::{Column, DFSchema}; +use datafusion_expr::expr::ScalarFunction; +use datafusion_expr::expr_rewriter::FunctionRewrite; +use datafusion_expr::{ + BinaryExpr, BuiltinScalarFunction, Expr, GetFieldAccess, GetIndexedField, Operator, +}; + +/// Rewrites expressions into function calls to array functions +pub(crate) struct ArrayFunctionRewriter {} + +impl FunctionRewrite for ArrayFunctionRewriter { + fn name(&self) -> &str { + "FunctionRewrite" + } + + fn rewrite( + &self, + expr: Expr, + schema: &DFSchema, + _config: &ConfigOptions, + ) -> datafusion_common::Result<Transformed<Expr>> { + let transformed = match expr { Review Comment: I suggest we add another match in this same file We could also add another FunctionRewriter but that would result in another (unecessary walk down the tree I think) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
