philo-he commented on code in PR #13529: URL: https://github.com/apache/gluten/pull/13529#discussion_r4546519521
########## backends-velox/src/main/scala/org/apache/gluten/expression/AnsiModeMismatch.scala: ########## @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.expression + +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.expressions.aggregate.{Average, Sum} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types._ + +/** + * Finds expressions whose ANSI mode differs from the session's. + * + * Spark resolves ANSI mode per expression during analysis and stores it in the expression + * (`failOnError`, `ansiEnabled` or `EvalMode`). Normally that is the session's mode, but a view + * runs its body with the mode it was created with (SPARK-33141, SPARK-53348), so one query can mix + * both modes. + * + * Velox's Spark functions have no per-call mode. Functions with a fixed mode or a name per mode + * (`checked_add`, `try_cast`, `get`, `try_make_timestamp`, ...) are picked from the expression and + * are always correct. The functions listed here read `spark.ansi_enabled` from the query config + * instead, so they would run in the session's mode even when the expression captured the other one. + * Such expressions are not offloaded and fall back to Spark. + */ +object AnsiModeMismatch { + + /** True if `expr` captured an ANSI mode that differs from the session's. */ + def apply(expr: Expression): Boolean = { + capturedAnsiMode(expr).exists(_ != SQLConf.get.ansiEnabled) + } + + /** + * The ANSI mode captured in `expr`, for the expressions whose Velox function reads the session + * config. None for everything else. + */ + private def capturedAnsiMode(expr: Expression): Option[Boolean] = expr match { + case e: Remainder => Some(e.evalMode == EvalMode.ANSI) + case e: Pmod => Some(e.evalMode == EvalMode.ANSI) + case e: UnaryMinus => Some(e.failOnError) + case e: Abs => Some(e.failOnError) + // Only integral round can overflow, on a negative scale. + case e: Round if isIntegral(e.child.dataType) => Some(e.ansiEnabled) + case e: BRound if isIntegral(e.child.dataType) => Some(e.ansiEnabled) + // Since Spark 4.3 (SPARK-58052), ceil / floor of a double throw on overflow with ANSI on and + // store the mode in `failOnError`. Before 4.3 nothing is captured: Spark saturates in both + // modes, which Velox does only with ANSI mode off. Returning `false` there is not a view + // mismatch, it only makes the expression fall back in an ANSI-on session. + case e: Ceil if e.child.dataType == DoubleType => Some(failOnErrorOrFalse(e)) + case e: Floor if e.child.dataType == DoubleType => Some(failOnErrorOrFalse(e)) + case e: Conv => Some(e.ansiEnabled) + case e: Elt => Some(e.failOnError) + case e: MakeDate => Some(e.failOnError) + // make_timestamp with failOnError disabled maps to try_make_timestamp, which has a fixed mode. + case e: MakeTimestamp if e.failOnError => Some(true) + case e: UnixTimestamp => Some(e.failOnError) + case e: ToUnixTimestamp => Some(e.failOnError) + case e: GetTimestamp => Some(e.failOnError) + case e: NextDay => Some(e.failOnError) + // A map lookup with failOnError disabled maps to get_map_value, which has a fixed mode. + case e: ElementAt if e.left.dataType.isInstanceOf[MapType] && !e.failOnError => None + case e: ElementAt => Some(e.failOnError) + case e: Sum => Some(sumEvalMode(e) == EvalMode.ANSI) + case e: Average => Some(e.evalMode == EvalMode.ANSI) + case _ => None + } + + private def isIntegral(dataType: DataType): Boolean = dataType match { + case ByteType | ShortType | IntegerType | LongType => true + case _ => false + } + + // `Sum` holds `evalMode` directly up to Spark 4.0 and inside `evalContext` since 4.1. + private def sumEvalMode(sum: Sum): EvalMode.Value = { + try { + sum.getClass.getMethod("evalMode").invoke(sum).asInstanceOf[EvalMode.Value] + } catch { + case _: NoSuchMethodException => + val context = sum.getClass.getMethod("evalContext").invoke(sum) + context.getClass.getMethod("evalMode").invoke(context).asInstanceOf[EvalMode.Value] + } + } + + // `Ceil` and `Floor` have `failOnError` only since Spark 4.3. + private def failOnErrorOrFalse(expr: Expression): Boolean = { + try { + expr.getClass.getMethod("failOnError").invoke(expr).asInstanceOf[Boolean] Review Comment: Given that `failOnError` was introduced only since Spark 4.3, checking Spark version to decide failOnError exists or not is cheaper. Then, no need to use try/catch. ``` SparkVersionUtil.gteSpark43 && expr.getClass.getMethod("failOnError").invoke(expr).asInstanceOf[Boolean] ``` ########## backends-velox/src/main/scala/org/apache/gluten/expression/AnsiModeMismatch.scala: ########## @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.expression + +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.expressions.aggregate.{Average, Sum} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types._ + +/** + * Finds expressions whose ANSI mode differs from the session's. + * + * Spark resolves ANSI mode per expression during analysis and stores it in the expression + * (`failOnError`, `ansiEnabled` or `EvalMode`). Normally that is the session's mode, but a view + * runs its body with the mode it was created with (SPARK-33141, SPARK-53348), so one query can mix + * both modes. + * + * Velox's Spark functions have no per-call mode. Functions with a fixed mode or a name per mode + * (`checked_add`, `try_cast`, `get`, `try_make_timestamp`, ...) are picked from the expression and + * are always correct. The functions listed here read `spark.ansi_enabled` from the query config + * instead, so they would run in the session's mode even when the expression captured the other one. + * Such expressions are not offloaded and fall back to Spark. + */ +object AnsiModeMismatch { + + /** True if `expr` captured an ANSI mode that differs from the session's. */ + def apply(expr: Expression): Boolean = { + capturedAnsiMode(expr).exists(_ != SQLConf.get.ansiEnabled) + } + + /** + * The ANSI mode captured in `expr`, for the expressions whose Velox function reads the session + * config. None for everything else. + */ + private def capturedAnsiMode(expr: Expression): Option[Boolean] = expr match { + case e: Remainder => Some(e.evalMode == EvalMode.ANSI) + case e: Pmod => Some(e.evalMode == EvalMode.ANSI) + case e: UnaryMinus => Some(e.failOnError) + case e: Abs => Some(e.failOnError) + // Only integral round can overflow, on a negative scale. + case e: Round if isIntegral(e.child.dataType) => Some(e.ansiEnabled) + case e: BRound if isIntegral(e.child.dataType) => Some(e.ansiEnabled) + // Since Spark 4.3 (SPARK-58052), ceil / floor of a double throw on overflow with ANSI on and + // store the mode in `failOnError`. Before 4.3 nothing is captured: Spark saturates in both + // modes, which Velox does only with ANSI mode off. Returning `false` there is not a view + // mismatch, it only makes the expression fall back in an ANSI-on session. + case e: Ceil if e.child.dataType == DoubleType => Some(failOnErrorOrFalse(e)) + case e: Floor if e.child.dataType == DoubleType => Some(failOnErrorOrFalse(e)) + case e: Conv => Some(e.ansiEnabled) + case e: Elt => Some(e.failOnError) + case e: MakeDate => Some(e.failOnError) + // make_timestamp with failOnError disabled maps to try_make_timestamp, which has a fixed mode. + case e: MakeTimestamp if e.failOnError => Some(true) + case e: UnixTimestamp => Some(e.failOnError) + case e: ToUnixTimestamp => Some(e.failOnError) + case e: GetTimestamp => Some(e.failOnError) + case e: NextDay => Some(e.failOnError) + // A map lookup with failOnError disabled maps to get_map_value, which has a fixed mode. + case e: ElementAt if e.left.dataType.isInstanceOf[MapType] && !e.failOnError => None + case e: ElementAt => Some(e.failOnError) + case e: Sum => Some(sumEvalMode(e) == EvalMode.ANSI) + case e: Average => Some(e.evalMode == EvalMode.ANSI) + case _ => None + } + + private def isIntegral(dataType: DataType): Boolean = dataType match { + case ByteType | ShortType | IntegerType | LongType => true + case _ => false + } + + // `Sum` holds `evalMode` directly up to Spark 4.0 and inside `evalContext` since 4.1. + private def sumEvalMode(sum: Sum): EvalMode.Value = { + try { + sum.getClass.getMethod("evalMode").invoke(sum).asInstanceOf[EvalMode.Value] Review Comment: ditto ########## docs/backends/velox/velox-backend-limitations.md: ########## @@ -18,8 +18,27 @@ Except the unsupported operators, functions, file formats, data sources listed i #### ANSI Gluten currently doesn't support ANSI mode. If ANSI is enabled, Spark plan's execution will always fall back to vanilla Spark. +This whole-plan fallback is controlled by `spark.gluten.sql.ansiFallback.enabled` (default `true`). Setting it to `false` lets Gluten try to run the plan natively with ANSI mode enabled. We now have a issue tracker on ANSI support progress. Please check [issue-10134](https://github.com/apache/gluten/issues/10134). +Spark stores the ANSI mode in each expression when the query is analyzed, so an expression can carry a different mode from the current session: + +- A view runs its body with the ANSI mode it was created with (SPARK-33141, and since Spark 4.0.1 the setting is always stored with the view, SPARK-53348). A view created with ANSI off and read after an upgrade to Spark 4, where ANSI is on by default, mixes both modes in one query. +- A DataFrame is analyzed when it is created. If `spark.sql.ansi.enabled` changes before an action runs it, its expressions keep the mode from creation. This also holds for a temporary view registered from it with `createOrReplaceTempView`, which keeps the analyzed plan. + +Some Velox functions have one name per mode and are always correct, for example `+`, `cast` and `try_*`. Others read the ANSI mode from the session config. An expression of the second group that carries a different mode from the session is not offloaded, and its operator falls back to vanilla Spark. This applies with either value of `spark.gluten.sql.ansiFallback.enabled` and cannot be disabled. With the default whole-plan fallback it matters when the session has ANSI off, for example a view created with ANSI on. The expressions are: Review Comment: Nit: could we wrap this and the other long lines to keep them within the usual line length? ########## backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala: ########## @@ -603,6 +603,11 @@ object VeloxAggregateFunctionsBuilder { case e: Throwable => throw e } + // Same backend check as AggregateFunctionsBuilder, which this builder replaces. + if (!BackendsApiManager.getValidatorApiInstance.doExprValidate(sigName, aggFunc)) { + throw new GlutenNotSupportException(s"Aggregate function not supported for $aggFunc.") Review Comment: When this check fails, the fallback reason is the caller's generic "Not supported to map spark function name ..." (or "Aggregate function not supported for sum(...)"), which reads as if the function weren't supported at all. Could we report a specific reason instead, e.g. "Fall back: expression's ANSI mode (captured when the view or DataFrame was analyzed) differs from the session's spark.sql.ansi.enabled"? That would make these fallbacks easy to recognize in the fallback report. ########## backends-velox/src/test/scala/org/apache/gluten/functions/AnsiModeMismatchSuite.scala: ########## @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.functions + +import org.apache.gluten.config.GlutenConfig +import org.apache.gluten.execution.{HashAggregateExecTransformer, ProjectExecTransformer, WholeStageTransformerSuite} + +import org.apache.spark.SparkConf +import org.apache.spark.sql.{DataFrame, Row} +import org.apache.spark.sql.internal.SQLConf + +import scala.reflect.ClassTag + +/** + * A view keeps the ANSI mode it was created with, so a query can run some expressions with ANSI + * mode on and others with it off. Velox functions that read the mode from the session config cannot + * follow that, so those expressions fall back to Spark; functions with a name per mode stay native. + */ +class AnsiModeMismatchSuite extends WholeStageTransformerSuite { + override protected val resourcePath: String = "/tpch-data-parquet" + override protected val fileFormat: String = "parquet" + + disableFallbackCheck + + override protected def sparkConf: SparkConf = { + super.sparkConf + .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") + .set("spark.sql.shuffle.partitions", "1") + .set("spark.memory.offHeap.size", "2g") + .set("spark.unsafe.exceptionOnMemoryLeak", "true") + .set(GlutenConfig.GLUTEN_ANSI_FALLBACK_ENABLED.key, "false") + } + + // Reads the rows from a Parquet file so the expressions are evaluated in Velox and not folded + // by the optimizer. The values never trigger an error in either mode, so both modes return the + // same rows and the test only has to look at where the query ran. + private def withIntTable(f: => Unit): Unit = { + withTempPath { + path => + sql("SELECT * FROM VALUES (10, 3), (-7, 2) AS t(c, d)") + .write + .parquet(path.getCanonicalPath) + withTempView("int_tab") { + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("int_tab") + f + } + } + } + + // Checks that no operator of type `T` was offloaded. + private def checkFallback[T: ClassTag](df: DataFrame): Unit = { + val tag = implicitly[ClassTag[T]] + assert( + !getExecutedPlan(df).exists(p => tag.runtimeClass.isInstance(p)), + df.queryExecution.executedPlan) + } + + // Creates a temporary view with ANSI mode set to `ansi`. The view captures that setting and + // keeps it when it is read under another setting. + private def createView(name: String, body: String, ansi: Boolean): Unit = { + withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi.toString) { + sql(s"CREATE OR REPLACE TEMP VIEW $name AS $body") + } + } + + test("view created with ANSI off, read with ANSI on") { + withIntTable { + withTempView("legacy_view") { + createView("legacy_view", "SELECT abs(c) AS a, c % d AS m, c + d AS s FROM int_tab", false) + withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { + // abs and % read the session config in Velox: the project falls back. + runQueryAndCompare("SELECT a, m FROM legacy_view", noFallBack = false) { + df => checkFallback[ProjectExecTransformer](df) + } + // + has a function per mode: the project stays native. + runQueryAndCompare("SELECT s FROM legacy_view") { + checkGlutenPlan[ProjectExecTransformer] + } + } + } + } + } + + test("view created with ANSI on, read with ANSI off") { + withIntTable { + withTempView("ansi_view") { + createView("ansi_view", "SELECT abs(c) AS a, c % d AS m, c + d AS s FROM int_tab", true) + withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") { + runQueryAndCompare("SELECT a, m FROM ansi_view", noFallBack = false) { + df => checkFallback[ProjectExecTransformer](df) + } + runQueryAndCompare("SELECT s FROM ansi_view") { + checkGlutenPlan[ProjectExecTransformer] + } + } + } + } + } + + test("view created with ANSI on still throws under the default ANSI fallback") { + // The default ANSI fallback looks only at the session, so with the session's ANSI mode off + // it leaves the plan native, and the view's abs must still throw on overflow. + withSQLConf(GlutenConfig.GLUTEN_ANSI_FALLBACK_ENABLED.key -> "true") { + withTempPath { + path => + sql("SELECT CAST(-2147483648 AS INT) AS c").write.parquet(path.getCanonicalPath) + withTempView("min_tab", "abs_view") { + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("min_tab") + createView("abs_view", "SELECT abs(c) AS a FROM min_tab", true) + withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") { + val e = intercept[Exception](sql("SELECT a FROM abs_view").collect()) + assert(e.getMessage.contains("overflow"), e.getMessage) + } + } + } + } + } + + test("view created and read in the same mode stays native") { + withIntTable { + Seq(true, false).foreach { + ansi => + withTempView("same_mode_view") { + createView("same_mode_view", "SELECT abs(c) AS a, c % d AS m FROM int_tab", ansi) + withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi.toString) { + runQueryAndCompare("SELECT a, m FROM same_mode_view") { + checkGlutenPlan[ProjectExecTransformer] + } + } + } + } + } + } + + test("DataFrame analyzed with ANSI off keeps its mode when run with ANSI on") { + withIntTable { + // A DataFrame is analyzed when it is created, so its expressions keep that ANSI mode. + var df: DataFrame = null + withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") { + df = sql("SELECT abs(c) AS a, c % d AS m FROM int_tab") + } + withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { + checkAnswer(df, Seq(Row(10, 1), Row(7, -1))) + checkFallback[ProjectExecTransformer](df) + } + } + } + + test("aggregate in a view keeps its mode") { + withIntTable { + withTempView("sum_view") { + createView("sum_view", "SELECT sum(c) AS total FROM int_tab", false) + withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { + runQueryAndCompare("SELECT total FROM sum_view", noFallBack = false) { + df => checkFallback[HashAggregateExecTransformer](df) + } + } + withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") { + runQueryAndCompare("SELECT total FROM sum_view") { + checkGlutenPlan[HashAggregateExecTransformer] + } + } + } + } + } + + test("array subscript with ANSI on falls back until Velox can throw") { + withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { + withTempPath { + path => + sql("SELECT array(1, 2, 3) AS arr, 1 AS i").write.parquet(path.getCanonicalPath) + withTempView("arr_tab") { + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("arr_tab") + runQueryAndCompare("SELECT arr[i] FROM arr_tab", noFallBack = false) { + df => checkFallback[ProjectExecTransformer](df) + } + // get() never throws and maps to Velox's get. + runQueryAndCompare("SELECT get(arr, i) FROM arr_tab") { + checkGlutenPlan[ProjectExecTransformer] + } + } + } + } + } + + // Spark throws on ceil / floor overflow of a double only since 4.3 (SPARK-58052). Earlier + // versions saturate in both modes, which Velox matches only with ANSI mode off. + private def withDoubleTable(f: => Unit): Unit = { + withTempPath { + path => + sql("SELECT * FROM VALUES (1.5D, 7L), (-2.5D, -7L), (1e20D, 1L), (-1e20D, -1L) AS t(c, l)") + .write + .parquet(path.getCanonicalPath) + withTempView("double_tab") { + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("double_tab") + f + } + } + } + + // Spark 4.3 throws on ceil(1e20D) with ANSI on. + testWithMaxSparkVersion( + "ceil and floor of a double fall back with ANSI on before Spark 4.3", + "4.2") { + withDoubleTable { + withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { + runQueryAndCompare("SELECT ceil(c), floor(c) FROM double_tab", noFallBack = false) { + df => checkFallback[ProjectExecTransformer](df) + } + // Only doubles can overflow: ceil and floor of a bigint stay native. + runQueryAndCompare("SELECT ceil(l), floor(l) FROM double_tab") { + checkGlutenPlan[ProjectExecTransformer] + } + } + } + } + + test("ceil and floor of a double stay native with ANSI off") { + withDoubleTable { + withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") { + runQueryAndCompare("SELECT ceil(c), floor(c) FROM double_tab") { + checkGlutenPlan[ProjectExecTransformer] + } + } + } + } +} Review Comment: Could you add a nested case to the suite, e.g. `SELECT cast(a + 1 AS string) FROM legacy_view` or `concat('x', cast(abs(c) AS string))`, where the outer expression isn't in the list? ########## backends-velox/src/test/scala/org/apache/gluten/functions/AnsiModeMismatchSuite.scala: ########## @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.functions + +import org.apache.gluten.config.GlutenConfig +import org.apache.gluten.execution.{HashAggregateExecTransformer, ProjectExecTransformer, WholeStageTransformerSuite} + +import org.apache.spark.SparkConf +import org.apache.spark.sql.{DataFrame, Row} +import org.apache.spark.sql.internal.SQLConf + +import scala.reflect.ClassTag + +/** + * A view keeps the ANSI mode it was created with, so a query can run some expressions with ANSI + * mode on and others with it off. Velox functions that read the mode from the session config cannot + * follow that, so those expressions fall back to Spark; functions with a name per mode stay native. + */ +class AnsiModeMismatchSuite extends WholeStageTransformerSuite { + override protected val resourcePath: String = "/tpch-data-parquet" Review Comment: Nit: tpch dataset is not used actually. Can we set an empty string for resourcePath? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
