(incubator-gluten) branch main updated: [VL][Core] Turn off InputFileNameReplaceRule by default

[hongze]

This is an automated email from the ASF dual-hosted git repository.

hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 35695d123 [VL][Core] Turn off InputFileNameReplaceRule by default
35695d123 is described below

commit 35695d1235017670cc11c2cfae71528bab36c307
Author: 高阳阳 <gyyco...@gmail.com>
AuthorDate: Thu Jun 20 16:44:24 2024 +0800

    [VL][Core] Turn off InputFileNameReplaceRule by default
---
 .../gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala    | 13 ++++++++-----
 .../gluten/execution/ScalarFunctionsValidateSuite.scala     | 10 +++++++---
 .../src/main/scala/org/apache/gluten/GlutenConfig.scala     | 11 +++++++++++
 3 files changed, 26 insertions(+), 8 deletions(-)

diff --git 
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
 
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
index 71930d7e0..1f868c4c2 100644
--- 
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
+++ 
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -805,11 +805,14 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi {
    *
    * @return
    */
-  override def genExtendedColumnarValidationRules(): List[SparkSession => 
Rule[SparkPlan]] = List(
-    BloomFilterMightContainJointRewriteRule.apply,
-    ArrowScanReplaceRule.apply,
-    InputFileNameReplaceRule.apply
-  )
+  override def genExtendedColumnarValidationRules(): List[SparkSession => 
Rule[SparkPlan]] = {
+    val buf: ListBuffer[SparkSession => Rule[SparkPlan]] =
+      ListBuffer(BloomFilterMightContainJointRewriteRule.apply, 
ArrowScanReplaceRule.apply)
+    if (GlutenConfig.getConf.enableInputFileNameReplaceRule) {
+      buf += InputFileNameReplaceRule.apply
+    }
+    buf.result
+  }
 
   /**
    * Generate extended columnar pre-rules.
diff --git 
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
 
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
index d08ba11ee..11eaa3289 100644
--- 
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
+++ 
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
@@ -624,9 +624,13 @@ class ScalarFunctionsValidateSuite extends 
FunctionsValidateTest {
   }
 
   test("Test input_file_name function") {
-    runQueryAndCompare("""SELECT input_file_name(), l_orderkey
-                         | from lineitem limit 100""".stripMargin) {
-      checkGlutenOperatorMatch[ProjectExecTransformer]
+    withSQLConf(
+      "spark.gluten.sql.enableInputFileNameReplaceRule" -> "true"
+    ) {
+      runQueryAndCompare("""SELECT input_file_name(), l_orderkey
+                           | from lineitem limit 100""".stripMargin) {
+        checkGlutenOperatorMatch[ProjectExecTransformer]
+      }
     }
   }
 
diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala 
b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
index 462032488..148e8cdc0 100644
--- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -37,6 +37,7 @@ case class GlutenNumaBindingInfo(
 class GlutenConfig(conf: SQLConf) extends Logging {
   import GlutenConfig._
 
+  def enableInputFileNameReplaceRule: Boolean = 
conf.getConf(INPUT_FILE_NAME_REPLACE_RULE_ENABLED)
   def enableAnsiMode: Boolean = conf.ansiEnabled
 
   def enableGluten: Boolean = conf.getConf(GLUTEN_ENABLED)
@@ -750,6 +751,16 @@ object GlutenConfig {
       .booleanConf
       .createWithDefault(GLUTEN_ENABLE_BY_DEFAULT)
 
+  val INPUT_FILE_NAME_REPLACE_RULE_ENABLED =
+    buildConf("spark.gluten.sql.enableInputFileNameReplaceRule")
+      .internal()
+      .doc(
+        "Experimental: This config apply for velox backend to specify whether 
to enable " +
+          "inputFileNameReplaceRule to support offload input_file_name " +
+          "expression to native.")
+      .booleanConf
+      .createWithDefault(false)
+
   // FIXME the option currently controls both JVM and native validation 
against a Substrait plan.
   val NATIVE_VALIDATION_ENABLED =
     buildConf("spark.gluten.sql.enable.native.validation")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@gluten.apache.org
For additional commands, e-mail: commits-h...@gluten.apache.org