This is an automated email from the ASF dual-hosted git repository.
philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new a9102e68d2 [GLUTEN-13180][CORE] Fall back the scan when the
ignoreMissingFiles read option differs from the conf (#13207)
a9102e68d2 is described below
commit a9102e68d2075f7c1c696ef1570d10bc3fe46de3
Author: rdtr <[email protected]>
AuthorDate: Sat Nov 7 17:22:47 2026 -0800
[GLUTEN-13180][CORE] Fall back the scan when the ignoreMissingFiles read
option differs from the conf (#13207)
---
.../apache/gluten/execution/VeloxScanSuite.scala | 54 +++++++++++++++++++++-
.../execution/BasicScanExecTransformer.scala | 15 ++++++
.../execution/BatchScanExecTransformer.scala | 12 +++++
.../execution/FileSourceScanExecTransformer.scala | 2 +
4 files changed, 82 insertions(+), 1 deletion(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxScanSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxScanSuite.scala
index 2a1fd3bf2b..117bea4333 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxScanSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxScanSuite.scala
@@ -22,12 +22,14 @@ import org.apache.gluten.config.{GlutenConfig, VeloxConfig}
import org.apache.gluten.utils.VeloxFileSystemValidationJniWrapper
import org.apache.spark.SparkConf
-import org.apache.spark.sql.Row
+import org.apache.spark.sql.{DataFrame, Row}
import org.apache.spark.sql.catalyst.expressions.GreaterThan
import org.apache.spark.sql.execution.ScalarSubquery
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
+import org.apache.hadoop.fs.{FileSystem, Path}
+
import scala.reflect.ClassTag
class VeloxScanSuite extends VeloxWholeStageTransformerSuite {
@@ -330,4 +332,54 @@ class VeloxScanSuite extends
VeloxWholeStageTransformerSuite {
}
}
}
+
+ test("ignoreMissingFiles read option overrides the session conf") {
+ // Reads two parquet dirs, then deletes the files of the first dir before
running the query.
+ def readWithMissingFile(basePath: String, options: Map[String, String]):
DataFrame = {
+ val path1 = new Path(basePath, "first")
+ val path2 = new Path(basePath, "second")
+ spark.range(1).toDF("a").write.parquet(path1.toString)
+ spark.range(1, 2).toDF("a").write.parquet(path2.toString)
+ val df = spark.read.options(options).parquet(path1.toString,
path2.toString)
+ val fs = FileSystem.get(spark.sessionState.newHadoopConf())
+ fs.listStatus(path1)
+ .filter(f => f.isFile && !f.getPath.getName.startsWith("_"))
+ .foreach(f => fs.delete(f.getPath, false))
+ df
+ }
+
+ def isOffloaded(df: DataFrame): Boolean =
+ df.queryExecution.executedPlan.collect { case s:
BasicScanExecTransformer => s }.nonEmpty
+
+ // (spark.sql.files.ignoreMissingFiles, read options, expected to ignore
the missing file)
+ val cases = Seq(
+ ("true", Map.empty[String, String], true),
+ ("false", Map("ignoreMissingFiles" -> "true"), true),
+ ("false", Map.empty[String, String], false),
+ ("true", Map("ignoreMissingFiles" -> "false"), false)
+ )
+ // "parquet" uses the V1 scan, "" uses the V2 scan.
+ Seq("parquet", "").foreach {
+ v1Sources =>
+ cases.foreach {
+ case (conf, options, ignored) =>
+ withTempDir {
+ dir =>
+ withSQLConf(
+ SQLConf.USE_V1_SOURCE_LIST.key -> v1Sources,
+ SQLConf.IGNORE_MISSING_FILES.key -> conf) {
+ val df = readWithMissingFile(dir.getCanonicalPath, options)
+ if (ignored) {
+ checkAnswer(df, Seq(Row(1)))
+ } else {
+ intercept[Exception](df.collect())
+ }
+ // The native scan only reads the session conf, so it must
fall back when
+ // the read option is set to a different value.
+ assert(isOffloaded(df) == options.isEmpty)
+ }
+ }
+ }
+ }
+ }
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala
index c65fe7d135..4b0a83cd6c 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala
@@ -27,8 +27,10 @@ import org.apache.gluten.substrait.rel.{ReadRelNode,
RelBuilder, SplitInfo}
import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat
import org.apache.spark.Partition
+import org.apache.spark.sql.catalyst.FileSourceOptions
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.execution.adaptive.InputStats
+import org.apache.spark.sql.internal.SQLConf
import com.google.protobuf.StringValue
import io.substrait.proto.NamedStruct
@@ -114,6 +116,9 @@ trait BasicScanExecTransformer extends LeafTransformSupport
with BaseDataSource
/** Returns the file format properties. */
def getProperties: Map[String, String] = Map.empty
+ /** Returns the options given to this read, e.g. `spark.read.option(...)`. */
+ def getReadOptions: Map[String, String] = Map.empty
+
def getInputStats: Option[InputStats] = Option.empty
protected def neededInputFileRelatedMetadataKeys: Seq[String] = {
@@ -170,6 +175,16 @@ trait BasicScanExecTransformer extends
LeafTransformSupport with BaseDataSource
"resolution: native scan cannot produce both simultaneously")
}
+ // The native scan only gets spark.sql.files.ignoreMissingFiles from the
session conf.
+ // Spark also accepts ignoreMissingFiles as a read option, and the option
wins over the conf.
+ // When the two differ, the native scan would use the wrong value, so fall
back.
+ val ignoreMissingFiles = new
FileSourceOptions(getReadOptions).ignoreMissingFiles
+ if (ignoreMissingFiles != SQLConf.get.ignoreMissingFiles) {
+ return ValidationResult.failed(
+ s"The ignoreMissingFiles read option ($ignoreMissingFiles) is
different from " +
+ s"${SQLConf.IGNORE_MISSING_FILES.key}
(${SQLConf.get.ignoreMissingFiles}).")
+ }
+
val validationResult = BackendsApiManager.getSettings
.validateScanExec(
fileFormat,
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala
index c95237c7ed..de61390d03 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala
@@ -32,11 +32,15 @@ import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.connector.catalog.Table
import org.apache.spark.sql.connector.read.Scan
import org.apache.spark.sql.execution.datasources.v2.{BatchScanExecShim,
FileScan}
+import org.apache.spark.sql.execution.datasources.v2.orc.OrcScan
+import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan
import org.apache.spark.sql.execution.metric.SQLMetric
import org.apache.spark.sql.types.StructType
import com.google.common.base.Objects
+import scala.collection.JavaConverters._
+
/** Columnar Based BatchScanExec. */
case class BatchScanExecTransformer(
override val output: Seq[AttributeReference],
@@ -145,6 +149,14 @@ abstract class BatchScanExecTransformerBase(
case _ => new StructType()
}
+ override def getReadOptions: Map[String, String] = {
+ scan match {
+ case parquetScan: ParquetScan =>
parquetScan.options.asCaseSensitiveMap.asScala.toMap
+ case orcScan: OrcScan => orcScan.options.asCaseSensitiveMap.asScala.toMap
+ case _ => Map.empty
+ }
+ }
+
override def getRootPathsInternal: Seq[String] = {
scan match {
case fileScan: FileScan =>
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala
index 4e587d3528..2fb4e5c303 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala
@@ -242,6 +242,8 @@ abstract class FileSourceScanExecTransformerBase(
s"${getClass.getSimpleName} $relation
${tableIdentifier.map(_.unquotedString).getOrElse("")}"
}
+ override def getReadOptions: Map[String, String] = relation.options
+
override def getProperties: Map[String, String] = {
this.fileFormat match {
case ReadFileFormat.TextReadFormat =>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]