This is an automated email from the ASF dual-hosted git repository.
jinchengchenghh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 66aec06951 [VL][Iceberg] Support SQL MERGE INTO for parquet(#12964)
66aec06951 is described below
commit 66aec06951daa5638995deed1724a763e74d4736
Author: inf / Hazmi <[email protected]>
AuthorDate: Fri Nov 13 10:02:02 2026 +0000
[VL][Iceberg] Support SQL MERGE INTO for parquet(#12964)
---
.../util/iceberg-upstream/flaky-tests.csv | 5 +-
.../gluten/execution/VeloxIcebergSuite.scala | 49 ++++++
.../gluten/execution/IcebergScanTransformer.scala | 164 +++++++++++++--------
.../spark/source/GlutenIcebergSourceUtil.scala | 39 +++--
.../execution/BasicScanExecTransformer.scala | 41 ++++--
.../execution/BatchScanExecTransformer.scala | 5 +-
.../parquet/GlutenParquetRowIndexSuite.scala | 40 ++++-
.../parquet/GlutenParquetRowIndexSuite.scala | 45 +++++-
.../parquet/GlutenParquetRowIndexSuite.scala | 45 +++++-
.../parquet/GlutenParquetRowIndexSuite.scala | 45 +++++-
10 files changed, 378 insertions(+), 100 deletions(-)
diff --git a/.github/workflows/util/iceberg-upstream/flaky-tests.csv
b/.github/workflows/util/iceberg-upstream/flaky-tests.csv
index 1dceb3bd3f..b34131c8f9 100644
--- a/.github/workflows/util/iceberg-upstream/flaky-tests.csv
+++ b/.github/workflows/util/iceberg-upstream/flaky-tests.csv
@@ -16,6 +16,7 @@
# All invocations of these methods run and are reported, but do not affect the
gate.
# Use exact class and method names; omit method arguments and invocation
indices.
class,method
-# Example
-#
org.apache.iceberg.spark.extensions.TestAlterTablePartitionFields,testDropPartitionAndSourceColumnLong
org.apache.iceberg.spark.extensions.TestViews,createViewWithCTEReferencingTempView
+org.apache.iceberg.spark.extensions.TestCopyOnWriteDelete,testDeleteWithVectorizedReads
+org.apache.iceberg.spark.extensions.TestCopyOnWriteMerge,testMergeWithVectorizedReads
+org.apache.iceberg.spark.extensions.TestCopyOnWriteUpdate,testUpdateWithVectorizedReads
diff --git
a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala
b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala
index 6ff43d256c..eaef2f10c3 100644
---
a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala
+++
b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala
@@ -20,6 +20,7 @@ import org.apache.gluten.config.{GlutenConfig, VeloxConfig}
import org.apache.gluten.extension.columnar.FallbackTags
import org.apache.gluten.utils.VeloxFileSystemValidationJniWrapper
+import org.apache.spark.SparkConf
import org.apache.spark.sql.Row
import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
@@ -36,6 +37,54 @@ import org.apache.iceberg.types.{Type, Types}
import scala.collection.JavaConverters._
class VeloxIcebergSuite extends IcebergSuite {
+ override protected def sparkConf: SparkConf =
+ super.sparkConf.set("spark.sql.ansi.enabled", "false")
+
+ test("iceberg metadata row positions are file-relative across splits and
filters") {
+ withTable("iceberg_metadata_split_tb") {
+ withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") {
+ spark.sql("""
+ |CREATE TABLE iceberg_metadata_split_tb (id BIGINT, data
STRING)
+ |USING iceberg
+ |TBLPROPERTIES (
+ | 'write.parquet.row-group-size-bytes' = '1024',
+ | 'write.parquet.compression-codec' = 'uncompressed',
+ | 'write.distribution-mode' = 'none',
+ | 'read.split.target-size' = '1024',
+ | 'read.split.open-file-cost' = '0')
+ |""".stripMargin)
+ spark.range(0, 2000, 1, 1)
+ .selectExpr("id", "sha2(cast(id as string), 256) AS data")
+ .writeTo("iceberg_metadata_split_tb")
+ .append()
+ }
+
+ Seq(
+ "" -> (0L until 2000L),
+ "WHERE id >= 1000" -> (1000L until 2000L),
+ "WHERE _pos >= 1000 AND _pos < 1500" -> (1000L until 1500L),
+ "WHERE id % 3 = 0" -> (0L until 2000L by 3)
+ ).foreach {
+ case (filter, expectedIds) =>
+ runQueryAndCompare(
+ s"SELECT id, _file, _pos FROM iceberg_metadata_split_tb $filter") {
+ df =>
+ checkGlutenPlanCount[IcebergScanTransformer](df, 1)
+ val scans = getExecutedPlan(df).collect { case scan:
IcebergScanTransformer => scan }
+ assert(scans.size == 1)
+ assert(
+ scans.head.inputRDD.partitions.length > 1,
+ "The single data file must be read using multiple splits")
+ val rows = df.collect()
+ assert(rows.map(_.getString(1)).distinct.length == 1)
+ assert(rows.map(_.getLong(0)).sorted.toSeq == expectedIds)
+ assert(rows.forall(row => !row.isNullAt(2)))
+ assert(rows.forall(row => row.getLong(0) == row.getLong(2)))
+ }
+ }
+ }
+ }
+
private def withCredentialTable(catalogName: String)(
f: (String, ResolvingFileIO) => Unit): Unit = {
withTempDir {
diff --git
a/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala
b/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala
index a96706bf4b..cbaaaeedc7 100644
---
a/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala
+++
b/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala
@@ -22,30 +22,34 @@ import org.apache.gluten.exception.GlutenNotSupportException
import
org.apache.gluten.execution.IcebergScanTransformer.{containsMetadataColumn,
containsUuidOrFixedType}
import org.apache.gluten.expression.ConverterUtils
import org.apache.gluten.sql.shims.SparkShimLoader
-import org.apache.gluten.substrait.rel.{LocalFilesNode, SplitInfo}
+import org.apache.gluten.substrait.SubstraitContext
+import org.apache.gluten.substrait.expression.{ExpressionBuilder,
ExpressionNode}
+import org.apache.gluten.substrait.rel.{LocalFilesNode, RelBuilder, SplitInfo}
import org.apache.gluten.substrait.rel.LocalFilesNode.ColumnMappingMode
import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat
import org.apache.spark.Partition
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference,
DynamicPruningExpression, Expression, Literal}
+import org.apache.spark.sql.catalyst.expressions.{Attribute,
AttributeReference, DynamicPruningExpression, Expression, Literal}
import org.apache.spark.sql.catalyst.plans.QueryPlan
import org.apache.spark.sql.connector.catalog.Table
import org.apache.spark.sql.connector.read.Scan
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
import org.apache.spark.sql.execution.metric.SQLMetrics
-import org.apache.spark.sql.types.{ArrayType, DataType, StructType}
+import org.apache.spark.sql.types.{ArrayType, DataType, IntegerType, LongType,
StringType, StructType}
import org.apache.iceberg.{BaseTable, MetadataColumns, SnapshotSummary,
TableProperties}
import org.apache.iceberg.avro.AvroSchemaUtil
-import org.apache.iceberg.spark.source.{GlutenIcebergSourceUtil, SparkTable}
+import org.apache.iceberg.spark.source.GlutenIcebergSourceUtil
import org.apache.iceberg.spark.source.metrics.NumSplits
-import org.apache.iceberg.types.{Type, Types}
+import org.apache.iceberg.types.{Type, Types, TypeUtil}
import org.apache.iceberg.types.Type.TypeID
import org.apache.iceberg.types.Types.{ListType, MapType, NestedField}
import java.util.{HashMap => JHashMap}
+import scala.collection.JavaConverters._
+
case class IcebergScanTransformer(
override val output: Seq[AttributeReference],
@transient override val scan: Scan,
@@ -92,58 +96,57 @@ case class IcebergScanTransformer(
"Iceberg scan uses catalog-vended credentials, which native readers do
not support")
}
+ if (GlutenIcebergSourceUtil.hasOrcScanTask(scan)) {
+ if
(output.exists(_.name.equalsIgnoreCase(MetadataColumns.ROW_POSITION.name()))) {
+ return ValidationResult.failed(
+ "Iceberg row position reads with ORC files are not supported")
+ }
+ val hasTimestampWithZone =
GlutenIcebergSourceUtil.getTable(scan).schemas().values().stream()
+ .flatMap(schema =>
TypeUtil.indexById(schema.asStruct()).values().stream())
+ .anyMatch(
+ field =>
+ field.`type`() match {
+ case timestamp: Types.TimestampType =>
timestamp.shouldAdjustToUTC()
+ case _ => false
+ })
+ if (hasTimestampWithZone) {
+ return ValidationResult.failed("Iceberg ORC timestamp-with-zone reads
are not supported")
+ }
+ }
+
val validationResult = super.doValidateInternal()
if (!validationResult.ok()) {
return validationResult
}
if (!BackendsApiManager.getSettings.supportIcebergEqualityDeleteRead()) {
- val notSupport = table match {
- case t: SparkTable =>
- t.table() match {
- case t: BaseTable =>
- t.operations()
- .current()
- .schema()
- .columns()
- .stream
- .anyMatch(c => containsUuidOrFixedType(c.`type`()) ||
containsMetadataColumn(c))
- case _ => false
- }
+ val notSupport = GlutenIcebergSourceUtil.getTable(scan) match {
+ case t: BaseTable =>
+ t.operations()
+ .current()
+ .schema()
+ .columns()
+ .stream
+ .anyMatch(c => containsUuidOrFixedType(c.`type`()) ||
containsMetadataColumn(c))
case _ => false
}
if (notSupport) {
return ValidationResult.failed("Contains not supported data type or
metadata column")
}
- // Allow input_file_name() and related metadata functions
- val allowedMetadataColumns =
- IcebergScanTransformer.InputFileRelatedMetadataColumnNames
val hasUnsupportedMetadata = scan.readSchema().fieldNames.exists {
f =>
MetadataColumns.isMetadataColumn(f) &&
- !allowedMetadataColumns.contains(ConverterUtils.normalizeColName(f))
+ !IcebergScanTransformer.NativeSupportedMetadataColumnNames
+ .contains(ConverterUtils.normalizeColName(f))
}
if (hasUnsupportedMetadata) {
return ValidationResult.failed("Read unsupported metadata column")
}
- val containsEqualityDelete = table match {
- case t: SparkTable =>
- t.table() match {
- case t: BaseTable =>
- val snapshot = t
- .operations()
- .current()
- .currentSnapshot()
- if (snapshot == null) {
- false
- } else {
- snapshot
- .summary()
- .getOrDefault(SnapshotSummary.TOTAL_EQ_DELETES_PROP, "0")
- .toInt > 0
- }
- case _ => false
- }
+ val containsEqualityDelete = GlutenIcebergSourceUtil.getTable(scan)
match {
+ case t: BaseTable =>
+ val snapshot = t.operations().current().currentSnapshot()
+ snapshot != null &&
+
snapshot.summary().getOrDefault(SnapshotSummary.TOTAL_EQ_DELETES_PROP,
"0").toInt > 0
case _ => false
}
if (containsEqualityDelete) {
@@ -156,12 +159,8 @@ case class IcebergScanTransformer(
}
}
- val baseTable = table match {
- case t: SparkTable =>
- t.table() match {
- case t: BaseTable => t
- case _ => null
- }
+ val baseTable = GlutenIcebergSourceUtil.getTable(scan) match {
+ case t: BaseTable => t
case _ => null
}
if (baseTable == null) {
@@ -217,17 +216,58 @@ case class IcebergScanTransformer(
}
override def getMetadataColumns(): Seq[AttributeReference] = {
- val extraMetadataColumns = inputFileRelatedMetadataColumns.filterNot {
- metadataAttr =>
- metadataColumns.exists {
- existingAttr =>
- ConverterUtils.normalizeColName(existingAttr.name) ==
- ConverterUtils.normalizeColName(metadataAttr.name)
- }
- }
+ val extraMetadataColumns =
+ (inputFileRelatedMetadataColumns ++ output.filter(
+ _.name.equalsIgnoreCase(MetadataColumns.FILE_PATH.name()))).filterNot {
+ metadataAttr =>
+ metadataColumns.exists {
+ existingAttr =>
+ ConverterUtils.normalizeColName(existingAttr.name) ==
+ ConverterUtils.normalizeColName(metadataAttr.name)
+ }
+ }
metadataColumns ++ extraMetadataColumns
}
+ override protected def isRowIndexMetadataColumn(attr: Attribute): Boolean = {
+ attr.name.equalsIgnoreCase(MetadataColumns.ROW_POSITION.name()) ||
+ super.isRowIndexMetadataColumn(attr)
+ }
+
+ // Velox's Iceberg reader does not fill row-index columns, so read `_pos`
from the
+ // `$target_table_row_id` column it synthesizes and project out its
`row_position` field.
+ // TODO: Remove once https://github.com/facebookincubator/velox/pull/20324
is merged.
+ override protected def doTransform(context: SubstraitContext):
TransformContext = {
+ val rowPositionIndex =
+
output.indexWhere(_.name.equalsIgnoreCase(MetadataColumns.ROW_POSITION.name()))
+ if (rowPositionIndex < 0) {
+ return super.doTransform(context)
+ }
+ val targetTableRowId = AttributeReference(
+ "$target_table_row_id",
+ new StructType()
+ .add("file_path", StringType)
+ .add("row_position", LongType)
+ .add("spec_id", IntegerType)
+ .add("partition_data", StringType)
+ )()
+ val readOutput: Seq[Attribute] = output.updated(rowPositionIndex,
targetTableRowId)
+ val operatorId = context.nextOperatorId(this.nodeName)
+ val readRel = makeReadRel(context, readOutput, operatorId)
+ val projections: Seq[ExpressionNode] = output.indices.map {
+ case `rowPositionIndex` =>
ExpressionBuilder.makeSelection(rowPositionIndex, 1)
+ case i => ExpressionBuilder.makeSelection(i)
+ }
+ val projectRel = RelBuilder.makeProjectRel(
+ readRel,
+ projections.asJava,
+ RelBuilder.createExtensionNode(readOutput.asJava),
+ context,
+ operatorId,
+ output.size)
+ TransformContext(output, projectRel)
+ }
+
override lazy val fileFormat: ReadFileFormat = fileFormatAndRootPaths._1
override def getSplitInfosFromPartitions(
@@ -274,6 +314,8 @@ case class IcebergScanTransformer(
runtimeFilters = QueryPlan.normalizePredicates(
runtimeFilters.filterNot(_ ==
DynamicPruningExpression(Literal.TrueLiteral)),
output),
+ keyGroupedPartitioning = keyGroupedPartitioning.map(
+ _.map(QueryPlan.normalizeExpressions(_, output))),
pushDownFilters = pushDownFilters.map(QueryPlan.normalizePredicates(_,
output))
)
}
@@ -285,12 +327,8 @@ case class IcebergScanTransformer(
private[execution] def getKeyGroupPartitioning: Option[Seq[Expression]] =
keyGroupedPartitioning
private def hasRenamedColumn: Boolean = {
- val icebergTable = table match {
- case t: SparkTable =>
- t.table() match {
- case t: BaseTable => t
- case _ => null
- }
+ val icebergTable = GlutenIcebergSourceUtil.getTable(scan) match {
+ case t: BaseTable => t
case _ => null
}
if (icebergTable == null) {
@@ -301,11 +339,13 @@ case class IcebergScanTransformer(
// but may have different id in history schemas
val ops = icebergTable.operations().current()
val currentSchema = ops.schema()
+ val dataReadSchema = StructType(
+ scan.readSchema().fields.filterNot(f =>
MetadataColumns.isMetadataColumn(f.name)))
val oldSchemas = icebergTable.operations().current().schemas()
oldSchemas
.stream()
.filter(s => s.schemaId() != ops.currentSchemaId())
- .anyMatch(s => !typesMatch(s.asStruct(), currentSchema.asStruct(),
scan.readSchema()))
+ .anyMatch(s => !typesMatch(s.asStruct(), currentSchema.asStruct(),
dataReadSchema))
}
private def typesMatch(icebergType: Type, currentType: Type, sparkType:
DataType): Boolean = {
@@ -366,6 +406,10 @@ object IcebergScanTransformer {
private val InputFileRelatedMetadataColumnNames =
Set("input_file_name", "input_file_block_start", "input_file_block_length")
+ private val NativeSupportedMetadataColumnNames =
InputFileRelatedMetadataColumnNames ++
+ Set(MetadataColumns.FILE_PATH.name(), MetadataColumns.ROW_POSITION.name())
+ .map(ConverterUtils.normalizeColName)
+
def apply(batchScan: BatchScanExec): IcebergScanTransformer = {
val transformer = new IcebergScanTransformer(
batchScan.output.map(a =>
a.withName(AvroSchemaUtil.makeCompatibleName(a.name))),
diff --git
a/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala
b/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala
index d460f674df..b4f3aa0166 100644
---
a/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala
+++
b/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala
@@ -46,12 +46,23 @@ object GlutenIcebergSourceUtil {
private val InputFileBlockLengthCol = "input_file_block_length"
def isSupportedScan(sparkScan: Scan): Boolean = sparkScan match {
- case _: SparkBatchQueryScan | _: SparkStagedScan => true
+ case _: SparkBatchQueryScan | _: SparkStagedScan | _: SparkCopyOnWriteScan
=> true
case _ => false
}
def isSparkStagedScan(sparkScan: Scan): Boolean =
sparkScan.isInstanceOf[SparkStagedScan]
+ def hasOrcScanTask(sparkScan: Scan): Boolean =
+ getScanTasks(sparkScan).exists {
+ case task if task.isFileScanTask =>
+ task.asFileScanTask().file().format() == FileFormat.ORC
+ case task: CombinedScanTask =>
+ task.tasks().asScala.exists(_.file().format() == FileFormat.ORC)
+ case _ =>
+ throw new UnsupportedOperationException(
+ "Only support iceberg CombinedScanTask and FileScanTask.")
+ }
+
def hasVendedCredentials(sparkScan: Scan): Boolean =
isSupportedScan(sparkScan) &&
hasVendedCredentials(getTable(sparkScan).io())
@@ -182,6 +193,8 @@ object GlutenIcebergSourceUtil {
case InputFileNameCol => metadataColumns.put(name, filePath)
case InputFileBlockStartCol => metadataColumns.put(name,
start.toString)
case InputFileBlockLengthCol => metadataColumns.put(name,
length.toString)
+ case filePathCol if filePathCol == MetadataColumns.FILE_PATH.name()
=>
+ metadataColumns.put(name, filePath)
case _ =>
}
}
@@ -233,14 +246,14 @@ object GlutenIcebergSourceUtil {
}
}
- asFileScanTask(getScanTasks(sparkScan)).foreach {
+ asFileScanTask(getScanTasks(sparkScan).toList).foreach {
task =>
+ val currentFileFormat = convertFileFormat(task.file().format())
if (fileFormat == ReadFileFormat.UnknownFormat) {
- task.file().format() match {
- case FileFormat.PARQUET => fileFormat =
ReadFileFormat.ParquetReadFormat
- case FileFormat.ORC => fileFormat = ReadFileFormat.OrcReadFormat
- case _ =>
- }
+ fileFormat = currentFileFormat
+ } else if (fileFormat != currentFileFormat) {
+ throw new GlutenNotSupportException(
+ "Iceberg scans with mixed file formats are not supported")
}
if (collectRootPaths) {
recordPath(task.file().path().toString)
@@ -255,7 +268,7 @@ object GlutenIcebergSourceUtil {
}
def getReadPartitionSchema(sparkScan: Scan): StructType = {
- asFileScanTask(getScanTasks(sparkScan)).foreach {
+ asFileScanTask(getScanTasks(sparkScan).toList).foreach {
task =>
val spec = task.spec()
if (spec.isPartitioned) {
@@ -293,18 +306,20 @@ object GlutenIcebergSourceUtil {
throw new UnsupportedOperationException("Failed to get partition schema
from iceberg scan.")
}
- private def getTable(sparkScan: Scan): Table = sparkScan match {
+ def getTable(sparkScan: Scan): Table = sparkScan match {
case scan: SparkBatchQueryScan => scan.table()
+ case scan: SparkCopyOnWriteScan => scan.table()
case scan: SparkStagedScan => scan.table()
case _ =>
throw new GlutenNotSupportException(
s"Unsupported Iceberg scan: ${sparkScan.getClass.getName}.")
}
- private def getScanTasks(sparkScan: Scan): List[ScanTask] = sparkScan match {
- case scan: SparkBatchQueryScan => scan.tasks().asScala.toList
+ private def getScanTasks(sparkScan: Scan): Iterator[ScanTask] = sparkScan
match {
+ case scan: SparkBatchQueryScan => scan.tasks().asScala.iterator
+ case scan: SparkCopyOnWriteScan => scan.tasks().asScala.iterator
case scan: SparkStagedScan =>
- scan.taskGroups().asScala.flatMap(_.tasks().asScala).toList
+ scan.taskGroups().asScala.iterator.flatMap(_.tasks().asScala.iterator)
case _ =>
throw new GlutenNotSupportException(
s"Unsupported Iceberg scan: ${sparkScan.getClass.getName}.")
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala
index 4b0a83cd6c..e552f1953f 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala
@@ -23,7 +23,7 @@ import
org.apache.gluten.extension.columnar.PushDownInputFileExpression
import org.apache.gluten.substrait.`type`.ColumnTypeNode
import org.apache.gluten.substrait.SubstraitContext
import org.apache.gluten.substrait.extensions.ExtensionBuilder
-import org.apache.gluten.substrait.rel.{ReadRelNode, RelBuilder, SplitInfo}
+import org.apache.gluten.substrait.rel.{ReadRelNode, RelBuilder, RelNode,
SplitInfo}
import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat
import org.apache.spark.Partition
@@ -46,24 +46,26 @@ trait BasicScanExecTransformer extends LeafTransformSupport
with BaseDataSource
val (_, scanFiltersNotInPushDownFilters) =
scanFilters.partition(pushDownFilters.get.contains(_))
// For filters that only exists in scan, we need to check if they are
supported.
- val unsupportedFilters = scanFiltersNotInPushDownFilters.filter(
-
!BackendsApiManager.getSparkPlanExecApiInstance.isSupportedScanFilter(_, this))
+ val unsupportedFilters =
scanFiltersNotInPushDownFilters.filterNot(isSupportedScanFilter)
if (unsupportedFilters.nonEmpty) {
throw new UnsupportedOperationException(
"Found unsupported filter in scan " + unsupportedFilters.mkString(",
"))
}
- val supportedPushDownFilters = pushDownFilters.get
-
.filter(BackendsApiManager.getSparkPlanExecApiInstance.isSupportedScanFilter(_,
this))
+ val supportedPushDownFilters =
pushDownFilters.get.filter(isSupportedScanFilter)
FilterHandler.combineFilters(supportedPushDownFilters,
scanFiltersNotInPushDownFilters)
} else {
// todo: When PushDownFilterToScan is not performed, find a way to throw
an
// exception to trigger scan fallback when encountering unsupported
scan filters,
// instead of simply filtering them out.
- scanFilters.filter(
-
BackendsApiManager.getSparkPlanExecApiInstance.isSupportedScanFilter(_, this))
+ scanFilters.filter(isSupportedScanFilter)
}
}
+ private def isSupportedScanFilter(filter: Expression): Boolean = {
+ !filter.references.exists(isRowIndexMetadataColumn) &&
+
BackendsApiManager.getSparkPlanExecApiInstance.isSupportedScanFilter(filter,
this)
+ }
+
/** Returns the filters that already exists in scan. */
def scanFilters: Seq[Expression]
@@ -205,10 +207,14 @@ trait BasicScanExecTransformer extends
LeafTransformSupport with BaseDataSource
doNativeValidation(substraitContext, relNode)
}
+ protected def isRowIndexMetadataColumn(attr: Attribute): Boolean = {
+
BackendsApiManager.getSparkPlanExecApiInstance.isRowIndexMetadataColumn(attr.name)
+ }
+
private def makeColumnTypeNode(attr: Attribute): ColumnTypeNode = {
if (getPartitionSchema.exists(_.name.equals(attr.name))) {
new ColumnTypeNode(NamedStruct.ColumnType.PARTITION_COL)
- } else if
(BackendsApiManager.getSparkPlanExecApiInstance.isRowIndexMetadataColumn(attr.name))
{
+ } else if (isRowIndexMetadataColumn(attr)) {
new ColumnTypeNode(NamedStruct.ColumnType.ROWINDEX_COL)
} else if (attr.isMetadataCol) {
new ColumnTypeNode(NamedStruct.ColumnType.METADATA_COL)
@@ -218,13 +224,20 @@ trait BasicScanExecTransformer extends
LeafTransformSupport with BaseDataSource
}
override protected def doTransform(context: SubstraitContext):
TransformContext = {
- val typeNodes = ConverterUtils.collectAttributeTypeNodes(output)
- val nameList = ConverterUtils.collectAttributeNamesWithoutExprId(output)
- val columnTypeNodes = output.map(makeColumnTypeNode).asJava
+ TransformContext(output, makeReadRel(context, output,
context.nextOperatorId(this.nodeName)))
+ }
+
+ protected def makeReadRel(
+ context: SubstraitContext,
+ attributes: Seq[Attribute],
+ operatorId: java.lang.Long): RelNode = {
+ val typeNodes = ConverterUtils.collectAttributeTypeNodes(attributes)
+ val nameList =
ConverterUtils.collectAttributeNamesWithoutExprId(attributes)
+ val columnTypeNodes = attributes.map(makeColumnTypeNode).asJava
// Will put all filter expressions into an AND expression
val exprNode = FilterHandler
.combineConjuncts(filterExprs().map(ExpressionConverter.replaceAttributeReference))
- .map(ExpressionConverter.replaceWithExpressionTransformer(_, output))
+ .map(ExpressionConverter.replaceWithExpressionTransformer(_, attributes))
.map(_.doTransform(context))
.orNull
@@ -247,12 +260,12 @@ trait BasicScanExecTransformer extends
LeafTransformSupport with BaseDataSource
columnTypeNodes,
extensionNode,
context,
- context.nextOperatorId(this.nodeName))
+ operatorId)
getInputStats.foreach(
inputStats => {
logInfo(s"hit scan inputStats with $inputStats")
readNode.asInstanceOf[ReadRelNode].setInputStats(inputStats)
})
- TransformContext(output, readNode)
+ readNode
}
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala
index de61390d03..645b3fc6a0 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala
@@ -176,8 +176,11 @@ abstract class BatchScanExecTransformerBase(
return ValidationResult.failed(s"Unsupported aggregation push down for
$scan.")
}
+ val hasRowIndexColumn =
+ SparkShimLoader.getSparkShims.findRowIndexColumnIndexInSchema(schema) >=
0 ||
+ output.exists(isRowIndexMetadataColumn)
if (
- SparkShimLoader.getSparkShims.findRowIndexColumnIndexInSchema(schema) >
0 &&
+ hasRowIndexColumn &&
!BackendsApiManager.getSettings.supportNativeRowIndexColumn()
) {
return ValidationResult.failed("Unsupported row index column scan in
native.")
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
index 5cf41b7a9e..9c3f9073e1 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
@@ -19,15 +19,16 @@ package org.apache.spark.sql.execution.datasources.parquet
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{BatchScanExecTransformer,
FileSourceScanExecTransformer}
-import org.apache.spark.sql.GlutenSQLTestsBaseTrait
+import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row}
import org.apache.spark.sql.execution.FileSourceScanExec
import org.apache.spark.sql.execution.datasources.FileFormat
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
import
org.apache.spark.sql.execution.datasources.v2.parquet.ParquetDataSourceV2
import org.apache.spark.sql.functions.{col, max, min}
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{LongType, StringType}
+import org.apache.spark.sql.types.{LongType, StringType, StructType}
+import org.apache.commons.lang3.exception.ExceptionUtils
import org.apache.hadoop.fs.Path
import org.apache.parquet.column.ParquetProperties._
import org.apache.parquet.format.converter.ParquetMetadataConverter
@@ -41,6 +42,41 @@ import scala.collection.JavaConverters._
class GlutenParquetRowIndexSuite extends ParquetRowIndexSuite with
GlutenSQLTestsBaseTrait {
import testImplicits._
+ for (rowIndexType <- Seq(LongType, StringType)) {
+ testGluten(s"datasource v2 row index at schema index zero -
${rowIndexType.typeName}") {
+ withSQLConf(
+ SQLConf.ANSI_ENABLED.key -> "false",
+ SQLConf.USE_V1_SOURCE_LIST.key -> "",
+ GlutenConfig.NATIVE_WRITER_ENABLED.key -> "false") {
+ withTempPath {
+ path =>
+ spark.range(0, 10, 1, 1).write.parquet(path.getAbsolutePath)
+ val rowIndexName =
ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME
+ val readSchema = new StructType()
+ .add(rowIndexName, rowIndexType)
+ .add("id", LongType)
+ val df = spark.read
+ .format(classOf[ParquetDataSourceV2].getCanonicalName)
+ .schema(readSchema)
+ .load(path.getAbsolutePath)
+ .select(rowIndexName, "id")
+
+ if (rowIndexType == LongType) {
+ checkAnswer(df, (0L until 10L).map(id => Row(id, id)))
+ val scans = getExecutedPlan(df).collect {
+ case scan: BatchScanExecTransformer => scan
+ }
+ assert(scans.size == 1)
+ assert(scans.head.schema.fieldNames.head == rowIndexName)
+ } else {
+ val exception = intercept[Exception](df.collect())
+
assert(ExceptionUtils.getRootCauseMessage(exception).contains(rowIndexName))
+ }
+ }
+ }
+ }
+ }
+
private def readRowGroupRowCounts(path: String): Seq[Long] = {
ParquetFooterReader
.readFooter(
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
index 570b6d5e0c..f85a3cf5e6 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
@@ -19,15 +19,16 @@ package org.apache.spark.sql.execution.datasources.parquet
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{BatchScanExecTransformer,
FileSourceScanExecTransformer}
-import org.apache.spark.sql.GlutenSQLTestsBaseTrait
+import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row}
import org.apache.spark.sql.execution.FileSourceScanExec
import org.apache.spark.sql.execution.datasources.FileFormat
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
import
org.apache.spark.sql.execution.datasources.v2.parquet.ParquetDataSourceV2
import org.apache.spark.sql.functions.{col, max, min}
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{LongType, StringType}
+import org.apache.spark.sql.types.{LongType, StringType, StructType}
+import org.apache.commons.lang3.exception.ExceptionUtils
import org.apache.hadoop.fs.Path
import org.apache.parquet.column.ParquetProperties._
import org.apache.parquet.format.converter.ParquetMetadataConverter
@@ -41,6 +42,41 @@ import scala.collection.JavaConverters._
class GlutenParquetRowIndexSuite extends ParquetRowIndexSuite with
GlutenSQLTestsBaseTrait {
import testImplicits._
+ for (rowIndexType <- Seq(LongType, StringType)) {
+ testGluten(s"datasource v2 row index at schema index zero -
${rowIndexType.typeName}") {
+ withSQLConf(
+ SQLConf.ANSI_ENABLED.key -> "false",
+ SQLConf.USE_V1_SOURCE_LIST.key -> "",
+ GlutenConfig.NATIVE_WRITER_ENABLED.key -> "false") {
+ withTempPath {
+ path =>
+ spark.range(0, 10, 1, 1).write.parquet(path.getAbsolutePath)
+ val rowIndexName =
ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME
+ val readSchema = new StructType()
+ .add(rowIndexName, rowIndexType)
+ .add("id", LongType)
+ val df = spark.read
+ .format(classOf[ParquetDataSourceV2].getCanonicalName)
+ .schema(readSchema)
+ .load(path.getAbsolutePath)
+ .select(rowIndexName, "id")
+
+ if (rowIndexType == LongType) {
+ checkAnswer(df, (0L until 10L).map(id => Row(id, id)))
+ val scans = getExecutedPlan(df).collect {
+ case scan: BatchScanExecTransformer => scan
+ }
+ assert(scans.size == 1)
+ assert(scans.head.schema.fieldNames.head == rowIndexName)
+ } else {
+ val exception = intercept[Exception](df.collect())
+
assert(ExceptionUtils.getRootCauseMessage(exception).contains(rowIndexName))
+ }
+ }
+ }
+ }
+ }
+
private def readRowGroupRowCounts(path: String): Seq[Long] = {
ParquetFooterReaderShim
.readFooter(
@@ -353,7 +389,10 @@ class GlutenParquetRowIndexSuite extends
ParquetRowIndexSuite with GlutenSQLTest
.load(path.getAbsolutePath)
val exception = intercept[Exception](dfRead.collect())
-
assert(exception.getMessage.contains(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME))
+ assert(
+ ExceptionUtils
+ .getRootCauseMessage(exception)
+ .contains(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME))
}
}
}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
index 570b6d5e0c..f85a3cf5e6 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
@@ -19,15 +19,16 @@ package org.apache.spark.sql.execution.datasources.parquet
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{BatchScanExecTransformer,
FileSourceScanExecTransformer}
-import org.apache.spark.sql.GlutenSQLTestsBaseTrait
+import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row}
import org.apache.spark.sql.execution.FileSourceScanExec
import org.apache.spark.sql.execution.datasources.FileFormat
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
import
org.apache.spark.sql.execution.datasources.v2.parquet.ParquetDataSourceV2
import org.apache.spark.sql.functions.{col, max, min}
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{LongType, StringType}
+import org.apache.spark.sql.types.{LongType, StringType, StructType}
+import org.apache.commons.lang3.exception.ExceptionUtils
import org.apache.hadoop.fs.Path
import org.apache.parquet.column.ParquetProperties._
import org.apache.parquet.format.converter.ParquetMetadataConverter
@@ -41,6 +42,41 @@ import scala.collection.JavaConverters._
class GlutenParquetRowIndexSuite extends ParquetRowIndexSuite with
GlutenSQLTestsBaseTrait {
import testImplicits._
+ for (rowIndexType <- Seq(LongType, StringType)) {
+ testGluten(s"datasource v2 row index at schema index zero -
${rowIndexType.typeName}") {
+ withSQLConf(
+ SQLConf.ANSI_ENABLED.key -> "false",
+ SQLConf.USE_V1_SOURCE_LIST.key -> "",
+ GlutenConfig.NATIVE_WRITER_ENABLED.key -> "false") {
+ withTempPath {
+ path =>
+ spark.range(0, 10, 1, 1).write.parquet(path.getAbsolutePath)
+ val rowIndexName =
ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME
+ val readSchema = new StructType()
+ .add(rowIndexName, rowIndexType)
+ .add("id", LongType)
+ val df = spark.read
+ .format(classOf[ParquetDataSourceV2].getCanonicalName)
+ .schema(readSchema)
+ .load(path.getAbsolutePath)
+ .select(rowIndexName, "id")
+
+ if (rowIndexType == LongType) {
+ checkAnswer(df, (0L until 10L).map(id => Row(id, id)))
+ val scans = getExecutedPlan(df).collect {
+ case scan: BatchScanExecTransformer => scan
+ }
+ assert(scans.size == 1)
+ assert(scans.head.schema.fieldNames.head == rowIndexName)
+ } else {
+ val exception = intercept[Exception](df.collect())
+
assert(ExceptionUtils.getRootCauseMessage(exception).contains(rowIndexName))
+ }
+ }
+ }
+ }
+ }
+
private def readRowGroupRowCounts(path: String): Seq[Long] = {
ParquetFooterReaderShim
.readFooter(
@@ -353,7 +389,10 @@ class GlutenParquetRowIndexSuite extends
ParquetRowIndexSuite with GlutenSQLTest
.load(path.getAbsolutePath)
val exception = intercept[Exception](dfRead.collect())
-
assert(exception.getMessage.contains(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME))
+ assert(
+ ExceptionUtils
+ .getRootCauseMessage(exception)
+ .contains(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME))
}
}
}
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
index 570b6d5e0c..f85a3cf5e6 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
@@ -19,15 +19,16 @@ package org.apache.spark.sql.execution.datasources.parquet
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{BatchScanExecTransformer,
FileSourceScanExecTransformer}
-import org.apache.spark.sql.GlutenSQLTestsBaseTrait
+import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row}
import org.apache.spark.sql.execution.FileSourceScanExec
import org.apache.spark.sql.execution.datasources.FileFormat
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
import
org.apache.spark.sql.execution.datasources.v2.parquet.ParquetDataSourceV2
import org.apache.spark.sql.functions.{col, max, min}
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{LongType, StringType}
+import org.apache.spark.sql.types.{LongType, StringType, StructType}
+import org.apache.commons.lang3.exception.ExceptionUtils
import org.apache.hadoop.fs.Path
import org.apache.parquet.column.ParquetProperties._
import org.apache.parquet.format.converter.ParquetMetadataConverter
@@ -41,6 +42,41 @@ import scala.collection.JavaConverters._
class GlutenParquetRowIndexSuite extends ParquetRowIndexSuite with
GlutenSQLTestsBaseTrait {
import testImplicits._
+ for (rowIndexType <- Seq(LongType, StringType)) {
+ testGluten(s"datasource v2 row index at schema index zero -
${rowIndexType.typeName}") {
+ withSQLConf(
+ SQLConf.ANSI_ENABLED.key -> "false",
+ SQLConf.USE_V1_SOURCE_LIST.key -> "",
+ GlutenConfig.NATIVE_WRITER_ENABLED.key -> "false") {
+ withTempPath {
+ path =>
+ spark.range(0, 10, 1, 1).write.parquet(path.getAbsolutePath)
+ val rowIndexName =
ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME
+ val readSchema = new StructType()
+ .add(rowIndexName, rowIndexType)
+ .add("id", LongType)
+ val df = spark.read
+ .format(classOf[ParquetDataSourceV2].getCanonicalName)
+ .schema(readSchema)
+ .load(path.getAbsolutePath)
+ .select(rowIndexName, "id")
+
+ if (rowIndexType == LongType) {
+ checkAnswer(df, (0L until 10L).map(id => Row(id, id)))
+ val scans = getExecutedPlan(df).collect {
+ case scan: BatchScanExecTransformer => scan
+ }
+ assert(scans.size == 1)
+ assert(scans.head.schema.fieldNames.head == rowIndexName)
+ } else {
+ val exception = intercept[Exception](df.collect())
+
assert(ExceptionUtils.getRootCauseMessage(exception).contains(rowIndexName))
+ }
+ }
+ }
+ }
+ }
+
private def readRowGroupRowCounts(path: String): Seq[Long] = {
ParquetFooterReaderShim
.readFooter(
@@ -353,7 +389,10 @@ class GlutenParquetRowIndexSuite extends
ParquetRowIndexSuite with GlutenSQLTest
.load(path.getAbsolutePath)
val exception = intercept[Exception](dfRead.collect())
-
assert(exception.getMessage.contains(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME))
+ assert(
+ ExceptionUtils
+ .getRootCauseMessage(exception)
+ .contains(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME))
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]