dcoliversun opened a new issue, #7291:
URL: https://github.com/apache/incubator-gluten/issues/7291
### Backend
VL (Velox)
### Bug description
Spark SQL:
```sql
CREATE DATABASE delta_db_0920;
USE delta_db_0920;
CREATE TABLE source_table(id INT, name STRING, age INT) using delta;
CREATE TABLE target_table(id INT, name STRING, age INT) using delta;
insert into source_table values(1, 'a', 10),(2, 'b', 20);
insert into target_table values(1, 'c', 10),(3, 'c', 30);""")
MERGE INTO target_table AS target
USING source_table AS source
ON target.id = source.id
WHEN MATCHED THEN
UPDATE SET
target.name = source.name,
target.age = source.age
WHEN NOT MATCHED THEN
INSERT (id, name, age) VALUES (source.id, source.name, source.age);
```
### Spark version
Spark-3.5.x
### Spark configurations
bin/spark-shell \
--conf spark.plugins=org.apache.gluten.GlutenPlugin \
--conf spark.memory.offHeap.enabled=true \
--conf spark.memory.offHeap.size=20g \
--conf
spark.driver.extraClassPath=/root/gluten/package/target/gluten-velox-bundle-spark3.5_2.12-ubuntu_22.04_x86_64-1.3.0-SNAPSHOT.jar:/root/delta/antlr4-runtime-4.9.3.jar:/root/delta/delta-spark_2.12-3.2.0.jar:/root/delta/delta-storage-3.2.0.jar
\
--conf
spark.executor.extraClassPath=/root/gluten/package/target/gluten-velox-bundle-spark3.5_2.12-ubuntu_22.04_x86_64-1.3.0-SNAPSHOT.jar:/root/delta/antlr4-runtime-4.9.3.jar:/root/delta/delta-spark_2.12-3.2.0.jar:/root/delta/delta-storage-3.2.0.jar
\
--conf
spark.shuffle.manager=org.apache.spark.shuffle.sort.ColumnarShuffleManager \
--conf spark.eventLog.enabled=true \
--conf spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension \
--conf
spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog
\
--conf spark.sql.warehouse.dir=/tmp/warehouse \
--conf spark.gluten.sql.native.writer.enabled=false \
--conf spark.gluten.sql.debug=false \
--conf spark.gluten.sql.validation.printStackOnFailure=false
### System information
Velox System Info v0.0.2
Commit: 0cb77714c4b0bcae5dd5b23312291245e3565180
CMake Version: 3.28.3
System: Linux-5.15.0-101-generic
Arch: x86_64
CPU Name: Model name: Intel(R) Xeon(R) Platinum
C++ Compiler: /usr/bin/c++
C++ Compiler Version: 11.4.0
C Compiler: /usr/bin/cc
C Compiler Version: 11.4.0
CMake Prefix Path:
/usr/local;/usr;/;/usr/local/lib/python3.10/dist-packages/cmake/data;/usr/local;/usr/X11R6;/usr/pkg;/opt
### Relevant logs
```bash
org.apache.spark.sql.delta.DeltaIllegalStateException:
[DELTA_FILE_TO_OVERWRITE_NOT_FOUND] File
(file:/tmp/warehouse/delta_db_0920.db/target_table) to be rewritten not found
among candidate files:
file:/tmp/warehouse/delta_db_0920.db/target_table/part-00000-12f8b9b7-e695-4e67-8d3f-673926c280fe-c000.snappy.parquet
file:/tmp/warehouse/delta_db_0920.db/target_table/part-00001-33d8048f-5cd4-4faa-b7ea-d0a68902e29c-c000.snappy.parquet
at
org.apache.spark.sql.delta.DeltaErrorsBase.notFoundFileToBeRewritten(DeltaErrors.scala:723)
at
org.apache.spark.sql.delta.DeltaErrorsBase.notFoundFileToBeRewritten$(DeltaErrors.scala:722)
at
org.apache.spark.sql.delta.DeltaErrors$.notFoundFileToBeRewritten(DeltaErrors.scala:3382)
at
org.apache.spark.sql.delta.commands.DeltaCommand.$anonfun$getTouchedFile$1(DeltaCommand.scala:170)
at scala.collection.immutable.Map$Map2.getOrElse(Map.scala:236)
at
org.apache.spark.sql.delta.commands.DeltaCommand.getTouchedFile(DeltaCommand.scala:170)
at
org.apache.spark.sql.delta.commands.DeltaCommand.getTouchedFile$(DeltaCommand.scala:163)
at
org.apache.spark.sql.delta.commands.MergeIntoCommand.getTouchedFile(MergeIntoCommand.scala:60)
at
org.apache.spark.sql.delta.commands.merge.ClassicMergeExecutor.$anonfun$findTouchedFiles$12(ClassicMergeExecutor.scala:185)
at scala.collection.immutable.Stream.map(Stream.scala:418)
at
org.apache.spark.sql.delta.commands.merge.ClassicMergeExecutor.$anonfun$findTouchedFiles$1(ClassicMergeExecutor.scala:184)
at
org.apache.spark.sql.delta.commands.MergeIntoCommandBase.executeThunk$1(MergeIntoCommandBase.scala:402)
at
org.apache.spark.sql.delta.commands.MergeIntoCommandBase.$anonfun$recordMergeOperation$7(MergeIntoCommandBase.scala:419)
at
org.apache.spark.sql.delta.util.DeltaProgressReporter.withJobDescription(DeltaProgressReporter.scala:53)
at
org.apache.spark.sql.delta.util.DeltaProgressReporter.withStatusCode(DeltaProgressReporter.scala:32)
at
org.apache.spark.sql.delta.util.DeltaProgressReporter.withStatusCode$(DeltaProgressReporter.scala:27)
at
org.apache.spark.sql.delta.commands.MergeIntoCommand.withStatusCode(MergeIntoCommand.scala:60)
at
org.apache.spark.sql.delta.commands.MergeIntoCommandBase.$anonfun$recordMergeOperation$6(MergeIntoCommandBase.scala:419)
at
org.apache.spark.sql.delta.metering.DeltaLogging.recordFrameProfile(DeltaLogging.scala:168)
at
org.apache.spark.sql.delta.metering.DeltaLogging.recordFrameProfile$(DeltaLogging.scala:166)
at
org.apache.spark.sql.delta.commands.MergeIntoCommand.recordFrameProfile(MergeIntoCommand.scala:60)
at
org.apache.spark.sql.delta.metering.DeltaLogging.$anonfun$recordDeltaOperationInternal$1(DeltaLogging.scala:136)
at
com.databricks.spark.util.DatabricksLogging.recordOperation(DatabricksLogging.scala:128)
at
com.databricks.spark.util.DatabricksLogging.recordOperation$(DatabricksLogging.scala:117)
at
org.apache.spark.sql.delta.commands.MergeIntoCommand.recordOperation(MergeIntoCommand.scala:60)
at
org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperationInternal(DeltaLogging.scala:135)
at
org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation(DeltaLogging.scala:125)
at
org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation$(DeltaLogging.scala:115)
at
org.apache.spark.sql.delta.commands.MergeIntoCommand.recordDeltaOperation(MergeIntoCommand.scala:60)
at
org.apache.spark.sql.delta.commands.MergeIntoCommandBase.recordMergeOperation(MergeIntoCommandBase.scala:416)
at
org.apache.spark.sql.delta.commands.MergeIntoCommandBase.recordMergeOperation$(MergeIntoCommandBase.scala:380)
at
org.apache.spark.sql.delta.commands.MergeIntoCommand.recordMergeOperation(MergeIntoCommand.scala:60)
at
org.apache.spark.sql.delta.commands.merge.ClassicMergeExecutor.findTouchedFiles(ClassicMergeExecutor.scala:76)
at
org.apache.spark.sql.delta.commands.merge.ClassicMergeExecutor.findTouchedFiles$(ClassicMergeExecutor.scala:70)
at
org.apache.spark.sql.delta.commands.MergeIntoCommand.findTouchedFiles(MergeIntoCommand.scala:60)
at
org.apache.spark.sql.delta.commands.MergeIntoCommand.$anonfun$runMerge$2(MergeIntoCommand.scala:126)
at
org.apache.spark.sql.delta.commands.MergeIntoCommand.$anonfun$runMerge$2$adapted(MergeIntoCommand.scala:84)
at
org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:227)
at
org.apache.spark.sql.delta.commands.MergeIntoCommand.$anonfun$runMerge$1(MergeIntoCommand.scala:84)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at
org.apache.spark.sql.delta.metering.DeltaLogging.recordFrameProfile(DeltaLogging.scala:168)
at
org.apache.spark.sql.delta.metering.DeltaLogging.recordFrameProfile$(DeltaLogging.scala:166)
at
org.apache.spark.sql.delta.commands.MergeIntoCommand.recordFrameProfile(MergeIntoCommand.scala:60)
at
org.apache.spark.sql.delta.metering.DeltaLogging.$anonfun$recordDeltaOperationInternal$1(DeltaLogging.scala:136)
at
com.databricks.spark.util.DatabricksLogging.recordOperation(DatabricksLogging.scala:128)
at
com.databricks.spark.util.DatabricksLogging.recordOperation$(DatabricksLogging.scala:117)
at
org.apache.spark.sql.delta.commands.MergeIntoCommand.recordOperation(MergeIntoCommand.scala:60)
at
org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperationInternal(DeltaLogging.scala:135)
at
org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation(DeltaLogging.scala:125)
at
org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation$(DeltaLogging.scala:115)
at
org.apache.spark.sql.delta.commands.MergeIntoCommand.recordDeltaOperation(MergeIntoCommand.scala:60)
at
org.apache.spark.sql.delta.commands.MergeIntoCommand.runMerge(MergeIntoCommand.scala:82)
at
org.apache.spark.sql.delta.commands.MergeIntoCommandBase.run(MergeIntoCommandBase.scala:169)
at
org.apache.spark.sql.delta.commands.MergeIntoCommandBase.run$(MergeIntoCommandBase.scala:152)
at
org.apache.spark.sql.delta.commands.MergeIntoCommand.run(MergeIntoCommand.scala:60)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:75)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:73)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:84)
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:107)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$6(SQLExecution.scala:125)
at
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:201)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:108)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:900)
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:66)
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:107)
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:98)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:461)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:76)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:461)
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:32)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:267)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:263)
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:32)
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:32)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:437)
at
org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:98)
at
org.apache.spark.sql.execution.QueryExecution.commandExecuted$lzycompute(QueryExecution.scala:85)
at
org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:83)
at org.apache.spark.sql.Dataset.<init>(Dataset.scala:220)
at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:100)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:900)
at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:97)
at org.apache.spark.sql.SparkSession.$anonfun$sql$4(SparkSession.scala:691)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:900)
at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:682)
at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:713)
at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:744)
... 56 elided
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
