This is an automated email from the ASF dual-hosted git repository.
mahongbin pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 98015999e [GLUTEN-5061][CH] Fix assert error when writing mergetree
data with select * from table limit n (#5068)
98015999e is described below
commit 98015999ee093a0db6bbf151c354e13ba6637bda
Author: Zhichao Zhang <[email protected]>
AuthorDate: Thu Mar 21 16:31:10 2024 +0800
[GLUTEN-5061][CH] Fix assert error when writing mergetree data with select
* from table limit n (#5068)
The below writing mergetree data sql will throw the assert error:
```
insert into table lineitem_mergetree
select * from lineitem **limit 10**
```
RC:
with limit n, there is a wrong wrapper logical with the FakeRowAdaptor for
the query plan.
Close #5061.
---
.../delta/ClickhouseOptimisticTransaction.scala | 2 +-
.../GlutenClickHouseMergeTreeWriteSuite.scala | 63 ++++++++++++++++++++++
2 files changed, 64 insertions(+), 1 deletion(-)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala
b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala
index e4786168e..9111bea7f 100644
---
a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala
@@ -94,7 +94,7 @@ class ClickhouseOptimisticTransaction(
aqe.isSubquery,
supportsColumnar = true
))
- case other => queryPlan.withNewChildren(Array(FakeRowAdaptor(other)))
+ case other => FakeRowAdaptor(other)
}
val statsTrackers: ListBuffer[WriteJobStatsTracker] = ListBuffer()
diff --git
a/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseMergeTreeWriteSuite.scala
b/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseMergeTreeWriteSuite.scala
index f8aa2cfa4..ca192cb89 100644
---
a/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseMergeTreeWriteSuite.scala
+++
b/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseMergeTreeWriteSuite.scala
@@ -1379,5 +1379,68 @@ class GlutenClickHouseMergeTreeWriteSuite
assert(plans(0).metrics("totalMarksPk").value === 74)
}
}
+
+ test(
+ "GLUTEN-5061: Fix assert error when writing mergetree data with select *
from table limit n") {
+ spark.sql(s"""
+ |DROP TABLE IF EXISTS lineitem_mergetree_5061;
+ |""".stripMargin)
+
+ spark.sql(s"""
+ |CREATE TABLE IF NOT EXISTS lineitem_mergetree_5061
+ |(
+ | l_orderkey bigint,
+ | l_partkey bigint,
+ | l_suppkey bigint,
+ | l_linenumber bigint,
+ | l_quantity double,
+ | l_extendedprice double,
+ | l_discount double,
+ | l_tax double,
+ | l_returnflag string,
+ | l_linestatus string,
+ | l_shipdate date,
+ | l_commitdate date,
+ | l_receiptdate date,
+ | l_shipinstruct string,
+ | l_shipmode string,
+ | l_comment string
+ |)
+ |USING clickhouse
+ |LOCATION '$basePath/lineitem_mergetree_5061'
+ |""".stripMargin)
+
+ spark.sql(s"""
+ | insert into table lineitem_mergetree_5061
+ | select * from lineitem limit 10
+ |""".stripMargin)
+
+ val sqlStr =
+ s"""
+ |SELECT
+ | count(1)
+ |FROM
+ | lineitem_mergetree_5061
+ |""".stripMargin
+ runSql(sqlStr)(
+ df => {
+ val result = df.collect()
+ assert(result.size == 1)
+ assert(result(0).getLong(0) == 10)
+
+ val scanExec = collect(df.queryExecution.executedPlan) {
+ case f: FileSourceScanExecTransformer => f
+ }
+ assert(scanExec.size == 1)
+
+ val mergetreeScan = scanExec(0)
+ assert(mergetreeScan.nodeName.startsWith("Scan mergetree"))
+
+ val fileIndex =
mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex]
+ val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f =>
f.asInstanceOf[AddMergeTreeParts])
+ assert(addFiles.size == 1)
+ assert(addFiles(0).rows == 10)
+ })
+ }
}
// scalastyle:off line.size.limit
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
