sunchao commented on code in PR #6170:
URL: https://github.com/apache/datafusion-comet/pull/6170#discussion_r4178686419
##########
spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrayExpressionBenchmark.scala:
##########
@@ -146,7 +149,66 @@ object CometArrayExpressionBenchmark extends
CometBenchmarkBase {
}
}
+ def arrayExtremaCollationBenchmark(values: Int, expectNative: Boolean =
true): Unit = {
+ // Repeated values exercise dictionaries; long prefixes and Unicode
exercise comparison work.
+ val shapes = Seq(
+ ("short ASCII, no nulls", "AbC", 1, 0),
+ ("long ASCII, sparse nulls", "AbC", 32, 8),
+ ("Unicode, dense nulls", "ΣİÄ", 4, 50))
+ for ((shape, prefix, repeats, nullPercent) <- shapes) {
+ withTempPath { dir =>
+ withTempTable("parquetV1Table") {
+ val columns = (0 until 8).map { i =>
+ val text = if (i % 2 == 0) prefix else
prefix.toLowerCase(java.util.Locale.ROOT)
+ s"""CASE WHEN PMOD(value + ${i * 13}, 100) < $nullPercent THEN NULL
+ |ELSE CONCAT(REPEAT('$text', $repeats),
+ | CAST(PMOD(value + ${i / 2}, 64) AS STRING), '${" " * (i %
3)}')
+ |END AS c$i""".stripMargin
+ }
+ prepareTable(dir, spark.sql(s"SELECT ${columns.mkString(", ")} FROM
$tbl"))
+ for (collation <- Seq(
+ "UTF8_BINARY",
+ "UTF8_BINARY_RTRIM",
+ "UTF8_LCASE",
+ "UTF8_LCASE_RTRIM");
+ function <- Seq("array_min", "array_max")) {
+ val input = (0 until 8)
+ .map(i => s"CAST(c$i AS STRING COLLATE $collation)")
+ .mkString("array(", ", ", ")")
+ val query = s"SELECT $function($input) FROM parquetV1Table"
+ // The dispatcher baseline uses the same inputs and verifies its
execution path too.
+ withSQLConf(
+ CometConf.COMET_ENABLED.key -> "true",
+ CometConf.COMET_EXEC_ENABLED.key -> "true") {
+ val plan =
stripAQEPlan(spark.sql(query).queryExecution.executedPlan)
+ val explain = new ExtendedExplainInfo()
+ if (expectNative || collation == "UTF8_BINARY") {
+ require(
+ explain.getNativeExpressions(plan).contains(function) &&
+
!explain.getCodegenDispatchExpressions(plan).contains(function),
+ s"$function did not run natively: $plan")
+ } else {
+ require(
+
explain.getCodegenDispatchExpressions(plan).contains(function),
+ s"$function did not use the dispatcher: $plan")
+ }
+ }
+ runExpressionBenchmark(s"$function $collation - $shape", values,
query)
Review Comment:
Changed the benchmark cardinality argument to values.toLong. Spark 4.1
compilation and six focused tests pass. The strict Spark 3.5 attempt was
blocked by unavailable AWS SDK dependencies locally, so the existing
all-profile CI label remains the verification for that profile.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
