This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 4c640033 chore: Fix most of the scala/java build warnings (#562)
4c640033 is described below
commit 4c640033de292e6181966fa809dbd534740787c0
Author: Andy Grove <[email protected]>
AuthorDate: Thu Jun 13 12:54:35 2024 -0600
chore: Fix most of the scala/java build warnings (#562)
* fix ArrowAssoc compiler warnings
* fix compiler warning: a pure expression does nothing in statement position
* Stop using deprecated withRowGroupSize(Int) and use
withRowGroupSize(Long) instead
* Explicitly specify that we are catching all Throwable types
* stop using deprecated methods on java.sql.Timestamp
* refactor
* revert whitespace changes
* fix regression
* add TODO
* add TODO
* fix two 'eliminated by erasure' warnings
* address feedback
---
.../apache/comet/CometSparkSessionExtensions.scala | 5 +-
.../comet/CometExpressionCoverageSuite.scala | 154 +++++++++++++--------
.../org/apache/comet/CometExpressionSuite.scala | 7 +-
.../scala/org/apache/spark/sql/CometTestBase.scala | 18 +--
4 files changed, 112 insertions(+), 72 deletions(-)
diff --git
a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala
b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala
index d6ec85f5..0136b62a 100644
--- a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala
+++ b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala
@@ -25,7 +25,6 @@ import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
import org.apache.spark.network.util.ByteUnit
import org.apache.spark.sql.{SparkSession, SparkSessionExtensions}
-import org.apache.spark.sql.catalyst.expressions.AttributeReference
import org.apache.spark.sql.catalyst.expressions.aggregate.{Final, Partial}
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.catalyst.trees.TreeNode
@@ -158,7 +157,7 @@ class CometSparkSessionExtensions
// data source V1
case scanExec @ FileSourceScanExec(
HadoopFsRelation(_, partitionSchema, _, _, _:
ParquetFileFormat, _),
- _: Seq[AttributeReference],
+ _: Seq[_],
requiredSchema,
_,
_,
@@ -172,7 +171,7 @@ class CometSparkSessionExtensions
// data source v1 not supported case
case scanExec @ FileSourceScanExec(
HadoopFsRelation(_, partitionSchema, _, _, _:
ParquetFileFormat, _),
- _: Seq[AttributeReference],
+ _: Seq[_],
requiredSchema,
_,
_,
diff --git
a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala
b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala
index 55d45a21..7ea2f502 100644
--- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala
@@ -80,62 +80,102 @@ class CometExpressionCoverageSuite extends CometTestBase
with AdaptiveSparkPlanH
// This can be simplified once comet supports MemoryScan, now Comet triggers
from the FileScan
// If MemoryScan supported we can just run Spark examples as is
val manualTests: Map[String, (String, String)] = Map(
- "!" -> ("select true a", "select ! true from tbl"),
- "%" -> ("select 1 a, 2 b", "select a % b from tbl"),
- "&" -> ("select 1 a, 2 b", "select a & b from tbl"),
- "*" -> ("select 1 a, 2 b", "select a * b from tbl"),
- "+" -> ("select 1 a, 2 b", "select a + b from tbl"),
- "-" -> ("select 1 a, 2 b", "select a - b from tbl"),
- "/" -> ("select 1 a, 2 b", "select a / b from tbl"),
- "<" -> ("select 1 a, 2 b", "select a < b from tbl"),
- "<=" -> ("select 1 a, 2 b", "select a <= b from tbl"),
- "<=>" -> ("select 1 a, 2 b", "select a <=> b from tbl"),
- "=" -> ("select 1 a, 2 b", "select a = b from tbl"),
- "==" -> ("select 1 a, 2 b", "select a == b from tbl"),
- ">" -> ("select 1 a, 2 b", "select a > b from tbl"),
- ">=" -> ("select 1 a, 2 b", "select a >= b from tbl"),
- "^" -> ("select 1 a, 2 b", "select a ^ b from tbl"),
- "|" -> ("select 1 a, 2 b", "select a | b from tbl"),
- "try_multiply" -> ("select 2000000 a, 30000000 b", "select try_multiply(a,
b) from tbl"),
- "try_add" -> ("select 2147483647 a, 1 b", "select try_add(a, b) from tbl"),
- "try_subtract" -> ("select cast(-2147483647 as int) a, cast(1 as int) b",
"select try_subtract(a, b) from tbl"),
- "stack" -> ("select 1 a, 2 b", "select stack(1, a, b) from tbl"),
- "~" -> ("select 1 a", "select ~ a from tbl"),
- "unhex" -> ("select '537061726B2053514C' a", "select unhex(a) from tbl"),
- "when" -> ("select 1 a, 2 b, 3 c, 4 d", "select case a > b then c else d
end from tbl"),
- "case" -> ("select 1 a, 2 b, 3 c, 4 d", "select case a when 1 then c else
d end from tbl"),
- "transform_values" -> ("select array(1, 2, 3) a", "select
transform_values(map_from_arrays(a, a), (k, v) -> v + 1) from tbl"),
- "transform_keys" -> ("select array(1, 2, 3) a", "select
transform_keys(map_from_arrays(a, a), (k, v) -> v + 1) from tbl"),
- "transform" -> ("select array(1, 2, 3) a", "select transform(a, (k, v) ->
v + 1) from tbl"),
- "reduce" -> ("select array(1, 2, 3) a", "select reduce(a, 0, (acc, x) ->
acc + x) from tbl"),
- "struct" -> ("select 1 a, 2 b", "select struct(a, b) from tbl"),
- "space" -> ("select 1 a", "select space(a) from tbl"),
- "sort_array" -> ("select array('b', 'd', null, 'c', 'a') a", "select
sort_array(a) from tbl"),
- "or" -> ("select true a, false b", "select a or b from tbl"),
- "overlay" -> ("select 'Spark SQL' a", "select overlay(a PLACING '_' FROM
6) from tbl"),
- "nvl" -> ("select 1 a, cast(null as int) b", "select nvl(b, a) from tbl"),
- "nvl2" -> ("select 1 a, cast(null as int) b, cast(null as int) c", "select
nvl2(c, b, a) from tbl"),
- "coalesce" -> ("select 1 a, cast(null as int) b, cast(null as int) c",
"select coalesce(c, b, a) from tbl"),
- "and" -> ("select true a, false b", "select a and b from tbl"),
- "not" -> ("select true a", "select not a from tbl"),
- "named_struct" -> ("select 1 a", "select named_struct('a', a) from tbl"),
- "mod" -> ("select 1 a, 1 b", "select mod(b, a) from tbl"),
- "div" -> ("select 1 a, 1 b", "select div(b, a) from tbl"),
- "map_zip_with" -> ("select map(1, 'a', 2, 'b') a, map(1, 'x', 2, 'y') b",
"SELECT map_zip_with(a, b, (k, v1, v2) -> concat(v1, v2)) from tbl"),
- "map_filter" -> ("select map(1, 0, 2, 2, 3, -1) a", "SELECT map_filter(a,
(k, v) -> k > v) from tbl"),
- "in" -> ("select 1 a", "SELECT a in ('1', '2', '3') from tbl"),
- "ifnull" -> ("select 1 a, cast(null as int) b", "SELECT ifnull(b, a) from
tbl"),
- "from_json" -> ("select '{\"a\":1, \"b\":0.8}' a", "SELECT from_json(a, 'a
INT, b DOUBLE') from tbl"),
- "from_csv" -> ("select '1, 0.8' a", "SELECT from_csv(a, 'a INT, b DOUBLE')
from tbl"),
- "forall" -> ("select array(1, 2, 3) a", "SELECT forall(a, x -> x % 2 == 0)
from tbl"),
- "filter" -> ("select array(1, 2, 3) a", "SELECT filter(a, x -> x % 2 == 1)
from tbl"),
- "exists" -> ("select array(1, 2, 3) a", "SELECT filter(a, x -> x % 2 == 0)
from tbl"),
- "aggregate" -> ("select array(1, 2, 3) a", "SELECT aggregate(a, 0, (acc,
x) -> acc + x) from tbl"),
- "extract" -> ("select TIMESTAMP '2019-08-12 01:00:00.123456' a", "SELECT
extract(YEAR FROM a) from tbl"),
- "datepart" -> ("select TIMESTAMP '2019-08-12 01:00:00.123456' a", "SELECT
datepart('YEAR', a) from tbl"),
- "date_part" -> ("select TIMESTAMP '2019-08-12 01:00:00.123456' a", "SELECT
date_part('YEAR', a) from tbl"),
- "cast" -> ("select '10' a", "SELECT cast(a as int) from tbl"),
- "aes_encrypt" -> ("select 'Spark' a, '0000111122223333' b", "SELECT
aes_encrypt(a, b) from tbl"))
+ ("!", ("select true a", "select ! true from tbl")),
+ ("%", ("select 1 a, 2 b", "select a % b from tbl")),
+ ("&", ("select 1 a, 2 b", "select a & b from tbl")),
+ ("*", ("select 1 a, 2 b", "select a * b from tbl")),
+ ("+", ("select 1 a, 2 b", "select a + b from tbl")),
+ ("-", ("select 1 a, 2 b", "select a - b from tbl")),
+ ("/", ("select 1 a, 2 b", "select a / b from tbl")),
+ ("<", ("select 1 a, 2 b", "select a < b from tbl")),
+ ("<=", ("select 1 a, 2 b", "select a <= b from tbl")),
+ ("<=>", ("select 1 a, 2 b", "select a <=> b from tbl")),
+ ("=", ("select 1 a, 2 b", "select a = b from tbl")),
+ ("==", ("select 1 a, 2 b", "select a == b from tbl")),
+ (">", ("select 1 a, 2 b", "select a > b from tbl")),
+ (">=", ("select 1 a, 2 b", "select a >= b from tbl")),
+ ("^", ("select 1 a, 2 b", "select a ^ b from tbl")),
+ ("|", ("select 1 a, 2 b", "select a | b from tbl")),
+ ("try_multiply", ("select 2000000 a, 30000000 b", "select try_multiply(a,
b) from tbl")),
+ ("try_add", ("select 2147483647 a, 1 b", "select try_add(a, b) from tbl")),
+ (
+ "try_subtract",
+ (
+ "select cast(-2147483647 as int) a, cast(1 as int) b",
+ "select try_subtract(a, b) from tbl")),
+ ("stack", ("select 1 a, 2 b", "select stack(1, a, b) from tbl")),
+ ("~", ("select 1 a", "select ~ a from tbl")),
+ ("unhex", ("select '537061726B2053514C' a", "select unhex(a) from tbl")),
+ ("when", ("select 1 a, 2 b, 3 c, 4 d", "select case a > b then c else d
end from tbl")),
+ ("case", ("select 1 a, 2 b, 3 c, 4 d", "select case a when 1 then c else d
end from tbl")),
+ (
+ "transform_values",
+ (
+ "select array(1, 2, 3) a",
+ "select transform_values(map_from_arrays(a, a), (k, v), v + 1) from
tbl")),
+ (
+ "transform_keys",
+ (
+ "select array(1, 2, 3) a",
+ "select transform_keys(map_from_arrays(a, a), (k, v), v + 1) from
tbl")),
+ ("transform", ("select array(1, 2, 3) a", "select transform(a, (k, v), v +
1) from tbl")),
+ ("reduce", ("select array(1, 2, 3) a", "select reduce(a, 0, (acc, x), acc
+ x) from tbl")),
+ ("struct", ("select 1 a, 2 b", "select struct(a, b) from tbl")),
+ ("space", ("select 1 a", "select space(a) from tbl")),
+ ("sort_array", ("select array('b', 'd', null, 'c', 'a') a", "select
sort_array(a) from tbl")),
+ ("or", ("select true a, false b", "select a or b from tbl")),
+ ("overlay", ("select 'Spark SQL' a", "select overlay(a PLACING '_' FROM 6)
from tbl")),
+ ("nvl", ("select 1 a, cast(null as int) b", "select nvl(b, a) from tbl")),
+ (
+ "nvl2",
+ ("select 1 a, cast(null as int) b, cast(null as int) c", "select nvl2(c,
b, a) from tbl")),
+ (
+ "coalesce",
+ (
+ "select 1 a, cast(null as int) b, cast(null as int) c",
+ "select coalesce(c, b, a) from tbl")),
+ ("and", ("select true a, false b", "select a and b from tbl")),
+ ("not", ("select true a", "select not a from tbl")),
+ ("named_struct", ("select 1 a", "select named_struct('a', a) from tbl")),
+ ("mod", ("select 1 a, 1 b", "select mod(b, a) from tbl")),
+ ("div", ("select 1 a, 1 b", "select div(b, a) from tbl")),
+ (
+ "map_zip_with",
+ (
+ "select map(1, 'a', 2, 'b') a, map(1, 'x', 2, 'y') b",
+ "SELECT map_zip_with(a, b, (k, v1, v2), concat(v1, v2)) from tbl")),
+ (
+ "map_filter",
+ ("select map(1, 0, 2, 2, 3, -1) a", "SELECT map_filter(a, (k, v), k > v)
from tbl")),
+ ("in", ("select 1 a", "SELECT a in ('1', '2', '3') from tbl")),
+ ("ifnull", ("select 1 a, cast(null as int) b", "SELECT ifnull(b, a) from
tbl")),
+ (
+ "from_json",
+ ("select '{\"a\":1, \"b\":0.8}' a", "SELECT from_json(a, 'a INT, b
DOUBLE') from tbl")),
+ ("from_csv", ("select '1, 0.8' a", "SELECT from_csv(a, 'a INT, b DOUBLE')
from tbl")),
+ ("forall", ("select array(1, 2, 3) a", "SELECT forall(a, x, x % 2 == 0)
from tbl")),
+ ("filter", ("select array(1, 2, 3) a", "SELECT filter(a, x, x % 2 == 1)
from tbl")),
+ ("exists", ("select array(1, 2, 3) a", "SELECT filter(a, x, x % 2 == 0)
from tbl")),
+ (
+ "aggregate",
+ ("select array(1, 2, 3) a", "SELECT aggregate(a, 0, (acc, x), acc + x)
from tbl")),
+ (
+ "extract",
+ (
+ "select TIMESTAMP '2019-08-12 01:00:00.123456' a",
+ "SELECT extract(YEAR FROM a) from tbl")),
+ (
+ "datepart",
+ ("select TIMESTAMP '2019-08-12 01:00:00.123456' a", "SELECT
datepart('YEAR', a) from tbl")),
+ (
+ "date_part",
+ (
+ "select TIMESTAMP '2019-08-12 01:00:00.123456' a",
+ "SELECT date_part('YEAR', a) from tbl")),
+ ("cast", ("select '10' a", "SELECT cast(a as int) from tbl")),
+ (
+ "aes_encrypt",
+ ("select 'Spark' a, '0000111122223333' b", "SELECT aes_encrypt(a, b)
from tbl")))
// key - function name
// value - examples
@@ -284,7 +324,7 @@ class CometExpressionCoverageSuite extends CometTestBase
with AdaptiveSparkPlanH
datafusionMessage = dfMessage.getOrElse("")),
group = func.group)
- case e =>
+ case e: Throwable =>
CoverageResult(
q,
CoverageResultStatus.Failed,
diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
index 9a1851a5..2457f70a 100644
--- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
@@ -229,9 +229,10 @@ class CometExpressionSuite extends CometTestBase with
AdaptiveSparkPlanHelper {
case None =>
Row(null, null, null)
case Some(i) =>
- val hour = new java.sql.Timestamp(i).getHours
- val minute = new java.sql.Timestamp(i).getMinutes
- val second = new java.sql.Timestamp(i).getSeconds
+ val timestamp = new java.sql.Timestamp(i).toLocalDateTime
+ val hour = timestamp.getHour
+ val minute = timestamp.getMinute
+ val second = timestamp.getSecond
Row(hour, minute, second)
})
diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala
b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala
index ddeb4fb5..97ae879a 100644
--- a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala
@@ -172,16 +172,14 @@ abstract class CometTestBase
protected def checkCometOperators(plan: SparkPlan, excludedClasses:
Class[_]*): Unit = {
val wrapped = wrapCometRowToColumnar(plan)
wrapped.foreach {
- case _: CometScanExec | _: CometBatchScanExec => true
- case _: CometSinkPlaceHolder | _: CometScanWrapper => false
- case _: CometRowToColumnarExec => false
- case _: CometExec | _: CometShuffleExchangeExec => true
- case _: CometBroadcastExchangeExec => true
- case _: WholeStageCodegenExec | _: ColumnarToRowExec | _: InputAdapter
=> true
+ case _: CometScanExec | _: CometBatchScanExec =>
+ case _: CometSinkPlaceHolder | _: CometScanWrapper =>
+ case _: CometRowToColumnarExec =>
+ case _: CometExec | _: CometShuffleExchangeExec =>
+ case _: CometBroadcastExchangeExec =>
+ case _: WholeStageCodegenExec | _: ColumnarToRowExec | _: InputAdapter =>
case op =>
- if (excludedClasses.exists(c => c.isAssignableFrom(op.getClass))) {
- true
- } else {
+ if (!excludedClasses.exists(c => c.isAssignableFrom(op.getClass))) {
assert(
false,
s"Expected only Comet native operators, but found
${op.nodeName}.\n" +
@@ -404,6 +402,8 @@ abstract class CometTestBase
.builder(path)
.withDictionaryEncoding(dictionaryEnabled)
.withType(schema)
+ // TODO we need to shim this and use withRowGroupSize(Long) with later
parquet-hadoop versions to remove
+ // the deprecated warning here
.withRowGroupSize(rowGroupSize.toInt)
.withPageSize(pageSize)
.withDictionaryPageSize(dictionaryPageSize)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
