andygrove commented on code in PR #1339:
URL: https://github.com/apache/datafusion-comet/pull/1339#discussion_r1929621860
##########
spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala:
##########
@@ -47,68 +84,20 @@ class CometArrayExpressionSuite extends CometTestBase with
AdaptiveSparkPlanHelp
}
}
- test("array_remove - test all types (native Parquet reader)") {
+ test("array_remove - test all types") {
withTempDir { dir =>
- val path = new Path(dir.toURI.toString, "test.parquet")
- val filename = path.toString
- val random = new Random(42)
- withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
- ParquetGenerator.makeParquetFile(
- random,
- spark,
- filename,
- 100,
- DataGenOptions(
- allowNull = true,
- generateNegativeZero = true,
- generateArray = false,
- generateStruct = false,
- generateMap = false))
- }
- val table = spark.read.parquet(filename)
- table.createOrReplaceTempView("t1")
+ val df = generateTestData(dir, dataGenOptions)
+ df.createOrReplaceTempView("t1")
// test with array of each column
- for (fieldName <- table.schema.fieldNames) {
+ for (field <- df.schema.fields) {
+ val fieldName = field.name
sql(s"SELECT array($fieldName, $fieldName) as a, $fieldName as b FROM
t1")
.createOrReplaceTempView("t2")
val df = sql("SELECT array_remove(a, b) FROM t2")
- checkSparkAnswerAndOperator(df)
- }
- }
- }
-
- test("array_remove - test all types (convert from Parquet)") {
Review Comment:
This turned out not to be very useful, since we don't support arrays when
using COMET_CONVERT_FROM_PARQUET_ENABLED
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
