zhli1142015 commented on code in PR #9585:
URL: https://github.com/apache/incubator-gluten/pull/9585#discussion_r2111638272
##########
backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingSuite.scala:
##########
@@ -16,45 +16,58 @@
*/
package org.apache.gluten.execution
-import org.apache.gluten.benchmarks.RandomParquetDataGenerator
-import org.apache.gluten.tags.SkipTest
+import org.apache.gluten.memory.memtarget.DynamicOffHeapSizingMemoryTarget
import org.apache.spark.SparkConf
-@SkipTest
class DynamicOffHeapSizingSuite extends VeloxWholeStageTransformerSuite {
override protected val resourcePath: String = "/tpch-data-parquet"
override protected val fileFormat: String = "parquet"
- private val dataGenerator =
RandomParquetDataGenerator(System.currentTimeMillis())
- private val outputPath = getClass.getResource("/").getPath +
"dynamicoffheapsizing_output.parquet"
- private val AGG_SQL =
- """select f_1, count(DISTINCT f_1)
- |from tbl group
- |group by 1""".stripMargin
-
override def beforeAll(): Unit = {
super.beforeAll()
+ createTPCHNotNullTables()
}
+
override protected def sparkConf: SparkConf = {
super.sparkConf
.set("spark.shuffle.manager",
"org.apache.spark.shuffle.sort.ColumnarShuffleManager")
- .set("spark.executor.memory", "6GB")
- .set("spark.gluten.memory.dynamic.offHeap.sizing.memory.fraction", "0.8")
+ .set("spark.executor.memory", "2GB")
+ .set("spark.memory.offHeap.enabled", "false")
+ .set("spark.memory.offHeap.size", "0")
+ .set("spark.gluten.memory.dynamic.offHeap.sizing.memory.fraction",
"0.95")
.set("spark.gluten.memory.dynamic.offHeap.sizing.enabled", "true")
}
- def getRootCause(e: Throwable): Throwable = {
- if (e.getCause == null) {
- return e
- }
- getRootCause(e.getCause)
- }
-
test("Dynamic off-heap sizing") {
- System.gc()
- dataGenerator.generateRandomData(spark, Some(outputPath))
-
spark.read.format("parquet").load(outputPath).createOrReplaceTempView("tbl")
- spark.sql(AGG_SQL)
+ if (DynamicOffHeapSizingMemoryTarget.isJava9OrLater()) {
+ val query =
Review Comment:
I think this check is necessary in case someone runs the unit tests locally
with Java 8.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
