This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new c2c82715ac [VL] Count total JVM memory as the on-heap portion for the
off-heap sizing feature (#9321)
c2c82715ac is described below
commit c2c82715ac2748ece58d5f9de28aeba71d01958c
Author: Hongze Zhang <[email protected]>
AuthorDate: Tue Apr 15 15:31:06 2025 +0100
[VL] Count total JVM memory as the on-heap portion for the off-heap sizing
feature (#9321)
---
.github/workflows/velox_backend.yml | 14 +++++++-------
.../scala/org/apache/gluten/config/VeloxConfig.scala | 2 +-
.../gluten/tags/{SkipTestTags.java => SkipTest.java} | 2 +-
.../gluten/execution/DynamicOffHeapSizingSuite.scala | 6 +++---
.../org/apache/gluten/expression/VeloxUdfSuite.scala | 4 ++--
.../org/apache/gluten/fuzzer/RowToColumnarFuzzer.scala | 4 ++--
.../org/apache/gluten/fuzzer/ShuffleWriterFuzzer.scala | 4 ++--
.../memtarget/DynamicOffHeapSizingMemoryTarget.java | 18 +++++++++++-------
8 files changed, 29 insertions(+), 25 deletions(-)
diff --git a/.github/workflows/velox_backend.yml
b/.github/workflows/velox_backend.yml
index 1481b65a99..8e23d2d68e 100644
--- a/.github/workflows/velox_backend.yml
+++ b/.github/workflows/velox_backend.yml
@@ -642,7 +642,7 @@ jobs:
export SPARK_SCALA_VERSION=2.12
$MVN_CMD clean test -Pspark-3.2 -Pspark-ut -Pbackends-velox
-Piceberg \
-Pdelta -Phudi
-DargLine="-Dspark.test.home=/opt/shims/spark32/spark_home/" \
-
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
+
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
@@ -729,7 +729,7 @@ jobs:
java -version
$MVN_CMD clean test -Pspark-3.3 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Phudi -Pspark-ut \
-DargLine="-Dspark.test.home=/opt/shims/spark33/spark_home/" \
-
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
+
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
@@ -823,7 +823,7 @@ jobs:
export SPARK_HOME=/opt/shims/spark34/spark_home/
ls -l $SPARK_HOME
$MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Phudi -Pspark-ut \
-
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
\
+
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest
\
-DargLine="-Dspark.test.home=$SPARK_HOME ${EXTRA_FLAGS}"
- name: Upload test report
if: always()
@@ -918,7 +918,7 @@ jobs:
java -version
$MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Phudi -Pspark-ut \
-DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
${EXTRA_FLAGS}" \
-
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
+
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
@@ -971,7 +971,7 @@ jobs:
java -version
$MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Piceberg \
-Pdelta -Pspark-ut
-DargLine="-Dspark.test.home=/opt/shims/spark35-scala-2.13/spark_home/" \
-
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
+
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
@@ -1057,7 +1057,7 @@ jobs:
java -version
$MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Pspark-ut \
-DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
-Dspark.gluten.ras.enabled=true" \
-
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
+
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest
- name: Upload test report
uses: actions/upload-artifact@v4
with:
@@ -1141,7 +1141,7 @@ jobs:
java -version
$MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Pspark-ut \
-DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
-Dspark.gluten.sql.columnar.forceShuffledHashJoin=false" \
-
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
+
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest
- name: Upload test report
uses: actions/upload-artifact@v4
with:
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
index 30f3954519..b94da4eafd 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
@@ -290,7 +290,7 @@ object VeloxConfig {
buildConf("spark.gluten.sql.columnar.backend.velox.resizeBatches.shuffleInput")
.internal()
.doc(s"If true, combine small columnar batches together before sending
to shuffle. " +
- s"The default minimum output batch size is equal to 0.8 *
${COLUMNAR_MAX_BATCH_SIZE.key}")
+ s"The default minimum output batch size is equal to 0.25 *
${COLUMNAR_MAX_BATCH_SIZE.key}")
.booleanConf
.createWithDefault(true)
diff --git
a/backends-velox/src/test/java/org/apache/gluten/tags/SkipTestTags.java
b/backends-velox/src/test/java/org/apache/gluten/tags/SkipTest.java
similarity index 96%
rename from
backends-velox/src/test/java/org/apache/gluten/tags/SkipTestTags.java
rename to backends-velox/src/test/java/org/apache/gluten/tags/SkipTest.java
index b58a17e7b1..b21e0a9d78 100644
--- a/backends-velox/src/test/java/org/apache/gluten/tags/SkipTestTags.java
+++ b/backends-velox/src/test/java/org/apache/gluten/tags/SkipTest.java
@@ -23,4 +23,4 @@ import java.lang.annotation.*;
@TagAnnotation
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.METHOD, ElementType.TYPE})
-public @interface SkipTestTags {}
+public @interface SkipTest {}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingSuite.scala
index 8ada8d1c81..30442c6645 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingSuite.scala
@@ -17,11 +17,11 @@
package org.apache.gluten.execution
import org.apache.gluten.benchmarks.RandomParquetDataGenerator
-import org.apache.gluten.tags.SkipTestTags
+import org.apache.gluten.tags.SkipTest
import org.apache.spark.SparkConf
-@SkipTestTags
+@SkipTest
class DynamicOffHeapSizingSuite extends VeloxWholeStageTransformerSuite {
override protected val resourcePath: String = "/tpch-data-parquet"
override protected val fileFormat: String = "parquet"
@@ -51,7 +51,7 @@ class DynamicOffHeapSizingSuite extends
VeloxWholeStageTransformerSuite {
getRootCause(e.getCause)
}
- test("Dynamic Off-Heap Sizing") {
+ test("Dynamic off-heap sizing") {
System.gc()
dataGenerator.generateRandomData(spark, Some(outputPath))
spark.read.format("parquet").load(outputPath).createOrReplaceTempView("tbl")
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite.scala
index 564b652678..0b22b51018 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite.scala
@@ -17,7 +17,7 @@
package org.apache.gluten.expression
import org.apache.gluten.execution.ProjectExecTransformer
-import org.apache.gluten.tags.{SkipTestTags, UDFTest}
+import org.apache.gluten.tags.{SkipTest, UDFTest}
import org.apache.spark.SparkConf
import org.apache.spark.sql.{GlutenQueryTest, Row, SparkSession}
@@ -255,7 +255,7 @@ class VeloxUdfSuiteLocal extends VeloxUdfSuite {
// /path/to/gluten/package/target/gluten-package-${project.version}.jar
// -Dvelox.udf.lib.path=\
// /path/to/gluten/cpp/build/velox/udf/examples/libmyudf.so
-@SkipTestTags
+@SkipTest
class VeloxUdfSuiteCluster extends VeloxUdfSuite {
override val master: String = "local-cluster[2,2,1024]"
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/fuzzer/RowToColumnarFuzzer.scala
b/backends-velox/src/test/scala/org/apache/gluten/fuzzer/RowToColumnarFuzzer.scala
index d60e577aca..20efa3ffbc 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/fuzzer/RowToColumnarFuzzer.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/fuzzer/RowToColumnarFuzzer.scala
@@ -18,13 +18,13 @@ package org.apache.gluten.fuzzer
import org.apache.gluten.execution.RowToVeloxColumnarExec
import org.apache.gluten.fuzzer.FuzzerResult.Successful
-import org.apache.gluten.tags.{FuzzerTest, SkipTestTags}
+import org.apache.gluten.tags.{FuzzerTest, SkipTest}
import org.apache.spark.SparkConf
import org.apache.spark.sql.DataFrame
@FuzzerTest
-@SkipTestTags
+@SkipTest
class RowToColumnarFuzzer extends FuzzerBase {
override protected def sparkConf: SparkConf = {
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/fuzzer/ShuffleWriterFuzzer.scala
b/backends-velox/src/test/scala/org/apache/gluten/fuzzer/ShuffleWriterFuzzer.scala
index 44f25b517b..ba7d8b90c9 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/fuzzer/ShuffleWriterFuzzer.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/fuzzer/ShuffleWriterFuzzer.scala
@@ -17,13 +17,13 @@
package org.apache.gluten.fuzzer
import org.apache.gluten.fuzzer.FuzzerResult.Successful
-import org.apache.gluten.tags.{FuzzerTest, SkipTestTags}
+import org.apache.gluten.tags.{FuzzerTest, SkipTest}
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.execution.ColumnarShuffleExchangeExec
@FuzzerTest
-@SkipTestTags
+@SkipTest
class ShuffleWriterFuzzer extends FuzzerBase {
private val REPARTITION_SQL = (numPartitions: Int) =>
s"select /*+ REPARTITION($numPartitions) */ * from tbl"
diff --git
a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java
index 387a758ef8..4056df062e 100644
---
a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java
+++
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java
@@ -43,12 +43,16 @@ public class DynamicOffHeapSizingMemoryTarget implements
MemoryTarget {
return 0;
}
- long totalMemory = Runtime.getRuntime().totalMemory();
- long freeMemory = Runtime.getRuntime().freeMemory();
- long usedOnHeapBytes = (totalMemory - freeMemory);
+ // Only JVM shrinking can reclaim space from the total JVM memory.
+ // See https://github.com/apache/incubator-gluten/issues/9276.
+ long totalHeapMemory = Runtime.getRuntime().totalMemory();
+ long freeHeapMemory = Runtime.getRuntime().freeMemory();
+
long usedOffHeapBytesNow = USED_OFFHEAP_BYTES.get();
- if (size + usedOffHeapBytesNow + usedOnHeapBytes > MAX_MEMORY_IN_BYTES) {
+ // Adds the total JVM memory which is the actual memory the JVM occupied
from the operating
+ // system into the counter.
+ if (size + usedOffHeapBytesNow + totalHeapMemory > MAX_MEMORY_IN_BYTES) {
LOG.warn(
String.format(
"Failing allocation as unified memory is OOM. "
@@ -56,9 +60,9 @@ public class DynamicOffHeapSizingMemoryTarget implements
MemoryTarget {
+ "Free On-heap: %d, Total On-heap: %d, "
+ "Max On-heap: %d, Allocation: %d.",
usedOffHeapBytesNow,
- usedOnHeapBytes,
- freeMemory,
- totalMemory,
+ totalHeapMemory - freeHeapMemory,
+ freeHeapMemory,
+ totalHeapMemory,
MAX_MEMORY_IN_BYTES,
size));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
