This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new ff84bcd8f [CORE] Remove static modifier on
TreeMemoryConsumers.Factory.map (#5849)
ff84bcd8f is described below
commit ff84bcd8f2e4bbbe234c6c36985d2705b267a8d1
Author: Hongze Zhang <[email protected]>
AuthorDate: Fri May 24 09:19:55 2024 +0800
[CORE] Remove static modifier on TreeMemoryConsumers.Factory.map (#5849)
---
dev/ci-velox-buildstatic.sh | 2 +-
.../memtarget/spark/TreeMemoryConsumers.java | 10 +-
.../memtarget/spark/TreeMemoryConsumerTest.java | 122 +++++++++++++++++++++
3 files changed, 127 insertions(+), 7 deletions(-)
diff --git a/dev/ci-velox-buildstatic.sh b/dev/ci-velox-buildstatic.sh
index 208490d1c..075440816 100755
--- a/dev/ci-velox-buildstatic.sh
+++ b/dev/ci-velox-buildstatic.sh
@@ -2,7 +2,7 @@ yum install sudo patch java-1.8.0-openjdk-devel -y
cd $GITHUB_WORKSPACE/ep/build-velox/src
./get_velox.sh
source /opt/rh/devtoolset-9/enable
-source /opt/gluten/dev/vcpkg/env.sh
+source $GITHUB_WORKSPACE/dev/vcpkg/env.sh
cd $GITHUB_WORKSPACE/
sed -i '/^headers/d' ep/build-velox/build/velox_ep/CMakeLists.txt
export NUM_THREADS=4
diff --git
a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java
index 46257d80e..1da23d15e 100644
---
a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java
+++
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java
@@ -30,7 +30,6 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public final class TreeMemoryConsumers {
-
private static final Map<Long, Factory> FACTORIES = new
ConcurrentHashMap<>();
private TreeMemoryConsumers() {}
@@ -61,8 +60,7 @@ public final class TreeMemoryConsumers {
}
public static class Factory {
-
- private static final ReferenceMap MAP = new
ReferenceMap(ReferenceMap.WEAK, ReferenceMap.WEAK);
+ private final ReferenceMap map = new ReferenceMap(ReferenceMap.WEAK,
ReferenceMap.WEAK);
private final long perTaskCapacity;
private Factory(long perTaskCapacity) {
@@ -71,9 +69,9 @@ public final class TreeMemoryConsumers {
@SuppressWarnings("unchecked")
private TreeMemoryTarget getSharedAccount(TaskMemoryManager tmm) {
- synchronized (MAP) {
+ synchronized (map) {
return (TreeMemoryTarget)
- MAP.computeIfAbsent(
+ map.computeIfAbsent(
tmm,
m -> {
TreeMemoryTarget tmc = new
TreeMemoryConsumer((TaskMemoryManager) m);
@@ -88,7 +86,7 @@ public final class TreeMemoryConsumers {
String name,
List<Spiller> spillers,
Map<String, MemoryUsageStatsBuilder> virtualChildren) {
- TreeMemoryTarget account = getSharedAccount(tmm);
+ final TreeMemoryTarget account = getSharedAccount(tmm);
return account.newChild(
name, TreeMemoryConsumer.CAPACITY_UNLIMITED, spillers,
virtualChildren);
}
diff --git
a/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java
b/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java
new file mode 100644
index 000000000..e26765d33
--- /dev/null
+++
b/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.memory.memtarget.spark;
+
+import org.apache.gluten.GlutenConfig;
+import org.apache.gluten.memory.memtarget.TreeMemoryTarget;
+
+import org.apache.spark.TaskContext;
+import org.apache.spark.sql.internal.SQLConf;
+import org.apache.spark.util.TaskResources$;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Collections;
+
+import scala.Function0;
+
+public class TreeMemoryConsumerTest {
+ @Test
+ public void testIsolated() {
+ final SQLConf conf = new SQLConf();
+ conf.setConfString(
+ GlutenConfig.COLUMNAR_CONSERVATIVE_TASK_OFFHEAP_SIZE_IN_BYTES().key(),
"100");
+ test(
+ conf,
+ () -> {
+ final TreeMemoryConsumers.Factory factory =
TreeMemoryConsumers.isolated();
+ final TreeMemoryTarget consumer =
+ factory.newConsumer(
+ TaskContext.get().taskMemoryManager(),
+ "FOO",
+ Collections.emptyList(),
+ Collections.emptyMap());
+ Assert.assertEquals(20, consumer.borrow(20));
+ Assert.assertEquals(70, consumer.borrow(70));
+ Assert.assertEquals(10, consumer.borrow(20));
+ Assert.assertEquals(0, consumer.borrow(20));
+ });
+ }
+
+ @Test
+ public void testShared() {
+ final SQLConf conf = new SQLConf();
+ conf.setConfString(
+ GlutenConfig.COLUMNAR_CONSERVATIVE_TASK_OFFHEAP_SIZE_IN_BYTES().key(),
"100");
+ test(
+ conf,
+ () -> {
+ final TreeMemoryConsumers.Factory factory =
TreeMemoryConsumers.shared();
+ final TreeMemoryTarget consumer =
+ factory.newConsumer(
+ TaskContext.get().taskMemoryManager(),
+ "FOO",
+ Collections.emptyList(),
+ Collections.emptyMap());
+ Assert.assertEquals(20, consumer.borrow(20));
+ Assert.assertEquals(70, consumer.borrow(70));
+ Assert.assertEquals(20, consumer.borrow(20));
+ Assert.assertEquals(20, consumer.borrow(20));
+ });
+ }
+
+ @Test
+ public void testIsolatedAndShared() {
+ final SQLConf conf = new SQLConf();
+ conf.setConfString(
+ GlutenConfig.COLUMNAR_CONSERVATIVE_TASK_OFFHEAP_SIZE_IN_BYTES().key(),
"100");
+ test(
+ conf,
+ () -> {
+ final TreeMemoryTarget shared =
+ TreeMemoryConsumers.shared()
+ .newConsumer(
+ TaskContext.get().taskMemoryManager(),
+ "FOO",
+ Collections.emptyList(),
+ Collections.emptyMap());
+ Assert.assertEquals(110, shared.borrow(110));
+ final TreeMemoryTarget isolated =
+ TreeMemoryConsumers.isolated()
+ .newConsumer(
+ TaskContext.get().taskMemoryManager(),
+ "FOO",
+ Collections.emptyList(),
+ Collections.emptyMap());
+ Assert.assertEquals(100, isolated.borrow(110));
+ });
+ }
+
+ private void test(SQLConf conf, Runnable r) {
+ TaskResources$.MODULE$.runUnsafe(
+ new Function0<Object>() {
+ @Override
+ public Object apply() {
+ SQLConf.withExistingConf(
+ conf,
+ new Function0<Object>() {
+ @Override
+ public Object apply() {
+ r.run();
+ return null;
+ }
+ });
+ return null;
+ }
+ });
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
