This is an automated email from the ASF dual-hosted git repository.
vgalaxies pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git
The following commit(s) were added to refs/heads/master by this push:
new 0cb6115fd feat(server): support in-heap memory JVM monitor (#2650)
0cb6115fd is described below
commit 0cb6115fdc3970c2c66d9c21c9b24bfeb9bf58eb
Author: MingzhenHan <[email protected]>
AuthorDate: Tue Oct 15 19:53:31 2024 +0800
feat(server): support in-heap memory JVM monitor (#2650)
---
.../org/apache/hugegraph/config/ServerOptions.java | 19 ++++
.../backend/store/BackendEntryIterator.java | 2 +-
.../assembly/static/conf/rest-server.properties | 4 +
.../org/apache/hugegraph/dist/HugeGraphServer.java | 7 ++
.../org/apache/hugegraph/dist/MemoryMonitor.java | 120 +++++++++++++++++++++
5 files changed, 151 insertions(+), 1 deletion(-)
diff --git
a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/config/ServerOptions.java
b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/config/ServerOptions.java
index 084e9a338..5c5aa86e3 100644
---
a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/config/ServerOptions.java
+++
b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/config/ServerOptions.java
@@ -21,6 +21,7 @@ import static
org.apache.hugegraph.config.OptionChecker.allowValues;
import static org.apache.hugegraph.config.OptionChecker.disallowEmpty;
import static org.apache.hugegraph.config.OptionChecker.nonNegativeInt;
import static org.apache.hugegraph.config.OptionChecker.positiveInt;
+import static org.apache.hugegraph.config.OptionChecker.rangeDouble;
import static org.apache.hugegraph.config.OptionChecker.rangeInt;
public class ServerOptions extends OptionHolder {
@@ -321,4 +322,22 @@ public class ServerOptions extends OptionHolder {
nonNegativeInt(),
1000L
);
+
+ public static final ConfigOption<Double> JVM_MEMORY_MONITOR_THRESHOLD =
+ new ConfigOption<>(
+ "memory_monitor.threshold",
+ "Threshold for JVM memory usage monitoring, 1 means
disabling the memory " +
+ "monitoring task.",
+ rangeDouble(0.0, 1.0),
+ 0.85
+ );
+
+ public static final ConfigOption<Integer> JVM_MEMORY_MONITOR_DETECT_PERIOD
=
+ new ConfigOption<>(
+ "memory_monitor.period",
+ "The period in ms of JVM memory usage monitoring, in each
period we will " +
+ "detect the jvm memory usage and take corresponding
actions.",
+ nonNegativeInt(),
+ 2000
+ );
}
diff --git
a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendEntryIterator.java
b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendEntryIterator.java
index f13798dd6..20469f77a 100644
---
a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendEntryIterator.java
+++
b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendEntryIterator.java
@@ -94,7 +94,7 @@ public abstract class BackendEntryIterator implements
CIter<BackendEntry> {
public static final void checkInterrupted() {
if (Thread.interrupted()) {
- throw new BackendException("Interrupted, maybe it is timed out",
+ throw new BackendException("Interrupted, maybe it is timed out or
uses too much memory",
new InterruptedException());
}
}
diff --git
a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/rest-server.properties
b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/rest-server.properties
index f89966e6c..9d6bb2ca8 100644
---
a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/rest-server.properties
+++
b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/rest-server.properties
@@ -53,3 +53,7 @@ server.role=master
# slow query log
log.slow_query_threshold=1000
+
+# jvm(in-heap) memory usage monitor, set 1 to disable it
+memory_monitor.threshold=0.85
+memory_monitor.period=2000
diff --git
a/hugegraph-server/hugegraph-dist/src/main/java/org/apache/hugegraph/dist/HugeGraphServer.java
b/hugegraph-server/hugegraph-dist/src/main/java/org/apache/hugegraph/dist/HugeGraphServer.java
index ae2c73e2c..2652324f4 100644
---
a/hugegraph-server/hugegraph-dist/src/main/java/org/apache/hugegraph/dist/HugeGraphServer.java
+++
b/hugegraph-server/hugegraph-dist/src/main/java/org/apache/hugegraph/dist/HugeGraphServer.java
@@ -37,6 +37,7 @@ public class HugeGraphServer {
private final RestServer restServer;
private final GremlinServer gremlinServer;
+ private final MemoryMonitor memoryMonitor;
public static void register() {
RegisterUtil.registerBackends();
@@ -78,9 +79,15 @@ public class HugeGraphServer {
} finally {
System.setSecurityManager(securityManager);
}
+
+ // Start (In-Heap) Memory Monitor
+ this.memoryMonitor = new MemoryMonitor(restServerConf);
+ this.memoryMonitor.start();
}
public void stop() {
+ this.memoryMonitor.stop();
+
try {
this.gremlinServer.stop().get();
LOG.info("HugeGremlinServer stopped");
diff --git
a/hugegraph-server/hugegraph-dist/src/main/java/org/apache/hugegraph/dist/MemoryMonitor.java
b/hugegraph-server/hugegraph-dist/src/main/java/org/apache/hugegraph/dist/MemoryMonitor.java
new file mode 100644
index 000000000..18b7cc170
--- /dev/null
+++
b/hugegraph-server/hugegraph-dist/src/main/java/org/apache/hugegraph/dist/MemoryMonitor.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hugegraph.dist;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryUsage;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hugegraph.config.HugeConfig;
+import org.apache.hugegraph.config.ServerOptions;
+import org.apache.hugegraph.util.ExecutorUtil;
+import org.apache.hugegraph.util.Log;
+import org.slf4j.Logger;
+
+import com.sun.management.ThreadMXBean;
+
+public class MemoryMonitor {
+
+ private static final Logger LOG = Log.logger(MemoryMonitor.class);
+ private final double MEMORY_MONITOR_THRESHOLD;
+ private final int MEMORY_MONITOR_DETECT_PERIOD;
+ private final ScheduledExecutorService scheduler;
+
+ public MemoryMonitor(String restServerConf) {
+ HugeConfig restServerConfig = new HugeConfig(restServerConf);
+ MEMORY_MONITOR_THRESHOLD =
+
restServerConfig.get(ServerOptions.JVM_MEMORY_MONITOR_THRESHOLD);
+ MEMORY_MONITOR_DETECT_PERIOD =
+
restServerConfig.get(ServerOptions.JVM_MEMORY_MONITOR_DETECT_PERIOD);
+ this.scheduler =
ExecutorUtil.newScheduledThreadPool("memory-monitor-thread-%d");
+ }
+
+ private void runMemoryDetect() {
+ double memoryUsagePercentage = getMemoryUsageRatio();
+
+ if (memoryUsagePercentage > MEMORY_MONITOR_THRESHOLD) {
+ LOG.warn("JVM memory usage is '{}', exceeding the threshold of
'{}'.",
+ memoryUsagePercentage, MEMORY_MONITOR_THRESHOLD);
+ System.gc();
+ LOG.warn("Trigger System.gc()");
+
+ double doubleCheckUsage = getMemoryUsageRatio();
+ if (doubleCheckUsage > MEMORY_MONITOR_THRESHOLD) {
+ LOG.warn("JVM memory usage is '{}', exceeding the threshold of
'{}'.",
+ doubleCheckUsage, MEMORY_MONITOR_THRESHOLD);
+ interruptHighestMemoryThread();
+ }
+ }
+ }
+
+ private double getMemoryUsageRatio() {
+ MemoryUsage heapMemoryUsage =
ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
+ return (double) heapMemoryUsage.getUsed() / heapMemoryUsage.getMax();
+ }
+
+ private Thread getHighestMemoryThread() {
+ long highestMemory = 0;
+ Thread highestThread = null;
+
+ ThreadMXBean threadMXBean = (ThreadMXBean)
ManagementFactory.getThreadMXBean();
+
+ Thread[] threads = new Thread[Thread.activeCount()];
+ Thread.enumerate(threads);
+ for (Thread thread : threads) {
+ if (thread.getState() != Thread.State.RUNNABLE || thread.getName()
== null ||
+ !thread.getName().startsWith("grizzly-http-server-")) {
+ continue;
+ }
+
+ long threadMemory =
threadMXBean.getThreadAllocatedBytes(thread.getId());
+ if (threadMemory > highestMemory) {
+ highestMemory = threadMemory;
+ highestThread = thread;
+ }
+ }
+ return highestThread;
+ }
+
+ private void interruptHighestMemoryThread() {
+ Thread targetThread = getHighestMemoryThread();
+ if (targetThread != null) {
+ targetThread.interrupt();
+ LOG.warn("Send interrupt to '{}' thread", targetThread.getName());
+ }
+ }
+
+ public void start() {
+ if (MEMORY_MONITOR_THRESHOLD >= 1.0) {
+ LOG.info("Invalid parameter, MEMORY_MONITOR_THRESHOLD should ≤
1.0.");
+ return;
+ }
+ this.scheduler.scheduleAtFixedRate(this::runMemoryDetect, 0,
MEMORY_MONITOR_DETECT_PERIOD,
+ TimeUnit.MILLISECONDS);
+ LOG.info("Memory monitoring started.");
+ }
+
+ public void stop() {
+ if (MEMORY_MONITOR_THRESHOLD >= 1.0) {
+ return;
+ }
+ this.scheduler.shutdownNow();
+ LOG.info("Memory monitoring stopped.");
+ }
+}
