This is an automated email from the ASF dual-hosted git repository. guangning pushed a commit to branch branch-2.5 in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 3c4ca9b75b30c1baf41a4da0b25e4c7715abea53 Author: Masahiro Sakamoto <[email protected]> AuthorDate: Sat Feb 8 15:10:19 2020 +0900 Output resource usage rate to log on broker (#6152) ### Motivation When a broker is under heavy load, the following log may be output and some topics may be unloaded. > Attempting to shed load on broker101.pulsar.xxx.yahoo.co.jp:4080, which has max resource usage above threshold 0.8708186149597168% > 0.85% -- Offloading at least 0.36224863451117845 MByte/s of traffic This log means that the usage rate of CPU, memory, direct memory, input bandwidth, or output bandwidth has exceeded the threshold, but we don't know which resource usage is high. ### Modifications Output these resource usages along with the above log. > Attempting to shed load on broker101.pulsar.xxx.yahoo.co.jp:4080, which has resource usage 87.08% above threshold 85.0% -- Offloading at least 0.36224863451117845 MByte/s of traffic (cpu: 87.08%, memory: 12.71%, directMemory: 17.19%, bandwidthIn: 11.28%, bandwidthOut: 0.00%) --- .../pulsar/broker/loadbalance/impl/OverloadShedder.java | 8 +++++--- .../broker/loadbalance/impl/OverloadShedderTest.java | 14 ++++++++++++++ .../pulsar/policies/data/loadbalancer/LocalBrokerData.java | 7 +++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedder.java index c61f8cb..bda69fe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedder.java @@ -73,7 +73,8 @@ public class OverloadShedder implements LoadSheddingStrategy { final double currentUsage = localData.getMaxResourceUsage(); if (currentUsage < overloadThreshold) { if (log.isDebugEnabled()) { - log.debug("[{}] Broker is not overloaded, ignoring at this point", broker); + log.debug("[{}] Broker is not overloaded, ignoring at this point ({})", broker, + localData.printResourceUsage()); } return; } @@ -86,8 +87,9 @@ public class OverloadShedder implements LoadSheddingStrategy { double minimumThroughputToOffload = brokerCurrentThroughput * percentOfTrafficToOffload; log.info( - "Attempting to shed load on {}, which has resource usage {}% above threshold {}% -- Offloading at least {} MByte/s of traffic", - broker, 100 * currentUsage, 100 * overloadThreshold, minimumThroughputToOffload / 1024 / 1024); + "Attempting to shed load on {}, which has resource usage {}% above threshold {}% -- Offloading at least {} MByte/s of traffic ({})", + broker, 100 * currentUsage, 100 * overloadThreshold, minimumThroughputToOffload / 1024 / 1024, + localData.printResourceUsage()); MutableDouble trafficMarkedToOffload = new MutableDouble(0); MutableBoolean atLeastOneBundleSelected = new MutableBoolean(false); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedderTest.java index 34725ee..14a86cc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedderTest.java @@ -188,4 +188,18 @@ public class OverloadShedderTest { assertFalse(bundlesToUnload.isEmpty()); assertEquals(bundlesToUnload.get("broker-1"), Lists.newArrayList("bundle-8", "bundle-7")); } + + @Test + public void testPrintResourceUsage() { + LocalBrokerData data = new LocalBrokerData(); + + data.setCpu(new ResourceUsage(10, 100)); + data.setMemory(new ResourceUsage(50, 100)); + data.setDirectMemory(new ResourceUsage(90, 100)); + data.setBandwidthIn(new ResourceUsage(30, 100)); + data.setBandwidthOut(new ResourceUsage(20, 100)); + + assertEquals(data.printResourceUsage(), + "cpu: 10.00%, memory: 50.00%, directMemory: 90.00%, bandwidthIn: 30.00%, bandwidthOut: 20.00%"); + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java index 865b691..a2d9814 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java @@ -205,6 +205,13 @@ public class LocalBrokerData extends JSONWritable implements LoadManagerReport { bandwidthOut.percentUsage()) / 100; } + public String printResourceUsage() { + return String.format( + "cpu: %.2f%%, memory: %.2f%%, directMemory: %.2f%%, bandwidthIn: %.2f%%, bandwidthOut: %.2f%%", + cpu.percentUsage(), memory.percentUsage(), directMemory.percentUsage(), bandwidthIn.percentUsage(), + bandwidthOut.percentUsage()); + } + private static float max(float...args) { float max = Float.NEGATIVE_INFINITY;
