This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new b5fe66ee754 Add eviction metrics in binary allocator (#14432)
b5fe66ee754 is described below
commit b5fe66ee754a43c7ed9a069c4d998f0bd97affde
Author: Mrquan <[email protected]>
AuthorDate: Tue Dec 17 14:28:32 2024 +0800
Add eviction metrics in binary allocator (#14432)
* add eviction metrics
* fix
* fix overflow
Signed-off-by: OneSizeFitQuorum <[email protected]>
---------
Signed-off-by: OneSizeFitQuorum <[email protected]>
Co-authored-by: OneSizeFitQuorum <[email protected]>
---
.../commons/binaryallocator/BinaryAllocator.java | 18 ++++++----
.../iotdb/commons/binaryallocator/arena/Arena.java | 39 ++++++++++++++--------
.../metric/BinaryAllocatorMetrics.java | 36 +++++++++++++++++++-
3 files changed, 72 insertions(+), 21 deletions(-)
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/binaryallocator/BinaryAllocator.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/binaryallocator/BinaryAllocator.java
index 3193fd6abd2..ca676176a60 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/binaryallocator/BinaryAllocator.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/binaryallocator/BinaryAllocator.java
@@ -155,10 +155,12 @@ public class BinaryAllocator {
return metrics;
}
- private void evict(double ratio) {
+ private long evict(double ratio) {
+ long evictedSize = 0;
for (Arena arena : heapArenas) {
- arena.evict(ratio);
+ evictedSize += arena.evict(ratio);
}
+ return evictedSize;
}
public static BinaryAllocator getInstance() {
@@ -231,17 +233,19 @@ public class BinaryAllocator {
return;
}
+ long evictedSize = 0;
if (curGcTimePercent > SHUTDOWN_GC_TIME_PERCENTAGE) {
LOGGER.info(
"Binary allocator is shutting down because of high GC time
percentage {}%.",
curGcTimePercent);
- evict(1.0);
+ evictedSize = evict(1.0);
close(false);
} else if (curGcTimePercent > HALF_GC_TIME_PERCENTAGE) {
- evict(0.5);
+ evictedSize = evict(0.5);
} else if (curGcTimePercent > WARNING_GC_TIME_PERCENTAGE) {
- evict(0.2);
+ evictedSize = evict(0.2);
}
+ metrics.updateGcEvictionCounter(evictedSize);
}
public class SampleEvictor extends Evictor {
@@ -252,9 +256,11 @@ public class BinaryAllocator {
@Override
public void run() {
+ long evictedSize = 0;
for (Arena arena : heapArenas) {
- arena.runSampleEviction();
+ evictedSize += arena.runSampleEviction();
}
+ metrics.updateSampleEvictionCounter(evictedSize);
}
}
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/binaryallocator/arena/Arena.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/binaryallocator/arena/Arena.java
index cfb83ed0708..9a38cb1fe58 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/binaryallocator/arena/Arena.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/binaryallocator/arena/Arena.java
@@ -68,10 +68,12 @@ public class Arena {
regions[sizeIdx].deallocate(bytes);
}
- public void evict(double ratio) {
+ public long evict(double ratio) {
+ long evictedSize = 0;
for (SlabRegion region : regions) {
- region.evict(ratio);
+ evictedSize += region.evict(ratio);
}
+ return evictedSize;
}
public void close() {
@@ -109,19 +111,23 @@ public class Arena {
this.numRegisteredThread.decrementAndGet();
}
- public void runSampleEviction() {
+ public long runSampleEviction() {
// update metric
- int allocateFromSlabDelta = 0;
- int allocateFromJVMDelta = 0;
+ long allocateFromSlabDelta = 0;
+ long allocateFromJVMDelta = 0;
for (SlabRegion region : regions) {
allocateFromSlabDelta +=
- region.byteArraySize * (region.allocationsFromAllocator.get() -
region.prevAllocations);
+ (long) region.byteArraySize
+ * (region.allocationsFromAllocator.get() -
region.prevAllocations);
region.prevAllocations = region.allocationsFromAllocator.get();
allocateFromJVMDelta +=
- region.byteArraySize * (region.allocationsFromJVM.get() -
region.prevAllocationsFromJVM);
+ (long) region.byteArraySize
+ * (region.allocationsFromJVM.get() -
region.prevAllocationsFromJVM);
region.prevAllocationsFromJVM = region.allocationsFromJVM.get();
}
- binaryAllocator.getMetrics().updateCounter(allocateFromSlabDelta,
allocateFromJVMDelta);
+ binaryAllocator
+ .getMetrics()
+ .updateAllocationCounter(allocateFromSlabDelta, allocateFromJVMDelta);
// Start sampling
for (SlabRegion region : regions) {
@@ -129,13 +135,15 @@ public class Arena {
}
sampleCount++;
+ long evictedSize = 0;
if (sampleCount == EVICT_SAMPLE_COUNT) {
// Evict
for (SlabRegion region : regions) {
- region.resize();
+ evictedSize += region.resize();
}
sampleCount = 0;
}
+ return evictedSize;
}
private static class SlabRegion {
@@ -182,22 +190,25 @@ public class Arena {
average.sample(getActiveSize());
}
- private void resize() {
+ private long resize() {
average.update();
int needRemain = (int) Math.ceil(average.average()) - getActiveSize();
- evict(getQueueSize() - needRemain);
+ return evict(getQueueSize() - needRemain);
}
- private void evict(double ratio) {
- evict((int) (getQueueSize() * ratio));
+ private long evict(double ratio) {
+ return evict((int) (getQueueSize() * ratio));
}
- private void evict(int num) {
+ private long evict(int num) {
+ long evicted = 0;
while (num > 0 && !queue.isEmpty()) {
queue.poll();
evictions.incrementAndGet();
num--;
+ evicted += byteArraySize;
}
+ return evicted;
}
private long getTotalUsedMemory() {
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/binaryallocator/metric/BinaryAllocatorMetrics.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/binaryallocator/metric/BinaryAllocatorMetrics.java
index 475f98860ac..305dadc432b 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/binaryallocator/metric/BinaryAllocatorMetrics.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/binaryallocator/metric/BinaryAllocatorMetrics.java
@@ -34,10 +34,14 @@ public class BinaryAllocatorMetrics implements IMetricSet {
private static final String ALLOCATE_FROM_SLAB = "allocate-from-slab";
private static final String ALLOCATE_FROM_JVM = "allocate-from-jvm";
private static final String ACTIVE_MEMORY = "active-memory";
+ private static final String EVICTED_BY_SAMPLE_EVICTION =
"evicted-by-sample-eviction";
+ private static final String EVICTED_BY_GC_EVICTION =
"evicted-by-gc-eviction";
private final BinaryAllocator binaryAllocator;
private Counter allocateFromSlab;
private Counter allocateFromJVM;
+ private Counter evictedBySampleEviction;
+ private Counter evictedByGcEviction;
public BinaryAllocatorMetrics(final BinaryAllocator binaryAllocator) {
this.binaryAllocator = binaryAllocator;
@@ -71,6 +75,18 @@ public class BinaryAllocatorMetrics implements IMetricSet {
MetricLevel.IMPORTANT,
Tag.NAME.toString(),
ALLOCATE_FROM_JVM);
+ evictedBySampleEviction =
+ metricService.getOrCreateCounter(
+ Metric.BINARY_ALLOCATOR.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.NAME.toString(),
+ EVICTED_BY_SAMPLE_EVICTION);
+ evictedByGcEviction =
+ metricService.getOrCreateCounter(
+ Metric.BINARY_ALLOCATOR.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.NAME.toString(),
+ EVICTED_BY_GC_EVICTION);
}
@Override
@@ -95,10 +111,28 @@ public class BinaryAllocatorMetrics implements IMetricSet {
Metric.BINARY_ALLOCATOR.toString(),
Tag.NAME.toString(),
ALLOCATE_FROM_JVM);
+ metricService.remove(
+ MetricType.COUNTER,
+ Metric.BINARY_ALLOCATOR.toString(),
+ Tag.NAME.toString(),
+ EVICTED_BY_SAMPLE_EVICTION);
+ metricService.remove(
+ MetricType.COUNTER,
+ Metric.BINARY_ALLOCATOR.toString(),
+ Tag.NAME.toString(),
+ EVICTED_BY_GC_EVICTION);
}
- public void updateCounter(int allocateFromSlabDelta, int
allocateFromJVMDelta) {
+ public void updateAllocationCounter(long allocateFromSlabDelta, long
allocateFromJVMDelta) {
allocateFromSlab.inc(allocateFromSlabDelta);
allocateFromJVM.inc(allocateFromJVMDelta);
}
+
+ public void updateGcEvictionCounter(long evictedByGcEvictionDelta) {
+ evictedByGcEviction.inc(evictedByGcEvictionDelta);
+ }
+
+ public void updateSampleEvictionCounter(long evictedBySampleEvictionDelta) {
+ evictedBySampleEviction.inc(evictedBySampleEvictionDelta);
+ }
}
