This is an automated email from the ASF dual-hosted git repository. marklau99 pushed a commit to branch IOTDB-3164 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 92d80f752e3ad96c6e2c33841884c5cc78a9ddbe Author: Liu Xuxin <[email protected]> AuthorDate: Wed Jul 20 10:51:10 2022 +0800 temp --- .../db/engine/storagegroup/TsFileProcessor.java | 3 +++ .../iotdb/db/rescon/memory/MemoryController.java | 2 +- .../db/rescon/memory/WriteMemoryController.java | 24 +++++++++++++++------- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java index 1f041b8285..156b640caf 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java @@ -786,6 +786,7 @@ public class TsFileProcessor { tsFileProcessorInfo.addTSPMemCost(chunkMetadataIncrement); WriteMemoryController controller = WriteMemoryController.getInstance(); boolean allocateSuccess = false; + long startTime = System.nanoTime(); try { while (workMemTable.needToAllocate(memTableIncrement)) { allocateSuccess = @@ -795,6 +796,8 @@ public class TsFileProcessor { } else { storageGroupInfo.addAllocateSize(WriteMemoryController.FRAME_SIZE); workMemTable.addAllocatedMemSize(WriteMemoryController.FRAME_SIZE); + logger.error( + "Allocation time for {} is {} ns", storageGroupName, System.nanoTime() - startTime); } } } catch (WriteProcessRejectException e) { diff --git a/server/src/main/java/org/apache/iotdb/db/rescon/memory/MemoryController.java b/server/src/main/java/org/apache/iotdb/db/rescon/memory/MemoryController.java index 22fad77108..b6fa801547 100644 --- a/server/src/main/java/org/apache/iotdb/db/rescon/memory/MemoryController.java +++ b/server/src/main/java/org/apache/iotdb/db/rescon/memory/MemoryController.java @@ -33,7 +33,7 @@ public class MemoryController<T> { protected AtomicBoolean triggerRunning = new AtomicBoolean(); protected long triggerThreshold = -1; protected long limitSize = -1; - protected ReentrantLock lock = new ReentrantLock(false); + protected ReentrantLock lock = new ReentrantLock(true); protected Condition condition = lock.newCondition(); protected MemoryControllerTrigger<T> trigger = null; diff --git a/server/src/main/java/org/apache/iotdb/db/rescon/memory/WriteMemoryController.java b/server/src/main/java/org/apache/iotdb/db/rescon/memory/WriteMemoryController.java index 60aa0a5ec8..1581ecd7ac 100644 --- a/server/src/main/java/org/apache/iotdb/db/rescon/memory/WriteMemoryController.java +++ b/server/src/main/java/org/apache/iotdb/db/rescon/memory/WriteMemoryController.java @@ -46,7 +46,7 @@ public class WriteMemoryController extends MemoryController<TsFileProcessor> { private Set<StorageGroupInfo> infoSet = new CopyOnWriteArraySet<>(); private ExecutorService flushTaskSubmitThreadPool = IoTDBThreadPoolFactory.newFixedThreadPool(1, "FlushTask-Submit-Pool"); - public static final long FRAME_SIZE = 2L * 1024L * 1024L; + public static final long FRAME_SIZE = 16L * 1024L * 1024L; public WriteMemoryController(long limitSize) { super(limitSize); @@ -66,6 +66,10 @@ public class WriteMemoryController extends MemoryController<TsFileProcessor> { } if (!info.isRecorded()) { info.setRecorded(true); + logger.error( + "Record {}-{}", + info.getDataRegion().getLogicalStorageGroupName(), + info.getDataRegion().getDataRegionId()); infoSet.add(info); } return success; @@ -102,6 +106,10 @@ public class WriteMemoryController extends MemoryController<TsFileProcessor> { if (rejected && memoryUsage.get() < REJECT_THRESHOLD) { rejected = false; } + logger.error( + "Release {} MB, current usage is {} MB", + ((double) size) / 1024.0d / 1024.0d, + ((double) memoryUsage.get()) / 1024.0d / 1024.0d); } public boolean isRejected() { @@ -120,15 +128,17 @@ public class WriteMemoryController extends MemoryController<TsFileProcessor> { } public void applyExternalMemoryForFlushing(long size) { - // memorySizeForWrite -= size; - // FLUSH_THRESHOLD = memorySizeForWrite * config.getFlushProportion(); - // REJECT_THRESHOLD = memorySizeForWrite * config.getRejectProportion(); + memorySizeForWrite -= size; + FLUSH_THRESHOLD = memorySizeForWrite * config.getFlushProportion(); + REJECT_THRESHOLD = memorySizeForWrite * config.getRejectProportion(); + END_FLUSH_THRESHOLD = 0.5 * FLUSH_THRESHOLD; } public void releaseExternalMemoryForFlushing(long size) { - // memorySizeForWrite += size; - // FLUSH_THRESHOLD = memorySizeForWrite * config.getFlushProportion(); - // REJECT_THRESHOLD = memorySizeForWrite * config.getRejectProportion(); + memorySizeForWrite += size; + FLUSH_THRESHOLD = memorySizeForWrite * config.getFlushProportion(); + REJECT_THRESHOLD = memorySizeForWrite * config.getRejectProportion(); + END_FLUSH_THRESHOLD = 0.5 * FLUSH_THRESHOLD; } protected void chooseMemtableToFlush(TsFileProcessor currentTsFileProcessor) {
