jt2594838 commented on code in PR #18848:
URL: https://github.com/apache/iotdb/pull/18848#discussion_r4269611732
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java:
##########
@@ -382,6 +407,68 @@ public static void recordFlushPointsMetricInternal(
WRITING_METRICS.recordFlushCost(WritingMetrics.FLUSH_STAGE_IO, ioTime);
};
+ /** Tracks actual worker exit, including cancellation before the executor
starts it. */
+ private final class FlushSubTask implements Runnable {
+ private final Runnable task;
+ private final CompletableFuture<Void> finished = new CompletableFuture<>();
+ private Thread runner;
+ private boolean cancelled;
+
+ private FlushSubTask(Runnable task) {
+ this.task = task;
+ }
+
+ @Override
+ public void run() {
+ synchronized (this) {
+ if (cancelled) {
+ return;
+ }
+ runner = Thread.currentThread();
+ }
+ try {
+ task.run();
+ } catch (Throwable failure) {
+ subTaskFailure.completeExceptionally(
+ failure instanceof CompletionException || failure instanceof
UncheckedIOException
+ ? failure.getCause()
+ : failure);
+ } finally {
+ synchronized (this) {
+ runner = null;
+ finished.complete(null);
+ }
+ }
+ }
+
+ private synchronized boolean isCancelled() {
+ return cancelled;
+ }
+
+ private synchronized void cancel() {
+ cancelled = true;
+ if (runner != null) {
+ runner.interrupt();
+ } else {
+ finished.complete(null);
+ }
+ }
+
+ private boolean awaitExit() {
+ boolean interrupted = false;
+ while (true) {
+ try {
+ finished.get();
+ return interrupted;
+ } catch (InterruptedException e) {
+ interrupted = true;
+ } catch (ExecutionException e) {
+ throw new AssertionError(e);
+ }
+ }
+ }
Review Comment:
Why does the interruption not exit?
And why is the ExecutionException wrapped with AssertionError?
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java:
##########
@@ -77,7 +79,13 @@ void writeAlignedRow(
*/
long memSize();
- /** only used when mem control enabled */
+ void setWriteMemoryManager(WriteMemoryManager manager);
+
+ WriteMemoryBlock getArrayMemoryBlock();
+
+ WriteMemoryBlock getBinaryMemoryBlock();
+
+ /** Adds array/TVList structural memory; Binary payload has a separate
block. */
void addTVListRamCost(long cost);
Review Comment:
Add a structureMemoryBlock?
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfo.java:
##########
@@ -369,8 +371,10 @@ public long getMemorySizeForCompaction() {
return compactionMemoryBlock.getTotalMemorySizeInBytes();
}
- public void loadWriteMemory() {
- memorySizeForMemtable =
memoryConfig.getMemtableMemoryManager().getTotalMemorySizeInBytes();
+ public synchronized void loadWriteMemory() {
+ memorySizeForMemtable =
+ memoryConfig.getMemtableMemoryManager().getTotalMemorySizeInBytes()
+ - flushMemoryManager.getMemoryUsageInBytes();
Review Comment:
explain the change
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java:
##########
@@ -268,12 +283,45 @@ public static void release(Object array) {
ArrayDeque<Object> arrays = POOLED_ARRAYS[order];
if (arrays.size() < LIMITS[order]) {
arrays.add(array);
+ POOLED_ARRAY_BYTES[order] += arraySize(array);
} else {
StorageEngineMemoryMetrics.getInstance().incPamReleaseFailure();
}
}
}
+ public static IMemoryBlock getMemoryBlock() {
+ return POOLED_ARRAYS_MEMORY_BLOCK;
+ }
+
+ private static long getPooledArrayMemoryInBytes(int order) {
+ if (order == POOLED_ARRAYS.length) {
+ return 0;
+ }
+ // Hold every queue lock in ascending order for a linearizable total.
Borrow/return paths
+ // hold only one queue lock and never call this method or the memory
manager.
+ synchronized (POOLED_ARRAYS[order]) {
+ return POOLED_ARRAY_BYTES[order] + getPooledArrayMemoryInBytes(order +
1);
+ }
+ }
+
+ private static long arraySize(Object array) {
+ // Use typed array lengths on this hot path instead of reflective
shallowSizeOf(Object).
+ if (array instanceof long[]) {
+ return RamUsageEstimator.sizeOf((long[]) array);
+ } else if (array instanceof int[]) {
+ return RamUsageEstimator.sizeOf((int[]) array);
+ } else if (array instanceof boolean[]) {
+ return RamUsageEstimator.sizeOf((boolean[]) array);
+ } else if (array instanceof float[]) {
+ return RamUsageEstimator.sizeOf((float[]) array);
+ } else if (array instanceof double[]) {
+ return RamUsageEstimator.sizeOf((double[]) array);
+ }
+ // Binary[] entries have been cleared; only the shallow reference array
belongs to the pool.
+ return RamUsageEstimator.shallowSizeOf((Binary[]) array);
+ }
Review Comment:
How about caching the size calculators in an array consistent with
POOLED_ARRAYS to reduce unnecessary type judgement.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]