This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit e60a6f09cc867ca40396a73fcbe1275f9e9204a9 Author: Michael Blow <[email protected]> AuthorDate: Thu Feb 20 16:58:00 2025 -0500 [NO ISSUE][HYR][STO] Introduce new modification op type to disregard cache budget += BUDGET_FREE_MODIFICATION opType to allow an LSM modification to proceed despite the buffer cache being full. Ext-ref: MB-65412 Change-Id: I62ecc294e8846bdf5a38652955bdbe8b62a7a1a9 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19451 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> --- .../common/context/PrimaryIndexOperationTracker.java | 6 ++++-- .../storage/am/lsm/common/api/LSMOperationType.java | 3 ++- .../am/lsm/common/impls/AbstractLSMDiskComponent.java | 2 ++ .../lsm/common/impls/AbstractLSMMemoryComponent.java | 19 +++++++++++++++++++ .../storage/am/lsm/common/impls/LSMHarness.java | 10 ++++++---- 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java index 34bc587c3b..33929ad5d1 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java @@ -89,7 +89,8 @@ public class PrimaryIndexOperationTracker extends BaseOperationTracker implement public void beforeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback, IModificationOperationCallback modificationCallback) throws HyracksDataException { super.beforeOperation(index, opType, searchCallback, modificationCallback); - if (opType == LSMOperationType.MODIFICATION || opType == LSMOperationType.FORCE_MODIFICATION) { + if (opType == LSMOperationType.MODIFICATION || opType == LSMOperationType.FORCE_MODIFICATION + || opType == LSMOperationType.BUDGET_FREE_MODIFICATION) { incrementNumActiveOperations(modificationCallback); } } @@ -99,7 +100,8 @@ public class PrimaryIndexOperationTracker extends BaseOperationTracker implement ISearchOperationCallback searchCallback, IModificationOperationCallback modificationCallback) throws HyracksDataException { super.completeOperation(index, opType, searchCallback, modificationCallback); - if (opType == LSMOperationType.MODIFICATION || opType == LSMOperationType.FORCE_MODIFICATION) { + if (opType == LSMOperationType.MODIFICATION || opType == LSMOperationType.FORCE_MODIFICATION + || opType == LSMOperationType.BUDGET_FREE_MODIFICATION) { decrementNumActiveOperations(modificationCallback); flushIfNeeded(); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/LSMOperationType.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/LSMOperationType.java index e200bfdec5..52a4f94bf6 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/LSMOperationType.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/LSMOperationType.java @@ -26,5 +26,6 @@ public enum LSMOperationType { FLUSH, MERGE, REPLICATE, - DISK_COMPONENT_SCAN + DISK_COMPONENT_SCAN, + BUDGET_FREE_MODIFICATION } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java index baf0931319..627b853919 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java @@ -79,6 +79,7 @@ public abstract class AbstractLSMDiskComponent extends AbstractLSMComponent impl switch (opType) { case FORCE_MODIFICATION: case MODIFICATION: + case BUDGET_FREE_MODIFICATION: case REPLICATE: case SEARCH: case DISK_COMPONENT_SCAN: @@ -107,6 +108,7 @@ public abstract class AbstractLSMDiskComponent extends AbstractLSMComponent impl break; case FORCE_MODIFICATION: case MODIFICATION: + case BUDGET_FREE_MODIFICATION: case REPLICATE: case SEARCH: case DISK_COMPONENT_SCAN: diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java index 919dc60081..77c5ec2999 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java @@ -110,6 +110,24 @@ public abstract class AbstractLSMMemoryComponent extends AbstractLSMComponent im } } break; + case BUDGET_FREE_MODIFICATION: + if (isMutableComponent) { + if (state == ComponentState.READABLE_WRITABLE) { + // Even when the memory component has the writable state, vbc may be temporarily full + // or this memory component may be full. + writerCount++; + } else { + return false; + } + } else { + if (state == ComponentState.READABLE_UNWRITABLE + || state == ComponentState.READABLE_UNWRITABLE_FLUSHING) { + readerCount++; + } else { + return false; + } + } + break; case MODIFICATION: if (isMutableComponent) { if (state == ComponentState.READABLE_WRITABLE && !vbc.isFull(this) && !vbc.isFull()) { @@ -161,6 +179,7 @@ public abstract class AbstractLSMMemoryComponent extends AbstractLSMComponent im boolean cleanup = false; switch (opType) { case FORCE_MODIFICATION: + case BUDGET_FREE_MODIFICATION: case MODIFICATION: if (isMutableComponent) { writerCount--; diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java index d019a088a1..6e7681da6c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java @@ -220,8 +220,9 @@ public class LSMHarness implements ILSMHarness { LOGGER.warn("Failure exiting components", e); throw e; } finally { - if (failedOperation && (opType == LSMOperationType.MODIFICATION - || opType == LSMOperationType.FORCE_MODIFICATION)) { + if (failedOperation + && (opType == LSMOperationType.MODIFICATION || opType == LSMOperationType.FORCE_MODIFICATION + || opType == LSMOperationType.BUDGET_FREE_MODIFICATION)) { //When the operation failed, completeOperation() method must be called //in order to decrement active operation count which was incremented // in beforeOperation() method. @@ -331,8 +332,9 @@ public class LSMHarness implements ILSMHarness { boolean needsCleanup = c.threadExit(opType, failedOperation, isMutableComponent); if (c.getType() == LSMComponentType.MEMORY) { if (c.getState() == ComponentState.READABLE_UNWRITABLE) { - if (isMutableComponent && (opType == LSMOperationType.MODIFICATION - || opType == LSMOperationType.FORCE_MODIFICATION)) { + if (isMutableComponent + && (opType == LSMOperationType.MODIFICATION || opType == LSMOperationType.FORCE_MODIFICATION + || opType == LSMOperationType.BUDGET_FREE_MODIFICATION)) { lsmIndex.changeFlushStatusForCurrentMutableCompoent(true); } }
