This is an automated email from the ASF dual-hosted git repository.
wyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new e3aae4a56d [ASTERIXDB-3437][STO] Make eviction plan reevaluation
threshold configurable
e3aae4a56d is described below
commit e3aae4a56d0704b81bbcfb95103bf32c190b9f21
Author: Ritik Raj <[email protected]>
AuthorDate: Wed Jun 19 00:03:43 2024 +0530
[ASTERIXDB-3437][STO] Make eviction plan reevaluation threshold configurable
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
This patch makes eviction plan reevaluation threshold configurable.
The default value is set to 50 operation that require cloud read,
(e.g., running 50 query that require cloud read).
Change-Id: I6b937610982c13f16552f8aff4acc7608b45ee7d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18384
Integration-Tests: Jenkins <[email protected]>
Reviewed-by: Ritik Raj <[email protected]>
Reviewed-by: Wail Alkowaileet <[email protected]>
Tested-by: Jenkins <[email protected]>
---
.../results/api/cluster_state_1/cluster_state_1.1.regexadm | 1 +
.../api/cluster_state_1_full/cluster_state_1_full.1.regexadm | 1 +
.../api/cluster_state_1_less/cluster_state_1_less.1.regexadm | 1 +
.../main/java/org/apache/asterix/cloud/CloudConfigurator.java | 4 ++--
.../java/org/apache/asterix/common/config/CloudProperties.java | 10 +++++++++-
.../service/CloudDiskCacheMonitoringAndPrefetchingService.java | 9 ++++++++-
.../btree/column/cloud/CloudColumnIndexDiskCacheManager.java | 6 +++---
.../am/lsm/btree/column/cloud/sweep/ColumnSweepPlanner.java | 6 ++++--
.../storage/am/lsm/btree/column/utils/LSMColumnBTreeUtil.java | 1 +
.../lsm/btree/column/cloud/sweep/ColumnSweepPlannerTest.java | 3 ++-
.../storage/common/disk/IDiskCacheMonitoringService.java | 5 +++++
.../storage/common/disk/NoOpDiskCacheMonitoringService.java | 5 +++++
12 files changed, 42 insertions(+), 10 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 107de0079a..805f785601 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -10,6 +10,7 @@
"active\.suspend\.timeout" : 3600,
"azure.request.timeout" : 120,
"cloud.deployment" : false,
+ "cloud.eviction.plan.reevaluate.threshold" : 50,
"cloud.profiler.log.interval" : 5,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index bdeedd67f7..35a488af6b 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -10,6 +10,7 @@
"active\.suspend\.timeout" : 3600,
"azure.request.timeout" : 120,
"cloud.deployment" : false,
+ "cloud.eviction.plan.reevaluate.threshold" : 50,
"cloud.profiler.log.interval" : 5,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 96fa23ca42..13e7a4e13d 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -10,6 +10,7 @@
"active\.suspend\.timeout" : 3600,
"azure.request.timeout" : 120,
"cloud.deployment" : false,
+ "cloud.eviction.plan.reevaluate.threshold" : 50,
"cloud.profiler.log.interval" : 5,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudConfigurator.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudConfigurator.java
index 3c0f1df00e..dced5b0f44 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudConfigurator.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudConfigurator.java
@@ -124,8 +124,8 @@ public final class CloudConfigurator {
resourceCacheManager, cloudIOManager, numOfSweepThreads,
maxSweepQueueSize, physicalDrive,
diskBufferCache, fileInfoMap, inactiveThreshold);
- IDiskCacheMonitoringService diskCacheService =
- new CloudDiskCacheMonitoringAndPrefetchingService(executor,
physicalDrive, monitorThread);
+ IDiskCacheMonitoringService diskCacheService = new
CloudDiskCacheMonitoringAndPrefetchingService(executor,
+ physicalDrive, monitorThread,
cloudProperties.getEvictionPlanReevaluationThreshold());
localIoManager.setSpaceMaker(monitorThread);
return diskCacheService;
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
index 7e7ac69867..a4d3b7ab56 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
@@ -60,7 +60,8 @@ public class CloudProperties extends AbstractProperties {
CLOUD_PROFILER_LOG_INTERVAL(NONNEGATIVE_INTEGER, 5),
CLOUD_WRITE_BUFFER_SIZE(
getRangedIntegerType(5, Integer.MAX_VALUE),
- StorageUtil.getIntSizeInBytes(8,
StorageUtil.StorageUnit.MEGABYTE));
+ StorageUtil.getIntSizeInBytes(8,
StorageUtil.StorageUnit.MEGABYTE)),
+ CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD(POSITIVE_INTEGER, 50);
private final IOptionType interpreter;
private final Object defaultValue;
@@ -88,6 +89,7 @@ public class CloudProperties extends AbstractProperties {
case CLOUD_STORAGE_DEBUG_MODE_ENABLED:
case CLOUD_PROFILER_LOG_INTERVAL:
case CLOUD_WRITE_BUFFER_SIZE:
+ case CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD:
return Section.COMMON;
default:
return Section.NC;
@@ -146,6 +148,8 @@ public class CloudProperties extends AbstractProperties {
+ " NOTE: Enabling the profiler could perturb the
performance of cloud requests";
case CLOUD_WRITE_BUFFER_SIZE:
return "The write buffer size in bytes. (default: 8MB,
min: 5MB)";
+ case CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD:
+ return "The number of cloud reads for re-evaluating an
eviction plan. (default: 50)";
default:
throw new IllegalStateException("NYI: " + this);
}
@@ -224,4 +228,8 @@ public class CloudProperties extends AbstractProperties {
public int getWriteBufferSize() {
return accessor.getInt(Option.CLOUD_WRITE_BUFFER_SIZE);
}
+
+ public int getEvictionPlanReevaluationThreshold() {
+ return
accessor.getInt(Option.CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD);
+ }
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/cache/service/CloudDiskCacheMonitoringAndPrefetchingService.java
b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/cache/service/CloudDiskCacheMonitoringAndPrefetchingService.java
index 4564856d94..b58a6bf266 100644
---
a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/cache/service/CloudDiskCacheMonitoringAndPrefetchingService.java
+++
b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/cache/service/CloudDiskCacheMonitoringAndPrefetchingService.java
@@ -33,12 +33,14 @@ public final class
CloudDiskCacheMonitoringAndPrefetchingService
private final ExecutorService executor;
private final DiskCacheSweeperThread monitorThread;
private final IPhysicalDrive drive;
+ private final int evictionPlanReevaluationThreshold;
public CloudDiskCacheMonitoringAndPrefetchingService(ExecutorService
executor, IPhysicalDrive drive,
- DiskCacheSweeperThread monitorThread) {
+ DiskCacheSweeperThread monitorThread, int
evictionPlanReevaluationThreshold) {
this.executor = executor;
this.drive = drive;
this.monitorThread = monitorThread;
+ this.evictionPlanReevaluationThreshold =
evictionPlanReevaluationThreshold;
}
@Override
@@ -81,4 +83,9 @@ public final class
CloudDiskCacheMonitoringAndPrefetchingService
public void request(AbstractPrefetchRequest request) throws
HyracksDataException {
// TODO implement
}
+
+ @Override
+ public int getEvictionPlanReevaluationThreshold() {
+ return evictionPlanReevaluationThreshold;
+ }
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/CloudColumnIndexDiskCacheManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/CloudColumnIndexDiskCacheManager.java
index 7187496874..417322e75c 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/CloudColumnIndexDiskCacheManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/CloudColumnIndexDiskCacheManager.java
@@ -48,11 +48,11 @@ public final class CloudColumnIndexDiskCacheManager
implements IColumnIndexDiskC
private final ColumnSweepPlanner planner;
private final ColumnSweeper sweeper;
- public CloudColumnIndexDiskCacheManager(int numberOfPrimaryKeys,
IColumnTupleProjector sweepProjector,
- IPhysicalDrive drive) {
+ public CloudColumnIndexDiskCacheManager(int numberOfPrimaryKeys, int
evictionPlanReevaluationThreshold,
+ IColumnTupleProjector sweepProjector, IPhysicalDrive drive) {
this.sweepProjector = sweepProjector;
this.drive = drive;
- planner = new ColumnSweepPlanner(numberOfPrimaryKeys,
System::nanoTime);
+ planner = new ColumnSweepPlanner(numberOfPrimaryKeys,
evictionPlanReevaluationThreshold, System::nanoTime);
sweeper = new ColumnSweeper(numberOfPrimaryKeys);
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlanner.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlanner.java
index 38758122c9..9d52603ebd 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlanner.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlanner.java
@@ -57,6 +57,7 @@ public final class ColumnSweepPlanner {
private final BitSet reevaluatedPlan;
private final IntSet indexedColumns;
private final ISweepClock clock;
+ private final int evictionPlanReevaluationThreshold;
private int numberOfColumns;
private long lastAccess;
private int maxSize;
@@ -67,7 +68,7 @@ public final class ColumnSweepPlanner {
private int numberOfSweptColumns;
private int numberOfCloudRequests;
- public ColumnSweepPlanner(int numberOfPrimaryKeys, ISweepClock clock) {
+ public ColumnSweepPlanner(int numberOfPrimaryKeys, int
evictionPlanReevaluationThreshold, ISweepClock clock) {
this.clock = clock;
active = new AtomicBoolean(false);
this.numberOfPrimaryKeys = numberOfPrimaryKeys;
@@ -77,6 +78,7 @@ public final class ColumnSweepPlanner {
plan = new BitSet();
reevaluatedPlan = new BitSet();
punchableThreshold = INITIAL_PUNCHABLE_THRESHOLD;
+ this.evictionPlanReevaluationThreshold =
evictionPlanReevaluationThreshold;
}
public boolean isActive() {
@@ -235,7 +237,7 @@ public final class ColumnSweepPlanner {
}
private void resetPlanIfNeeded() {
- if (numberOfCloudRequests < REEVALUATE_PLAN_THRESHOLD) {
+ if (numberOfCloudRequests < evictionPlanReevaluationThreshold) {
return;
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/LSMColumnBTreeUtil.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/LSMColumnBTreeUtil.java
index 6335acd21a..04b49846d5 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/LSMColumnBTreeUtil.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/LSMColumnBTreeUtil.java
@@ -76,6 +76,7 @@ public class LSMColumnBTreeUtil {
IColumnManager columnManager =
columnManagerFactory.createColumnManager();
IColumnIndexDiskCacheManager diskCacheManager = diskCacheEnabled
? new
CloudColumnIndexDiskCacheManager(columnManager.getNumberOfPrimaryKeys(),
+
diskCacheService.getEvictionPlanReevaluationThreshold(),
columnManager.getMergeColumnProjector(),
diskCacheService.getPhysicalDrive())
: NoOpColumnIndexDiskCacheManager.INSTANCE;
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/test/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlannerTest.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/test/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlannerTest.java
index a5a87c48fb..e4b3449af7 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/test/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlannerTest.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/test/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlannerTest.java
@@ -38,6 +38,7 @@ import it.unimi.dsi.fastutil.ints.IntSet;
public class ColumnSweepPlannerTest {
private static final int MAX_MEGA_LEAF_NODE_SIZE = getIntSizeInBytes(10,
StorageUtil.StorageUnit.MEGABYTE);
+ private static final int REEVALUATE_PLAN_THRESHOLD = 50;
private static final Random RANDOM = new Random(0);
private final DummySweepClock clock = new DummySweepClock();
@@ -46,7 +47,7 @@ public class ColumnSweepPlannerTest {
int numberOfPrimaryKeys = 1;
int numberOfColumns = numberOfPrimaryKeys + 10;
int[] columnSizes = createNormalColumnSizes(numberOfPrimaryKeys,
numberOfColumns);
- ColumnSweepPlanner planner = new
ColumnSweepPlanner(numberOfPrimaryKeys, clock);
+ ColumnSweepPlanner planner = new
ColumnSweepPlanner(numberOfPrimaryKeys, REEVALUATE_PLAN_THRESHOLD, clock);
IntList projectedColumns = new IntArrayList();
DummyColumnProjectionInfo info = new
DummyColumnProjectionInfo(numberOfPrimaryKeys, QUERY, projectedColumns);
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/disk/IDiskCacheMonitoringService.java
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/disk/IDiskCacheMonitoringService.java
index 4151c77fe5..94974638ad 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/disk/IDiskCacheMonitoringService.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/disk/IDiskCacheMonitoringService.java
@@ -62,4 +62,9 @@ public interface IDiskCacheMonitoringService {
* @return physical drive
*/
IPhysicalDrive getPhysicalDrive();
+
+ /**
+ * @return threshold for re-evaluation of eviction plan
+ */
+ int getEvictionPlanReevaluationThreshold();
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/disk/NoOpDiskCacheMonitoringService.java
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/disk/NoOpDiskCacheMonitoringService.java
index 5cbf5cb5c4..ad153fea10 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/disk/NoOpDiskCacheMonitoringService.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/disk/NoOpDiskCacheMonitoringService.java
@@ -62,4 +62,9 @@ public final class NoOpDiskCacheMonitoringService implements
IDiskCacheMonitorin
public IPhysicalDrive getPhysicalDrive() {
return DummyPhysicalDrive.INSTANCE;
}
+
+ @Override
+ public int getEvictionPlanReevaluationThreshold() {
+ return 0;
+ }
}
