This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 27f5467f8a [NO ISSUE][OTH] Configure cached resource ids based on
storage partitions
27f5467f8a is described below
commit 27f5467f8a15a51ec06f0bfe79dd917f9e336436
Author: Murtadha Hubail <[email protected]>
AuthorDate: Thu Sep 28 18:47:21 2023 +0300
[NO ISSUE][OTH] Configure cached resource ids based on storage partitions
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- To ensure each node has enough cached resource ids to create at least
one resource per storage partition, configure the cached resource ids
block size based on the storage partition.
Change-Id: I531fd366168209a132e552306a0f2b667e6e0342
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17815
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Ali Alsuliman <[email protected]>
---
.../apache/asterix/app/nc/NCAppRuntimeContext.java | 8 +++++++-
.../runtime/transaction/GlobalResourceIdFactory.java | 20 +++++++++++---------
.../transaction/GlobalResourceIdFactoryProvider.java | 6 ++++--
3 files changed, 22 insertions(+), 12 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index 630265ca18..9e796f2651 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -186,7 +186,8 @@ public class NCAppRuntimeContext implements
INcApplicationContext {
cloudProperties = propertiesFactory.newCloudProperties();
ncExtensionManager = extensionManager;
componentProvider = new StorageComponentProvider();
- resourceIdFactory = new
GlobalResourceIdFactoryProvider(ncServiceContext).createResourceIdFactory();
+ resourceIdFactory = new
GlobalResourceIdFactoryProvider(ncServiceContext, getResourceIdBlockSize())
+ .createResourceIdFactory();
persistedResourceRegistry =
ncServiceContext.getPersistedResourceRegistry();
cacheManager = new CacheManager();
}
@@ -677,4 +678,9 @@ public class NCAppRuntimeContext implements
INcApplicationContext {
public IPartitionBootstrapper getPartitionBootstrapper() {
return partitionBootstrapper;
}
+
+ private int getResourceIdBlockSize() {
+ return isCloudDeployment() ?
storageProperties.getStoragePartitionsCount()
+ : ncServiceContext.getIoManager().getIODevices().size();
+ }
}
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/transaction/GlobalResourceIdFactory.java
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/transaction/GlobalResourceIdFactory.java
index 908663fe63..30877d95b2 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/transaction/GlobalResourceIdFactory.java
+++
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/transaction/GlobalResourceIdFactory.java
@@ -41,21 +41,23 @@ import it.unimi.dsi.fastutil.longs.LongPriorityQueues;
public class GlobalResourceIdFactory implements IResourceIdFactory {
private static final Logger LOGGER = LogManager.getLogger();
- private static final int RESOURCE_ID_INITIAL_BLOCK_SIZE = 24;
- private static final int MAX_BLOCK_SIZE = 35;
private final INCServiceContext serviceCtx;
- private final LongPriorityQueue resourceIds =
- LongPriorityQueues.synchronize(new
LongArrayFIFOQueue(RESOURCE_ID_INITIAL_BLOCK_SIZE));
+ private final LongPriorityQueue resourceIds;
private final LinkedBlockingQueue<ResourceIdRequestResponseMessage>
resourceIdResponseQ;
private final String nodeId;
- private volatile boolean reset = false;
+ private final int initialBlockSize;
+ private final int maxBlockSize;
private int currentBlockSize;
+ private volatile boolean reset = false;
- public GlobalResourceIdFactory(INCServiceContext serviceCtx) {
+ public GlobalResourceIdFactory(INCServiceContext serviceCtx, int
initialBlockSize) {
this.serviceCtx = serviceCtx;
this.resourceIdResponseQ = new LinkedBlockingQueue<>();
this.nodeId = serviceCtx.getNodeId();
- this.currentBlockSize = RESOURCE_ID_INITIAL_BLOCK_SIZE;
+ this.initialBlockSize = initialBlockSize;
+ maxBlockSize = initialBlockSize * 2;
+ currentBlockSize = initialBlockSize;
+ resourceIds = LongPriorityQueues.synchronize(new
LongArrayFIFOQueue(initialBlockSize));
}
public synchronized void addNewIds(ResourceIdRequestResponseMessage
resourceIdResponse)
@@ -120,8 +122,8 @@ public class GlobalResourceIdFactory implements
IResourceIdFactory {
public synchronized void reset() {
reset = true;
currentBlockSize += 1;
- if (currentBlockSize > MAX_BLOCK_SIZE) {
- currentBlockSize = RESOURCE_ID_INITIAL_BLOCK_SIZE;
+ if (currentBlockSize > maxBlockSize) {
+ currentBlockSize = initialBlockSize;
}
LOGGER.debug("current resource ids block size: {}", currentBlockSize);
}
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/transaction/GlobalResourceIdFactoryProvider.java
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/transaction/GlobalResourceIdFactoryProvider.java
index d99ea90d52..aaeeb20be9 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/transaction/GlobalResourceIdFactoryProvider.java
+++
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/transaction/GlobalResourceIdFactoryProvider.java
@@ -23,12 +23,14 @@ import org.apache.hyracks.api.application.INCServiceContext;
public class GlobalResourceIdFactoryProvider {
private final INCServiceContext ncServiceCtx;
+ private final int initialBlockSize;
- public GlobalResourceIdFactoryProvider(INCServiceContext ncServiceCtx) {
+ public GlobalResourceIdFactoryProvider(INCServiceContext ncServiceCtx, int
initialBlockSize) {
this.ncServiceCtx = ncServiceCtx;
+ this.initialBlockSize = initialBlockSize;
}
public GlobalResourceIdFactory createResourceIdFactory() {
- return new GlobalResourceIdFactory(ncServiceCtx);
+ return new GlobalResourceIdFactory(ncServiceCtx, initialBlockSize);
}
}
