Murtadha Hubail has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/2362
Change subject: [ASTERIXDB-2276][CONF] Introduce Max Active Writable Datasets
......................................................................
[ASTERIXDB-2276][CONF] Introduce Max Active Writable Datasets
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Introduce number of metadata datasets parameter
and use it to reserve memory for all metadata
datasets.
- Default metadata dataset memory component number
of pages to 32 pages.
- Introduce max active writable datasets and use
it to automatically calculate the number of pages
of a user dataset memory component.
- Remve the assumpation of reserving an NC core
for heartbeats.
- Validate memory configuration early.
Change-Id: I9909c26b1e12b431f913e201d2c3d83769be7269
---
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
M asterixdb/asterix-app/src/main/resources/cc.conf
M asterixdb/asterix-app/src/main/resources/cc2.conf
M asterixdb/asterix-app/src/main/resources/cc3.conf
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
M asterixdb/asterix-app/src/test/resources/cc-multipart.conf
M asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
M asterixdb/asterix-app/src/test/resources/cc.conf
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
10 files changed, 51 insertions(+), 54 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/62/2362/1
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index a23a763..d33ee42 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -224,19 +224,9 @@
@Override
public NodeCapacity getCapacity() {
StorageProperties storageProperties =
runtimeContext.getStorageProperties();
- // Deducts the reserved buffer cache size and memory component size
from the maxium heap size,
- // and deducts one core for processing heartbeats.
- long memorySize = Runtime.getRuntime().maxMemory() -
storageProperties.getBufferCacheSize()
- - storageProperties.getMemoryComponentGlobalBudget();
- if (memorySize <= 0) {
- throw new IllegalStateException("Invalid node memory
configuration, more memory budgeted than available "
- + "in JVM. Runtime max memory: " +
Runtime.getRuntime().maxMemory() + " Buffer cache size: "
- + storageProperties.getBufferCacheSize() + " Memory
component global budget: "
- + storageProperties.getMemoryComponentGlobalBudget());
- }
+ final long memorySize =
storageProperties.getJobExecutionMemoryBudget();
int allCores = Runtime.getRuntime().availableProcessors();
- int maximumCoresForComputation = allCores > 1 ? allCores - 1 :
allCores;
- return new NodeCapacity(memorySize, maximumCoresForComputation);
+ return new NodeCapacity(memorySize, allCores);
}
private void performLocalCleanUp() throws HyracksDataException {
diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf
b/asterixdb/asterix-app/src/main/resources/cc.conf
index c5a3fdb..deebe41 100644
--- a/asterixdb/asterix-app/src/main/resources/cc.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc.conf
@@ -37,7 +37,6 @@
jvm.args=-Xmx4096m
-Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.pagesize=32KB
storage.buffercache.size=48MB
-storage.memorycomponent.numpages=16
storage.memorycomponent.globalbudget=512MB
[cc]
diff --git a/asterixdb/asterix-app/src/main/resources/cc2.conf
b/asterixdb/asterix-app/src/main/resources/cc2.conf
index 941b6c1..2026ee7 100644
--- a/asterixdb/asterix-app/src/main/resources/cc2.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc2.conf
@@ -37,7 +37,6 @@
jvm.args=-Xmx4096m
-Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.pagesize=32KB
storage.buffercache.size=48MB
-storage.memorycomponent.numpages=16
storage.memorycomponent.globalbudget=512MB
[cc]
diff --git a/asterixdb/asterix-app/src/main/resources/cc3.conf
b/asterixdb/asterix-app/src/main/resources/cc3.conf
index 01383ef..91ddb7a 100644
--- a/asterixdb/asterix-app/src/main/resources/cc3.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc3.conf
@@ -37,7 +37,6 @@
jvm.args=-Xmx4096m
-Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.pagesize=32KB
storage.buffercache.size=48MB
-storage.memorycomponent.numpages=16
storage.memorycomponent.globalbudget=512MB
[cc]
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java
index f029669..e752493 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java
@@ -87,8 +87,8 @@
Assert.assertTrue(getValue(row) == maxHeap / 4);
matchCount++;
}
- if (row.contains("storage.memorycomponent.numpages")) {
- Assert.assertTrue(getValue(row) == maxHeap / (131072 * 64));
+ if (row.contains("storage.max.active.writable.datasets")) {
+ Assert.assertTrue(getValue(row) == 8);
matchCount++;
}
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
index 92e403c..05e5aad 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
@@ -48,10 +48,7 @@
@Before
public void setUp() throws Exception {
- // Read default test configurations
- // Write test config file
-
integrationUtil.addOption(StorageProperties.Option.STORAGE_MEMORYCOMPONENT_GLOBALBUDGET,
"128MB");
-
integrationUtil.addOption(StorageProperties.Option.STORAGE_MEMORYCOMPONENT_NUMPAGES,
32);
+
integrationUtil.addOption(StorageProperties.Option.STORAGE_MAX_ACTIVE_WRITABLE_DATASETS,
20);
integrationUtil.setGracefulShutdown(false);
integrationUtil.init(true, TEST_CONFIG_FILE_PATH);
}
diff --git a/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
b/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
index f28545d..5cf1bbe 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
@@ -38,7 +38,6 @@
storage.subdir=test_storage
storage.buffercache.pagesize=32KB
storage.buffercache.size=48MB
-storage.memorycomponent.numpages=32
storage.memorycomponent.globalbudget=512MB
[cc]
diff --git
a/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
b/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
index 0b66cfe..0b2ddbd 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
@@ -35,8 +35,8 @@
jvm.args=-Xmx4096m
-Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.pagesize=32KB
storage.buffercache.size=48MB
-storage.memorycomponent.numpages=16
-storage.memorycomponent.globalbudget=512MB
+storage.max.active.writable.datasets=50
+storage.memorycomponent.globalbudget=128MB
[cc]
address = 127.0.0.1
diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf
b/asterixdb/asterix-app/src/test/resources/cc.conf
index 3e4ecd9..5cf1bbe 100644
--- a/asterixdb/asterix-app/src/test/resources/cc.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc.conf
@@ -38,7 +38,6 @@
storage.subdir=test_storage
storage.buffercache.pagesize=32KB
storage.buffercache.size=48MB
-storage.memorycomponent.numpages=16
storage.memorycomponent.globalbudget=512MB
[cc]
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index 963f0ca..23e7d75 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -42,16 +42,11 @@
STORAGE_BUFFERCACHE_MAXOPENFILES(INTEGER, Integer.MAX_VALUE),
STORAGE_MEMORYCOMPONENT_GLOBALBUDGET(LONG_BYTE_UNIT,
Runtime.getRuntime().maxMemory() / 4),
STORAGE_MEMORYCOMPONENT_PAGESIZE(INTEGER_BYTE_UNIT,
StorageUtil.getIntSizeInBytes(128, KILOBYTE)),
- STORAGE_MEMORYCOMPONENT_NUMPAGES(INTEGER,
(Function<IApplicationConfig, Integer>) accessor ->
- // By default, uses 1/16 of the STORAGE_MEMORYCOMPONENT_GLOBALBUDGET
for the write buffer
- // budget for a dataset, including data and indexes.
- (int) (accessor.getLong(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET) / (16 *
accessor.getInt(STORAGE_MEMORYCOMPONENT_PAGESIZE)))),
STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS(INTEGER, 2),
- STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES(INTEGER,
(Function<IApplicationConfig, Integer>) accessor ->
- // By default, uses the min of 1/64 of the
STORAGE_MEMORYCOMPONENT_GLOBALBUDGET and 256 pages
- // for the write buffer budget for a metadata dataset, including data
and indexes.
- Math.min((int) (accessor.getLong(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET)
/ (64 * accessor.getInt(STORAGE_MEMORYCOMPONENT_PAGESIZE))), 256)),
- STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE(DOUBLE, 0.01d);
+ STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES(INTEGER, 32),
+ STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE(DOUBLE, 0.01d),
+ STORAGE_METADATA_DATASETS(INTEGER, 14),
+ STORAGE_MAX_ACTIVE_WRITABLE_DATASETS(INTEGER, 8);
private final IOptionType interpreter;
private final Object defaultValue;
@@ -59,11 +54,6 @@
<T> Option(IOptionType<T> interpreter, T defaultValue) {
this.interpreter = interpreter;
this.defaultValue = defaultValue;
- }
-
- <T> Option(IOptionType<T> interpreter, Function<IApplicationConfig, T>
defaultValueFunction) {
- this.interpreter = interpreter;
- this.defaultValue = defaultValueFunction;
}
@Override
@@ -86,17 +76,16 @@
+ "of the memory component page size";
case STORAGE_MEMORYCOMPONENT_PAGESIZE:
return "The page size in bytes for pages allocated to
memory components";
- case STORAGE_MEMORYCOMPONENT_NUMPAGES:
- return "The number of pages to allocate for a memory
component. This budget is shared by all "
- + "the memory components of the primary index and
all its secondary indexes across all I/O "
- + "devices on a node. Note: in-memory components
usually has fill factor of 75% since "
- + "the pages are 75% full and the remaining 25% is
un-utilized";
case STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS:
return "The number of memory components to be used per lsm
index";
case STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES:
return "The number of pages to allocate for a metadata
memory component";
case STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE:
return "The maximum acceptable false positive rate for
bloom filters associated with LSM indexes";
+ case STORAGE_METADATA_DATASETS:
+ return "The number of metadata datasets";
+ case STORAGE_MAX_ACTIVE_WRITABLE_DATASETS:
+ return "The maximum number of datasets that can be
concurrently modified";
default:
throw new IllegalStateException("NYI: " + this);
}
@@ -114,21 +103,16 @@
@Override
public String usageDefaultOverride(IApplicationConfig accessor,
Function<IOption, String> optionPrinter) {
- switch (this) {
- case STORAGE_MEMORYCOMPONENT_NUMPAGES:
- return "1/16th of the " +
optionPrinter.apply(Option.STORAGE_MEMORYCOMPONENT_GLOBALBUDGET)
- + " value";
- case STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES:
- return "1/64th of the " +
optionPrinter.apply(Option.STORAGE_MEMORYCOMPONENT_GLOBALBUDGET)
- + " value or 256, whichever is larger";
- default:
- return null;
+ if (this == STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES) {
+ return "32 pages";
}
+ return null;
}
}
public StorageProperties(PropertiesAccessor accessor) {
super(accessor);
+ validateMemoryConfig();
}
public int getBufferCachePageSize() {
@@ -148,7 +132,10 @@
}
public int getMemoryComponentNumPages() {
- return accessor.getInt(Option.STORAGE_MEMORYCOMPONENT_NUMPAGES);
+ final long metadataReservedMem = getMetadataReservedMemory();
+ final long globalUserDatasetMem = getMemoryComponentGlobalBudget() -
metadataReservedMem;
+ final long userDatasetMem = globalUserDatasetMem /
getMaxActiveWritableDatasets();
+ return (int) (userDatasetMem / getMemoryComponentPageSize());
}
public int getMetadataMemoryComponentNumPages() {
@@ -170,4 +157,32 @@
public int getBufferCacheNumPages() {
return (int) (getBufferCacheSize() / (getBufferCachePageSize() +
IBufferCache.RESERVED_HEADER_BYTES));
}
+
+ public long getJobExecutionMemoryBudget() {
+ return Runtime.getRuntime().maxMemory() - getBufferCacheSize() +
getMemoryComponentGlobalBudget();
+ }
+
+ public int getMaxActiveWritableDatasets() {
+ return accessor.getInt(Option.STORAGE_MAX_ACTIVE_WRITABLE_DATASETS);
+ }
+
+ public int getMetadataDatasets() {
+ return accessor.getInt(Option.STORAGE_METADATA_DATASETS);
+ }
+
+ private long getMetadataReservedMemory() {
+ return (getMetadataMemoryComponentNumPages() * (long)
getMemoryComponentPageSize()) * getMetadataDatasets();
+ }
+
+ private void validateMemoryConfig() {
+ final long remainingMemory =
+ Runtime.getRuntime().maxMemory() - getBufferCacheSize() -
getMemoryComponentGlobalBudget();
+ if (remainingMemory <= 0) {
+ final String msg = String.format(
+ "Invalid node memory configuration, more memory budgeted
than available in JVM. Runtime max memory:"
+ + " (%o), Buffer cache memory (%o), memory
component global budget (%o)",
+ Runtime.getRuntime().maxMemory(), getBufferCacheSize(),
getMemoryComponentGlobalBudget());
+ throw new IllegalStateException(msg);
+ }
+ }
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/2362
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I9909c26b1e12b431f913e201d2c3d83769be7269
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <[email protected]>
