Yingyi Bu has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1417
Change subject: Automatically set parameters.
......................................................................
Automatically set parameters.
The parameters include:
-- storage.buffercache.size
-- storage.memorycomponent.numpages
-- storage.memorycomponent.globalbudget
Change-Id: I02c233b86b10a998482b8d44123d45b53ebf14dc
---
M asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
M asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml
M asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
M
hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java
5 files changed, 57 insertions(+), 13 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/17/1417/1
diff --git
a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
index 8a96882..b7103dd 100644
--- a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
+++ b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
@@ -77,8 +77,7 @@
<name>storage.buffercache.size</name>
<value>48MB</value>
<description>The size of memory allocated to the disk buffer cache.
- The value should be a multiple of the buffer cache page size
- (Default = "512MB")
+ The value should be a multiple of the buffer cache page size.
</description>
</property>
<property>
@@ -88,7 +87,14 @@
This budget is shared by all the memory components of the primary
index and all its secondary indexes across all I/O devices on a node.
Note: in-memory components usually has fill factor of 75% since
- the pages are 75% full and the remaining 25% is un-utilized. (Default =
256)
+ the pages are 75% full and the remaining 25% is un-utilized.
+ </description>
+ </property>
+ <property>
+ <name>storage.memorycomponent.globalbudget</name>
+ <value>512MB</value>
+ <description>The size of memory allocated to the memory components.
+ The value should be a multiple of the memory component page size.
</description>
</property>
<property>
diff --git
a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml
b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml
index 2989fa9..c5b8470 100644
--- a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml
+++ b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml
@@ -81,8 +81,7 @@
<name>storage.buffercache.size</name>
<value>48MB</value>
<description>The size of memory allocated to the disk buffer cache.
- The value should be a multiple of the buffer cache page size
- (Default = "512MB")
+ The value should be a multiple of the buffer cache page size.
</description>
</property>
<property>
@@ -92,7 +91,14 @@
This budget is shared by all the memory components of the primary
index and all its secondary indexes across all I/O devices on a
node.
Note: in-memory components usually has fill factor of 75% since
- the pages are 75% full and the remaining 25% is un-utilized.
(Default = 256)
+ the pages are 75% full and the remaining 25% is un-utilized.
+ </description>
+ </property>
+ <property>
+ <name>storage.memorycomponent.globalbudget</name>
+ <value>512MB</value>
+ <description>The size of memory allocated to the memory components.
+ The value should be a multiple of the memory component page size.
</description>
</property>
<property>
diff --git
a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml
b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml
index 85881f9..73a8ea8 100644
--- a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml
+++ b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml
@@ -81,8 +81,7 @@
<name>storage.buffercache.size</name>
<value>48MB</value>
<description>The size of memory allocated to the disk buffer cache.
- The value should be a multiple of the buffer cache page size
- (Default = "512MB")
+ The value should be a multiple of the buffer cache page size.
</description>
</property>
<property>
@@ -92,7 +91,14 @@
This budget is shared by all the memory components of the primary
index and all its secondary indexes across all I/O devices on a
node.
Note: in-memory components usually has fill factor of 75% since
- the pages are 75% full and the remaining 25% is un-utilized.
(Default = 256)
+ the pages are 75% full and the remaining 25% is un-utilized.
+ </description>
+ </property>
+ <property>
+ <name>storage.memorycomponent.globalbudget</name>
+ <value>512MB</value>
+ <description>The size of memory allocated to the memory components.
+ The value should be a multiple of the memory component page size.
</description>
</property>
<property>
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index b86d961..029be98 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -30,7 +30,7 @@
private static final int STORAGE_BUFFERCACHE_PAGESIZE_DEFAULT =
StorageUtil.getSizeInBytes(128, KILOBYTE);
private static final String STORAGE_BUFFERCACHE_SIZE_KEY =
"storage.buffercache.size";
- private static final long STORAGE_BUFFERCACHE_SIZE_DEFAULT =
StorageUtil.getSizeInBytes(512, MEGABYTE);
+ private static final long STORAGE_BUFFERCACHE_SIZE_DEFAULT;
private static final String STORAGE_BUFFERCACHE_MAXOPENFILES_KEY =
"storage.buffercache.maxopenfiles";
private static final int STORAGE_BUFFERCACHE_MAXOPENFILES_DEFAULT =
Integer.MAX_VALUE;
@@ -39,7 +39,7 @@
private static final int STORAGE_MEMORYCOMPONENT_PAGESIZE_DEFAULT =
StorageUtil.getSizeInBytes(128, KILOBYTE);
private static final String STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY =
"storage.memorycomponent.numpages";
- private static final int STORAGE_MEMORYCOMPONENT_NUMPAGES_DEFAULT = 256;
// ... so 32MB components
+ private static int STORAGE_MEMORYCOMPONENT_NUMPAGES_DEFAULT; //
private static final String STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY =
"storage.metadata.memorycomponent.numpages";
@@ -49,14 +49,29 @@
private static final int STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_DEFAULT =
2; // 2 components
private static final String STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY =
"storage.memorycomponent.globalbudget";
- private static final long STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT =
StorageUtil.getSizeInBytes(512, MEGABYTE);
+ private static final long STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT;
private static final String STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE_KEY =
"storage.lsm.bloomfilter.falsepositiverate";
private static final double
STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE_DEFAULT = 0.01;
+ static {
+ // Gets the -Xmx value for the JVM.
+ long maxHeapSize = Runtime.getRuntime().maxMemory();
+
+ // By default, uses 1/4 of the maximum heap size for read cache, i.e.,
disk buffer cache.
+ STORAGE_BUFFERCACHE_SIZE_DEFAULT = maxHeapSize / 4;
+ // By default, uses 1/4 of the maximum heap size for the write buffer,
i.e., globalbudget for memory components.
+ STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT = maxHeapSize / 4;
+ }
+
public StorageProperties(PropertiesAccessor accessor) {
super(accessor);
+
+ // By default, uses 1/16 of the
STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT for the write buffer budget
+ // for a dataset, including data and indexes.
+ STORAGE_MEMORYCOMPONENT_NUMPAGES_DEFAULT = (int)
STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT
+ / (16 * getMemoryComponentPageSize());
}
@PropertyKey(STORAGE_BUFFERCACHE_PAGESIZE_KEY)
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java
index 8ca61ad..fe3cb2d 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java
@@ -23,6 +23,8 @@
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.StringReader;
+import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
import java.net.InetAddress;
import java.net.ServerSocket;
import java.net.Socket;
@@ -72,6 +74,11 @@
*/
private static Process proc = null;
+ /**
+ * The management bean for obtaining settings of the underlying operating
system and hardware.
+ */
+ private static OperatingSystemMXBean osMXBean;
+
private static List<String> buildCommand() throws IOException {
List<String> cList = new ArrayList<>();
@@ -114,7 +121,8 @@
LOGGER.info("Using JAVA_OPTS from environment");
} else {
LOGGER.info("Using default JAVA_OPTS");
- jvmargs = "-Xmx1536m";
+ long ramSize = ((com.sun.management.OperatingSystemMXBean)
osMXBean).getTotalPhysicalMemorySize();
+ jvmargs = "-Xmx" + (0.6 * ramSize / (1024 * 1024)) + "m";
}
}
env.put("JAVA_OPTS", jvmargs);
@@ -243,6 +251,9 @@
}
config.loadConfigAndApplyDefaults();
+ // Initializes the oxMXBean.
+ osMXBean = ManagementFactory.getOperatingSystemMXBean();
+
// For now we implement a trivial listener which just
// accepts an IP/port combination from the CC. This could
// be made more advanced in several ways depending on whether
--
To view, visit https://asterix-gerrit.ics.uci.edu/1417
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I02c233b86b10a998482b8d44123d45b53ebf14dc
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <[email protected]>
