Yingyi Bu has submitted this change and it was merged. Change subject: Automatically set parameters. ......................................................................
Automatically set parameters. The parameters include: -- storage.buffercache.size -- storage.memorycomponent.numpages -- storage.memorycomponent.globalbudget -- -Xmx for NCDriver Change-Id: I02c233b86b10a998482b8d44123d45b53ebf14dc Reviewed-on: https://asterix-gerrit.ics.uci.edu/1417 Reviewed-by: Till Westmann <[email protected]> Tested-by: Jenkins <[email protected]> BAD: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> --- M asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml M asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml M asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml A asterixdb/asterix-app/src/main/resources/asterix-build-configuration4.xml A asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml M hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java A hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/test/java/org/apache/hyracks/control/nc/service/NCServiceTest.java 9 files changed, 319 insertions(+), 18 deletions(-) Approvals: Till Westmann: Looks good to me, approved Jenkins: Verified; No violations found; Verified Objections: Jenkins: Violations found diff --git a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml index 8a96882..b7103dd 100644 --- a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml +++ b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml @@ -77,8 +77,7 @@ <name>storage.buffercache.size</name> <value>48MB</value> <description>The size of memory allocated to the disk buffer cache. - The value should be a multiple of the buffer cache page size - (Default = "512MB") + The value should be a multiple of the buffer cache page size. </description> </property> <property> @@ -88,7 +87,14 @@ This budget is shared by all the memory components of the primary index and all its secondary indexes across all I/O devices on a node. Note: in-memory components usually has fill factor of 75% since - the pages are 75% full and the remaining 25% is un-utilized. (Default = 256) + the pages are 75% full and the remaining 25% is un-utilized. + </description> + </property> + <property> + <name>storage.memorycomponent.globalbudget</name> + <value>512MB</value> + <description>The size of memory allocated to the memory components. + The value should be a multiple of the memory component page size. </description> </property> <property> diff --git a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml index 2989fa9..c5b8470 100644 --- a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml +++ b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml @@ -81,8 +81,7 @@ <name>storage.buffercache.size</name> <value>48MB</value> <description>The size of memory allocated to the disk buffer cache. - The value should be a multiple of the buffer cache page size - (Default = "512MB") + The value should be a multiple of the buffer cache page size. </description> </property> <property> @@ -92,7 +91,14 @@ This budget is shared by all the memory components of the primary index and all its secondary indexes across all I/O devices on a node. Note: in-memory components usually has fill factor of 75% since - the pages are 75% full and the remaining 25% is un-utilized. (Default = 256) + the pages are 75% full and the remaining 25% is un-utilized. + </description> + </property> + <property> + <name>storage.memorycomponent.globalbudget</name> + <value>512MB</value> + <description>The size of memory allocated to the memory components. + The value should be a multiple of the memory component page size. </description> </property> <property> diff --git a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml index 85881f9..73a8ea8 100644 --- a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml +++ b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml @@ -81,8 +81,7 @@ <name>storage.buffercache.size</name> <value>48MB</value> <description>The size of memory allocated to the disk buffer cache. - The value should be a multiple of the buffer cache page size - (Default = "512MB") + The value should be a multiple of the buffer cache page size. </description> </property> <property> @@ -92,7 +91,14 @@ This budget is shared by all the memory components of the primary index and all its secondary indexes across all I/O devices on a node. Note: in-memory components usually has fill factor of 75% since - the pages are 75% full and the remaining 25% is un-utilized. (Default = 256) + the pages are 75% full and the remaining 25% is un-utilized. + </description> + </property> + <property> + <name>storage.memorycomponent.globalbudget</name> + <value>512MB</value> + <description>The size of memory allocated to the memory components. + The value should be a multiple of the memory component page size. </description> </property> <property> diff --git a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration4.xml b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration4.xml new file mode 100644 index 0000000..f7bb7fc --- /dev/null +++ b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration4.xml @@ -0,0 +1,98 @@ +<!-- + ! Licensed to the Apache Software Foundation (ASF) under one + ! or more contributor license agreements. See the NOTICE file + ! distributed with this work for additional information + ! regarding copyright ownership. The ASF licenses this file + ! to you under the Apache License, Version 2.0 (the + ! "License"); you may not use this file except in compliance + ! with the License. You may obtain a copy of the License at + ! + ! http://www.apache.org/licenses/LICENSE-2.0 + ! + ! Unless required by applicable law or agreed to in writing, + ! software distributed under the License is distributed on an + ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ! KIND, either express or implied. See the License for the + ! specific language governing permissions and limitations + ! under the License. + !--> +<asterixConfiguration xmlns="asterixconf"> + <metadataNode>asterix_nc1</metadataNode> + <store> + <ncId>asterix_nc1</ncId> + <storeDirs>iodevice0,iodevice1</storeDirs> + </store> + <store> + <ncId>asterix_nc2</ncId> + <storeDirs>iodevice0,iodevice1</storeDirs> + </store> + <transactionLogDir> + <ncId>asterix_nc1</ncId> + <txnLogDirPath>target/txnLogDir/asterix_nc1</txnLogDirPath> + </transactionLogDir> + <transactionLogDir> + <ncId>asterix_nc2</ncId> + <txnLogDirPath>target/txnLogDir/asterix_nc2</txnLogDirPath> + </transactionLogDir> + + <property> + <name>max.wait.active.cluster</name> + <value>60</value> + <description>Maximum wait (in seconds) for a cluster to be ACTIVE (all + nodes are available) + before a submitted query/statement can be + executed. (Default = 60 seconds) + </description> + </property> + + <property> + <name>log.level</name> + <value>INFO</value> + <description>Log level for running tests/build</description> + </property> + <property> + <name>compiler.framesize</name> + <value>32KB</value> + </property> + <property> + <name>compiler.sortmemory</name> + <value>320KB</value> + </property> + <property> + <name>compiler.groupmemory</name> + <value>160KB</value> + </property> + <property> + <name>compiler.joinmemory</name> + <value>256KB</value> + </property> + <property> + <name>compiler.parallelism</name> + <value>-1</value> + </property> + <property> + <name>storage.buffercache.pagesize</name> + <value>32KB</value> + <description>The page size in bytes for pages in the buffer cache. + (Default = "128KB") + </description> + </property> + <property> + <name>plot.activate</name> + <value>false</value> + <description>Enabling plot of Algebricks plan to tmp folder. (Default = false) + </description> + </property> + <property> + <name>messaging.frame.size</name> + <value>4096</value> + <description>The frame size to be used for NC to NC messaging. (Default = 4kb) + </description> + </property> + <property> + <name>messaging.frame.count</name> + <value>512</value> + <description>Number of reusable frames for NC to NC messaging. (Default = 512) + </description> + </property> +</asterixConfiguration> diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java new file mode 100644 index 0000000..3defc40 --- /dev/null +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.test.runtime; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Collection; + +import org.apache.asterix.common.config.StorageProperties; +import org.apache.asterix.testframework.context.TestCaseContext; +import org.json.JSONObject; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; +import org.mortbay.util.ajax.JSON; + +/** + * Runs the cluster state runtime tests with the storage parallelism. + */ +@RunWith(Parameterized.class) +public class ClusterStateDefaultParameterTest { + protected static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration4.xml"; + + @BeforeClass + public static void setUp() throws Exception { + LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME); + } + + @AfterClass + public static void tearDown() throws Exception { + LangExecutionUtil.tearDown(); + } + + @Parameters(name = "ClusterStateExecutionTest {index}: {0}") + public static Collection<Object[]> tests() throws Exception { + return LangExecutionUtil.tests("only_cluster_state.xml", "cluster_state.xml"); + } + + protected TestCaseContext tcCtx; + + public ClusterStateDefaultParameterTest(TestCaseContext tcCtx) { + this.tcCtx = tcCtx; + } + + @Test + public void test() throws Exception { + StringBuilder result = new StringBuilder(); + URL url = new URL("http://localhost:19002/admin/cluster";); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream())); + String line; + while ((line = rd.readLine()) != null) { + result.append(line); + } + rd.close(); + long maxHeap = Runtime.getRuntime().maxMemory(); + int matchCount = 0; + String[] rows = result.toString().split(","); + for (String row : rows) { + if (row.contains("storage.buffercache.size")) { + Assert.assertTrue(getValue(row) == maxHeap / 4); + matchCount++; + } + if (row.contains("storage.memorycomponent.globalbudget")) { + Assert.assertTrue(getValue(row) == maxHeap / 4); + matchCount++; + } + if (row.contains("storage.memorycomponent.numpages")) { + Assert.assertTrue(getValue(row) == maxHeap / (131072 * 64)); + matchCount++; + } + } + Assert.assertTrue(matchCount == 3); + } + + // Parses a long value parameter. + private long getValue(String row) { + String valueStr = row.split(":")[1].trim(); + return Long.parseLong(valueStr); + } +} diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java index b86d961..fbaf0a7 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java @@ -22,7 +22,6 @@ import org.apache.hyracks.util.StorageUtil; import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE; -import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE; public class StorageProperties extends AbstractProperties { @@ -30,7 +29,6 @@ private static final int STORAGE_BUFFERCACHE_PAGESIZE_DEFAULT = StorageUtil.getSizeInBytes(128, KILOBYTE); private static final String STORAGE_BUFFERCACHE_SIZE_KEY = "storage.buffercache.size"; - private static final long STORAGE_BUFFERCACHE_SIZE_DEFAULT = StorageUtil.getSizeInBytes(512, MEGABYTE); private static final String STORAGE_BUFFERCACHE_MAXOPENFILES_KEY = "storage.buffercache.maxopenfiles"; private static final int STORAGE_BUFFERCACHE_MAXOPENFILES_DEFAULT = Integer.MAX_VALUE; @@ -39,7 +37,6 @@ private static final int STORAGE_MEMORYCOMPONENT_PAGESIZE_DEFAULT = StorageUtil.getSizeInBytes(128, KILOBYTE); private static final String STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY = "storage.memorycomponent.numpages"; - private static final int STORAGE_MEMORYCOMPONENT_NUMPAGES_DEFAULT = 256; // ... so 32MB components private static final String STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY = "storage.metadata.memorycomponent.numpages"; @@ -49,14 +46,28 @@ private static final int STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_DEFAULT = 2; // 2 components private static final String STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY = "storage.memorycomponent.globalbudget"; - private static final long STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT = StorageUtil.getSizeInBytes(512, MEGABYTE); private static final String STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE_KEY = "storage.lsm.bloomfilter.falsepositiverate"; private static final double STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE_DEFAULT = 0.01; + private final long storageBufferCacheSizeDefault; + private final int storageMemoryComponentNumPages; + private final long storageMemorycomponentGlobalbudgetDefault; + public StorageProperties(PropertiesAccessor accessor) { super(accessor); + + // Gets the -Xmx value for the JVM. + long maxHeapSize = Runtime.getRuntime().maxMemory(); + // By default, uses 1/4 of the maximum heap size for read cache, i.e., disk buffer cache. + storageBufferCacheSizeDefault = maxHeapSize / 4; + // By default, uses 1/4 of the maximum heap size for the write buffer, i.e., globalbudget for memory components. + storageMemorycomponentGlobalbudgetDefault = maxHeapSize / 4; + // By default, uses 1/16 of the storageMemorycomponentGlobalbudgetDefault for the write buffer budget + // for a dataset, including data and indexes. + storageMemoryComponentNumPages = (int) storageMemorycomponentGlobalbudgetDefault + / (16 * getMemoryComponentPageSize()); } @PropertyKey(STORAGE_BUFFERCACHE_PAGESIZE_KEY) @@ -67,7 +78,7 @@ @PropertyKey(STORAGE_BUFFERCACHE_SIZE_KEY) public long getBufferCacheSize() { - return accessor.getProperty(STORAGE_BUFFERCACHE_SIZE_KEY, STORAGE_BUFFERCACHE_SIZE_DEFAULT, + return accessor.getProperty(STORAGE_BUFFERCACHE_SIZE_KEY, storageBufferCacheSizeDefault, PropertyInterpreters.getLongBytePropertyInterpreter()); } @@ -89,7 +100,7 @@ @PropertyKey(STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY) public int getMemoryComponentNumPages() { - return accessor.getProperty(STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY, STORAGE_MEMORYCOMPONENT_NUMPAGES_DEFAULT, + return accessor.getProperty(STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY, storageMemoryComponentNumPages, PropertyInterpreters.getIntegerPropertyInterpreter()); } @@ -108,8 +119,8 @@ @PropertyKey(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY) public long getMemoryComponentGlobalBudget() { - return accessor.getProperty(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY, - STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT, PropertyInterpreters.getLongBytePropertyInterpreter()); + return accessor.getProperty(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY, storageMemorycomponentGlobalbudgetDefault, + PropertyInterpreters.getLongBytePropertyInterpreter()); } @PropertyKey(STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE_KEY) diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml index dfa1807..2f966a3 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml @@ -65,6 +65,17 @@ <artifactId>commons-lang3</artifactId> <version>3.5</version> </dependency> + <dependency> + <groupId>com.e-movimento.tinytools</groupId> + <artifactId>privilegedaccessor</artifactId> + <version>1.2.2</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> </dependencies> </project> diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java index 8ca61ad..5a03d3c 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java @@ -23,6 +23,8 @@ import java.io.InputStream; import java.io.ObjectInputStream; import java.io.StringReader; +import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; import java.net.InetAddress; import java.net.ServerSocket; import java.net.Socket; @@ -72,6 +74,11 @@ */ private static Process proc = null; + /** + * The management bean for obtaining settings of the underlying operating system and hardware. + */ + private static OperatingSystemMXBean osMXBean; + private static List<String> buildCommand() throws IOException { List<String> cList = new ArrayList<>(); @@ -114,7 +121,8 @@ LOGGER.info("Using JAVA_OPTS from environment"); } else { LOGGER.info("Using default JAVA_OPTS"); - jvmargs = "-Xmx1536m"; + long ramSize = ((com.sun.management.OperatingSystemMXBean) osMXBean).getTotalPhysicalMemorySize(); + jvmargs = "-Xmx" + (int) Math.ceil(0.6 * ramSize / (1024 * 1024)) + "m"; } } env.put("JAVA_OPTS", jvmargs); @@ -243,6 +251,9 @@ } config.loadConfigAndApplyDefaults(); + // Initializes the oxMXBean. + osMXBean = ManagementFactory.getOperatingSystemMXBean(); + // For now we implement a trivial listener which just // accepts an IP/port combination from the CC. This could // be made more advanced in several ways depending on whether diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/test/java/org/apache/hyracks/control/nc/service/NCServiceTest.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/test/java/org/apache/hyracks/control/nc/service/NCServiceTest.java new file mode 100644 index 0000000..00d7ca4 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/test/java/org/apache/hyracks/control/nc/service/NCServiceTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.control.nc.service; + +import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; +import java.util.HashMap; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; + +import junit.extensions.PA; + +public class NCServiceTest { + + @Test + public void testJvmArgs() { + Map<String, String> configuration = new HashMap<>(); + OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean(); + PA.setValue(NCService.class, "nodeSection", "nc/nc1"); + PA.setValue(NCService.class, "osMXBean", osMXBean); + PA.invokeMethod(NCService.class, "configEnvironment(java.util.Map)", configuration); + String javaOpts = configuration.get("JAVA_OPTS"); + String prefix = javaOpts.substring(4); + String sizeStr = prefix.substring(0, prefix.length() - 1); + int size = Integer.parseInt(sizeStr); + long ramSize = ((com.sun.management.OperatingSystemMXBean) osMXBean).getTotalPhysicalMemorySize(); + int base = 1024 * 1024 * 5; + Assert.assertTrue(size == ramSize * 3 / base + ((ramSize * 3) % base == 0 ? 0 : 1)); + } +} -- To view, visit https://asterix-gerrit.ics.uci.edu/1417 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I02c233b86b10a998482b8d44123d45b53ebf14dc Gerrit-PatchSet: 8 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Yingyi Bu <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: Yingyi Bu <[email protected]>
