Fixed IGNITE-2419 Ignite on YARN do not handle memory overhead. This closes #414.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1945b988 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1945b988 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1945b988 Branch: refs/heads/sql-store Commit: 1945b98849cb52fc297c726fa79a270706135581 Parents: 5969129 Author: Edouard Chevalier <[email protected]> Authored: Mon Feb 1 07:34:59 2016 +0300 Committer: Tikhonov Nikolay <[email protected]> Committed: Mon Feb 1 07:34:59 2016 +0300 ---------------------------------------------------------------------- .../apache/ignite/yarn/ApplicationMaster.java | 12 +- .../apache/ignite/yarn/ClusterProperties.java | 144 +++++++++++-------- .../yarn/IgniteApplicationMasterSelfTest.java | 52 +++++++ 3 files changed, 140 insertions(+), 68 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/1945b988/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java ---------------------------------------------------------------------- diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java index 755e4e4..b9ab02d 100644 --- a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java +++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java @@ -137,8 +137,8 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { + "cp -r ./libs/* ./ignite/*/libs/ || true && " + "./ignite/*/bin/ignite.sh " + "./ignite-config.xml" - + " -J-Xmx" + c.getResource().getMemory() + "m" - + " -J-Xms" + c.getResource().getMemory() + "m" + + " -J-Xmx" + ((int)props.memoryPerNode()) + "m" + + " -J-Xms" + ((int)props.memoryPerNode()) + "m" + IgniteYarnUtils.YARN_LOG_OUT )); @@ -178,7 +178,7 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { // Check that slave satisfies min requirements. if (cont.getResource().getVirtualCores() < props.cpusPerNode() - || cont.getResource().getMemory() < props.memoryPerNode()) { + || cont.getResource().getMemory() < props.totalMemoryPerNode()) { log.log(Level.FINE, "Container resources not sufficient requirements. Host: {0}, cpu: {1}, mem: {2}", new Object[]{cont.getNodeId().getHost(), cont.getResource().getVirtualCores(), cont.getResource().getMemory()}); @@ -291,7 +291,7 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { // Resource requirements for worker containers. Resource capability = Records.newRecord(Resource.class); - capability.setMemory((int)props.memoryPerNode()); + capability.setMemory((int)props.totalMemoryPerNode()); capability.setVirtualCores((int)props.cpusPerNode()); for (int i = 0; i < props.instances() - runningCnt; ++i) { @@ -302,7 +302,7 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { rmClient.addContainerRequest(containerAsk); log.log(Level.INFO, "Making request. Memory: {0}, cpu {1}.", - new Object[]{props.memoryPerNode(), props.cpusPerNode()}); + new Object[]{props.totalMemoryPerNode(), props.cpusPerNode()}); } } @@ -329,7 +329,7 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { private boolean checkAvailableResource() { Resource availableRes = rmClient.getAvailableResources(); - return availableRes == null || availableRes.getMemory() >= props.memoryPerNode() + return availableRes == null || availableRes.getMemory() >= props.totalMemoryPerNode() && availableRes.getVirtualCores() >= props.cpusPerNode(); } http://git-wip-us.apache.org/repos/asf/ignite/blob/1945b988/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java ---------------------------------------------------------------------- diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java index d040e9f..647aef2 100644 --- a/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java +++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java @@ -48,6 +48,12 @@ public class ClusterProperties { /** */ public static final double DEFAULT_MEM_PER_NODE = 2048; + + /** + * The minimum memory overhead: overhead is by default 0.1* MEMORY_PER_NODE, + * with a minimum of DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE. + */ + public static final double DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE = 384; /** Cluster name. */ private String clusterName = DEFAULT_CLUSTER_NAME; @@ -63,6 +69,12 @@ public class ClusterProperties { /** Memory limit. */ private double memPerNode = DEFAULT_MEM_PER_NODE; + + /** */ + public static final String IGNITE_MEMORY_OVERHEAD_PER_NODE = "IGNITE_MEMORY_OVERHEAD_PER_NODE"; + + /** Memory over head to request yarn. */ + private double memOverHeadPerNode = 0; /** */ public static final String IGNITE_NODE_COUNT = "IGNITE_NODE_COUNT"; @@ -185,7 +197,30 @@ public class ClusterProperties { } /** - * @return instance count limit. + * @return Memory overhead for requested memory. + */ + public double memoryOverHeadPerNode() { + return memOverHeadPerNode; + } + + /** + * Sets memory overhead requested to YARN. + * + * @param memOverHeadPerNode Memory over head per node. + */ + public void memoryOverHeadPerNode(double memOverHeadPerNode) { + this.memOverHeadPerNode = memOverHeadPerNode; + } + + /** + * @return Provide the total memory requested to ResourceManagers (memoryPerNode + memoryOverheadPerNode). + */ + public double totalMemoryPerNode(){ + return memoryPerNode() + memoryOverHeadPerNode(); + } + + /** + * @return Instance count limit. */ public double instances() { return nodeCnt; @@ -278,7 +313,50 @@ public class ClusterProperties { } /** - * @param config path to config file. + * Instantiate a ClusterProperties from a set of properties. + * + * @param props If {@code null} will be used system properties. + * @return Cluster properties. + */ + private static ClusterProperties fromProperties(Properties props) { + ClusterProperties prop = new ClusterProperties(); + + prop.clusterName = getStringProperty(IGNITE_CLUSTER_NAME, props, DEFAULT_CLUSTER_NAME); + + prop.cpuPerNode = getDoubleProperty(IGNITE_RUN_CPU_PER_NODE, props, DEFAULT_CPU_PER_NODE); + prop.memPerNode = getDoubleProperty(IGNITE_MEMORY_PER_NODE, props, DEFAULT_MEM_PER_NODE); + // The minimum memory overhead: overhead is by default 0.1* MEMORY_PER_NODE, + // with a minimum of DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE + prop.memOverHeadPerNode = getDoubleProperty(IGNITE_MEMORY_OVERHEAD_PER_NODE, props, + Math.max( 0.1 * prop.memPerNode, DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE)); + prop.nodeCnt = getDoubleProperty(IGNITE_NODE_COUNT, props, DEFAULT_IGNITE_NODE_COUNT); + + prop.igniteUrl = getStringProperty(IGNITE_URL, props, null); + prop.ignitePath = getStringProperty(IGNITE_PATH, props, null); + prop.licencePath = getStringProperty(LICENCE_PATH, props, null); + prop.jvmOpts = getStringProperty(IGNITE_JVM_OPTS, props, null); + prop.igniteWorkDir = getStringProperty(IGNITE_WORKING_DIR, props, DEFAULT_IGNITE_WORK_DIR); + prop.igniteLocalWorkDir = getStringProperty(IGNITE_LOCAL_WORK_DIR, props, DEFAULT_IGNITE_LOCAL_WORK_DIR); + prop.igniteReleasesDir = getStringProperty(IGNITE_RELEASES_DIR, props, DEFAULT_IGNITE_RELEASES_DIR); + prop.igniteCfg = getStringProperty(IGNITE_CONFIG_XML, props, null); + prop.userLibs = getStringProperty(IGNITE_USERS_LIBS, props, null); + + String pattern = getStringProperty(IGNITE_HOSTNAME_CONSTRAINT, props, null); + + if (pattern != null) { + try { + prop.hostnameConstraint = Pattern.compile(pattern); + } + catch (PatternSyntaxException e) { + log.log(Level.WARNING, "IGNITE_HOSTNAME_CONSTRAINT has invalid pattern. It will be ignore.", e); + } + } + + return prop; + } + + /** + * @param config Path to config file. * @return Cluster configuration. */ public static ClusterProperties from(String config) { @@ -291,36 +369,7 @@ public class ClusterProperties { props.load(new FileInputStream(config)); } - ClusterProperties prop = new ClusterProperties(); - - prop.clusterName = getStringProperty(IGNITE_CLUSTER_NAME, props, DEFAULT_CLUSTER_NAME); - - prop.cpuPerNode = getDoubleProperty(IGNITE_RUN_CPU_PER_NODE, props, DEFAULT_CPU_PER_NODE); - prop.memPerNode = getDoubleProperty(IGNITE_MEMORY_PER_NODE, props, DEFAULT_MEM_PER_NODE); - prop.nodeCnt = getDoubleProperty(IGNITE_NODE_COUNT, props, DEFAULT_IGNITE_NODE_COUNT); - - prop.igniteUrl = getStringProperty(IGNITE_URL, props, null); - prop.ignitePath = getStringProperty(IGNITE_PATH, props, null); - prop.licencePath = getStringProperty(LICENCE_PATH, props, null); - prop.jvmOpts = getStringProperty(IGNITE_JVM_OPTS, props, null); - prop.igniteWorkDir = getStringProperty(IGNITE_WORKING_DIR, props, DEFAULT_IGNITE_WORK_DIR); - prop.igniteLocalWorkDir = getStringProperty(IGNITE_LOCAL_WORK_DIR, props, DEFAULT_IGNITE_LOCAL_WORK_DIR); - prop.igniteReleasesDir = getStringProperty(IGNITE_RELEASES_DIR, props, DEFAULT_IGNITE_RELEASES_DIR); - prop.igniteCfg = getStringProperty(IGNITE_CONFIG_XML, props, null); - prop.userLibs = getStringProperty(IGNITE_USERS_LIBS, props, null); - - String pattern = getStringProperty(IGNITE_HOSTNAME_CONSTRAINT, props, null); - - if (pattern != null) { - try { - prop.hostnameConstraint = Pattern.compile(pattern); - } - catch (PatternSyntaxException e) { - log.log(Level.WARNING, "IGNITE_HOSTNAME_CONSTRAINT has invalid pattern. It will be ignore.", e); - } - } - - return prop; + return fromProperties(props); } catch (IOException e) { throw new RuntimeException(e); @@ -331,36 +380,7 @@ public class ClusterProperties { * @return Cluster configuration. */ public static ClusterProperties from() { - ClusterProperties prop = new ClusterProperties(); - - prop.clusterName = getStringProperty(IGNITE_CLUSTER_NAME, null, DEFAULT_CLUSTER_NAME); - - prop.cpuPerNode = getDoubleProperty(IGNITE_RUN_CPU_PER_NODE, null, DEFAULT_CPU_PER_NODE); - prop.memPerNode = getDoubleProperty(IGNITE_MEMORY_PER_NODE, null, DEFAULT_MEM_PER_NODE); - prop.nodeCnt = getDoubleProperty(IGNITE_NODE_COUNT, null, DEFAULT_IGNITE_NODE_COUNT); - - prop.igniteUrl = getStringProperty(IGNITE_URL, null, null); - prop.ignitePath = getStringProperty(IGNITE_PATH, null, null); - prop.licencePath = getStringProperty(LICENCE_PATH, null, null); - prop.jvmOpts = getStringProperty(IGNITE_JVM_OPTS, null, null); - prop.igniteWorkDir = getStringProperty(IGNITE_WORKING_DIR, null, DEFAULT_IGNITE_WORK_DIR); - prop.igniteLocalWorkDir = getStringProperty(IGNITE_LOCAL_WORK_DIR, null, DEFAULT_IGNITE_LOCAL_WORK_DIR); - prop.igniteReleasesDir = getStringProperty(IGNITE_RELEASES_DIR, null, DEFAULT_IGNITE_RELEASES_DIR); - prop.igniteCfg = getStringProperty(IGNITE_CONFIG_XML, null, null); - prop.userLibs = getStringProperty(IGNITE_USERS_LIBS, null, null); - - String pattern = getStringProperty(IGNITE_HOSTNAME_CONSTRAINT, null, null); - - if (pattern != null) { - try { - prop.hostnameConstraint = Pattern.compile(pattern); - } - catch (PatternSyntaxException e) { - log.log(Level.WARNING, "IGNITE_HOSTNAME_CONSTRAINT has invalid pattern. It will be ignore.", e); - } - } - - return prop; + return fromProperties(null); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/1945b988/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java b/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java index 97f6a12..1190313 100644 --- a/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java +++ b/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java @@ -103,6 +103,58 @@ public class IgniteApplicationMasterSelfTest extends TestCase { assertEquals(1024, req.getCapability().getMemory()); } } + + /** + * Tests whether memory overhead is allocated within container memory. + * + * @throws Exception If failed. + */ + public void testMemoryOverHeadAllocation() throws Exception { + appMaster.setRmClient(rmMock); + appMaster.setNmClient(new NMMock()); + + props.cpusPerNode(2); + props.memoryPerNode(1024); + props.memoryOverHeadPerNode(512); + props.instances(3); + + Thread thread = runAppMaster(appMaster); + + List<AMRMClient.ContainerRequest> contRequests = collectRequests(rmMock, 1, 1000); + + interruptedThread(thread); + + assertEquals(3, contRequests.size()); + + for (AMRMClient.ContainerRequest req : contRequests) { + assertEquals(2, req.getCapability().getVirtualCores()); + assertEquals(1024 + 512, req.getCapability().getMemory()); + } + } + + /** + * Tests whether memory overhead prevents from allocating container. + * + * @throws Exception If failed. + */ + public void testMemoryOverHeadPreventAllocation() throws Exception { + rmMock.availableRes(new MockResource(1024, 2)); + appMaster.setRmClient(rmMock); + appMaster.setNmClient(new NMMock()); + + props.cpusPerNode(2); + props.memoryPerNode(1024); + props.memoryOverHeadPerNode(512); + props.instances(3); + + Thread thread = runAppMaster(appMaster); + + List<AMRMClient.ContainerRequest> contRequests = collectRequests(rmMock, 1, 1000); + + interruptedThread(thread); + + assertEquals(0, contRequests.size()); + } /** * @throws Exception If failed.
