mikewalch closed pull request #15: fixes apache/fluo#671 made reserved mem
configurable
URL: https://github.com/apache/fluo-yarn/pull/15
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
index c5a8629..15e77b0 100644
--- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
+++ b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
@@ -116,6 +116,16 @@ public int getWorkerMaxMemory() {
FluoYarnProperties.WORKER_MAX_MEMORY_MB_DEFAULT));
}
+ public String getWorkerReservedMemory() {
+ String rmem =
props.getProperty(FluoYarnProperties.WORKER_RESERVED_MEMORY_MB_PROP, null);
+ if (rmem == null) {
+ return rmem;
+ }
+
+ Preconditions.checkArgument(Integer.parseInt(rmem) > 0);
+ return rmem;
+ }
+
public int getOracleCores() {
return
Integer.valueOf(props.getProperty(FluoYarnProperties.ORACLE_NUM_CORES_PROP,
FluoYarnProperties.ORACLE_NUM_CORES_DEFAULT));
@@ -131,6 +141,16 @@ public int getOracleMaxMemory() {
FluoYarnProperties.ORACLE_MAX_MEMORY_MB_DEFAULT));
}
+ public String getOracleReservedMemory() {
+ String rmem =
props.getProperty(FluoYarnProperties.ORACLE_RESERVED_MEMORY_MB_PROP, null);
+ if (rmem == null) {
+ return rmem;
+ }
+
+ Preconditions.checkArgument(Integer.parseInt(rmem) > 0);
+ return rmem;
+ }
+
public String getYarnQueue() {
return props.getProperty(FluoYarnProperties.QUEUE_PROP, null);
}
diff --git a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java
b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java
index e79f9dc..056604f 100644
--- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java
+++ b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java
@@ -17,7 +17,9 @@
import java.io.File;
import java.util.Collection;
+import java.util.Collections;
+import org.apache.twill.api.Configs;
import org.apache.twill.api.ResourceReport;
import org.apache.twill.api.ResourceSpecification;
import org.apache.twill.api.TwillApplication;
@@ -129,6 +131,16 @@ public static void main(String[] args) throws Exception {
preparer.setSchedulerQueue(env.getYarnQueue());
}
+ if (env.getWorkerReservedMemory() != null) {
+ preparer.withConfiguration(WORKER_ID, Collections
+ .singletonMap(Configs.Keys.JAVA_RESERVED_MEMORY_MB,
env.getWorkerReservedMemory()));
+ }
+
+ if (env.getOracleReservedMemory() != null) {
+ preparer.withConfiguration(ORACLE_ID, Collections
+ .singletonMap(Configs.Keys.JAVA_RESERVED_MEMORY_MB,
env.getOracleReservedMemory()));
+ }
+
TwillController controller = preparer.start();
ResourceReport report = controller.getResourceReport();
diff --git
a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
index 990afa7..c707928 100644
--- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
+++ b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
@@ -35,6 +35,8 @@
// Worker properties
public static final String WORKER_INSTANCES_PROP = YARN_PREFIX +
".worker.instances";
public static final String WORKER_MAX_MEMORY_MB_PROP = YARN_PREFIX +
".worker.max.memory.mb";
+ public static final String WORKER_RESERVED_MEMORY_MB_PROP =
+ YARN_PREFIX + ".worker.reserved.memory.mb";
public static final String WORKER_NUM_CORES_PROP = YARN_PREFIX +
".worker.num.cores";
public static final String WORKER_INSTANCES_DEFAULT = "1";
public static final String WORKER_MAX_MEMORY_MB_DEFAULT = "1024";
@@ -43,6 +45,8 @@
// Oracle properties
public static final String ORACLE_INSTANCES_PROP = YARN_PREFIX +
".oracle.instances";
public static final String ORACLE_MAX_MEMORY_MB_PROP = YARN_PREFIX +
".oracle.max.memory.mb";
+ public static final String ORACLE_RESERVED_MEMORY_MB_PROP =
+ YARN_PREFIX + ".oracle.reserved.memory.mb";
public static final String ORACLE_NUM_CORES_PROP = YARN_PREFIX +
".oracle.num.cores";
public static final String ORACLE_INSTANCES_DEFAULT = "1";
public static final String ORACLE_MAX_MEMORY_MB_DEFAULT = "512";
diff --git a/distribution/conf/fluo-yarn.properties
b/distribution/conf/fluo-yarn.properties
index c2f3546..f2b1705 100644
--- a/distribution/conf/fluo-yarn.properties
+++ b/distribution/conf/fluo-yarn.properties
@@ -43,6 +43,9 @@
## Max memory of Oracle yarn containers (in MB)
#fluo.yarn.oracle.max.memory.mb=512
+## Reserved memory. Behaves same as worker property, so see its docs.
+#fluo.yarn.oracle.reserved.memory.mb=200
+
## Number of Oracle virtual cores
#fluo.yarn.oracle.num.cores=1
@@ -51,12 +54,14 @@
## Number of Worker instances
#fluo.yarn.worker.instances=1
-## Max memory of worker YARN containers (in MB). If YARN is killing worker
processes consider
-## increasing twill.java.reserved.memory.mb (which defaults to 200 and is set
in yarn-site.xml).
-## The twill.java.reserved.memory.mb config determines the gap between the
YARN memory limit set
-## below and the java -Xmx setting. For example, if max memory is 1024 and
twill reserved memory
-## is 200, the java -Xmx setting will be 1024-200 = 824 MB.
+## Max memory of worker YARN containers (in MB).
#fluo.yarn.worker.max.memory.mb=1024
+## This property is used to compute the Java heap size by subtracting it from
the max memory
+## property. For example, if max memory is 1024 and reserved memory is 200,
then java -Xmx setting
+## will be 1024-200 = 824 MB. If YARN is killing workers because too much
memory is used, then
+## consider increasing this property.
+#fluo.yarn.worker.reserved.memory.mb=200
+
## Number of worker virtual cores
#fluo.yarn.worker.num.cores=1
diff --git a/distribution/lib/fetch.sh b/distribution/lib/fetch.sh
index 0881187..2c351f0 100755
--- a/distribution/lib/fetch.sh
+++ b/distribution/lib/fetch.sh
@@ -17,7 +17,7 @@ lib_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
maven_prefix=https://repo1.maven.org/maven2
function download {
- IFS=':' read -ra DEP <<< "$1"
+ IFS=':' read -ra DEP <<< "$1"
dir=$lib_dir/
if [ -n "$2" ]; then
dir=$lib_dir/$2
@@ -35,7 +35,7 @@ function download {
if [ -f $dir/$fn ]; then
echo "SUCCESS: Dependency exists - $dir/$fn"
- else
+ else
wget -q $download_url -P $dir
if [ $? == 0 ]; then
echo "SUCCESS: Dependency downloaded from $download_url"
@@ -57,14 +57,14 @@ download com.yammer.metrics:metrics-annotation:jar:2.2.0
download com.yammer.metrics:metrics-core:jar:2.2.0
download net.sf.jopt-simple:jopt-simple:jar:3.2
download org.apache.kafka:kafka_2.10:jar:0.8.0
-download org.apache.twill:twill-api:jar:0.11.0
-download org.apache.twill:twill-common:jar:0.11.0
-download org.apache.twill:twill-core:jar:0.11.0
-download org.apache.twill:twill-discovery-api:jar:0.11.0
-download org.apache.twill:twill-discovery-core:jar:0.11.0
-download org.apache.twill:twill-ext:jar:0.11.0
-download org.apache.twill:twill-yarn:jar:0.11.0
-download org.apache.twill:twill-zookeeper:jar:0.11.0
+download org.apache.twill:twill-api:jar:0.12.0
+download org.apache.twill:twill-common:jar:0.12.0
+download org.apache.twill:twill-core:jar:0.12.0
+download org.apache.twill:twill-discovery-api:jar:0.12.0
+download org.apache.twill:twill-discovery-core:jar:0.12.0
+download org.apache.twill:twill-ext:jar:0.12.0
+download org.apache.twill:twill-yarn:jar:0.12.0
+download org.apache.twill:twill-zookeeper:jar:0.12.0
download org.ow2.asm:asm-all:jar:5.0.2
download org.scala-lang:scala-compiler:jar:2.10.1
download org.scala-lang:scala-library:jar:2.10.1
diff --git a/pom.xml b/pom.xml
index 00cda0d..ec732cb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -42,7 +42,7 @@
<hadoop.version>2.6.3</hadoop.version>
<logback.version>1.1.3</logback.version>
<slf4j.version>1.7.12</slf4j.version>
- <twill.version>0.11.0</twill.version>
+ <twill.version>0.12.0</twill.version>
</properties>
<dependencyManagement>
<dependencies>
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
