This is an automated email from the ASF dual-hosted git repository.
mwalch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo-yarn.git
The following commit(s) were added to refs/heads/master by this push:
new 2ba874f Fixes #11 Apps are failing to run on cluster (#12)
2ba874f is described below
commit 2ba874f99baccd2315ec581b85a981330f3a5110
Author: Mike Walch <[email protected]>
AuthorDate: Tue Dec 19 17:28:38 2017 -0500
Fixes #11 Apps are failing to run on cluster (#12)
* Created new property so that 'fs.defaultFS' is set in hadoop
config that is passed to YarnTwillRunnerService
---
core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java | 5 +++++
.../main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java | 2 ++
distribution/conf/fluo-yarn.properties | 8 ++++++++
3 files changed, 15 insertions(+)
diff --git a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
index dcf2041..2c76fec 100644
--- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
+++ b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
@@ -82,6 +82,7 @@ public class FluoYarnEnv {
yarnConfig = new YarnConfiguration();
yarnConfig.set("mapreduce.framework.name", "yarn");
yarnConfig.set("yarn.resourcemanager.hostname",
getYarnResourceManager());
+ yarnConfig.set("fs.defaultFS", getDfsRoot());
}
return yarnConfig;
}
@@ -96,6 +97,10 @@ public class FluoYarnEnv {
FluoYarnProperties.ZOOKEEPERS_DEFAULT);
}
+ public String getDfsRoot() {
+ return props.getProperty(FluoYarnProperties.DFS_ROOT_PROP,
FluoYarnProperties.DFS_ROOT_DEFAULT);
+ }
+
public int getWorkerCores() {
return
Integer.valueOf(props.getProperty(FluoYarnProperties.WORKER_NUM_CORES_PROP,
FluoYarnProperties.WORKER_NUM_CORES_DEFAULT));
diff --git
a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
index 4d34647..d61dca6 100644
--- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
+++ b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
@@ -26,8 +26,10 @@ public class FluoYarnProperties {
// General properties
public static final String YARN_RESOURCE_MANAGER_PROP = YARN_PREFIX +
".resource.manager";
public static final String ZOOKEEPERS_PROP = YARN_PREFIX + ".zookeepers";
+ public static final String DFS_ROOT_PROP = YARN_PREFIX + ".dfs.root";
public static final String YARN_RESOURCE_MANAGER_DEFAULT = "localhost";
public static final String ZOOKEEPERS_DEFAULT = "localhost/fluo-yarn";
+ public static final String DFS_ROOT_DEFAULT = "hdfs://localhost:8020/";
// Worker properties
public static final String WORKER_INSTANCES_PROP = YARN_PREFIX +
".worker.instances";
diff --git a/distribution/conf/fluo-yarn.properties
b/distribution/conf/fluo-yarn.properties
index 996b132..0af7bf8 100644
--- a/distribution/conf/fluo-yarn.properties
+++ b/distribution/conf/fluo-yarn.properties
@@ -25,15 +25,21 @@
## Zookeeper connection string specifying host and chroot where Fluo YARN
launcher stores data.
## A chroot directory suffix (with max depth of 1) must be specified but
doesn't need to be named '/fluo-yarn'.
#fluo.yarn.zookeepers=localhost/fluo-yarn
+
## YARN resource manager hostname. Should match
'yarn.resourcemanager.hostname' property in Hadoop's yarn-site.xml
#fluo.yarn.resource.manager=localhost
+## HDFS root path. Should be prefixed with 'fs.defaultFS' property in Hadoop's
core-site.xml
+#fluo.yarn.dfs.root=hdfs://localhost:8020/
+
## Oracle properties
## -----------------
## Number of oracle yarn instances
#fluo.yarn.oracle.instances=1
+
## Max memory of Oracle yarn containers (in MB)
#fluo.yarn.oracle.max.memory.mb=512
+
## Number of Oracle virtual cores
#fluo.yarn.oracle.num.cores=1
@@ -41,11 +47,13 @@
## -----------------
## Number of Worker instances
#fluo.yarn.worker.instances=1
+
## Max memory of worker YARN containers (in MB). If YARN is killing worker
processes consider
## increasing twill.java.reserved.memory.mb (which defaults to 200 and is set
in yarn-site.xml).
## The twill.java.reserved.memory.mb config determines the gap between the
YARN memory limit set
## below and the java -Xmx setting. For example, if max memory is 1024 and
twill reserved memory
## is 200, the java -Xmx setting will be 1024-200 = 824 MB.
#fluo.yarn.worker.max.memory.mb=1024
+
## Number of worker virtual cores
#fluo.yarn.worker.num.cores=1
--
To stop receiving notification emails like this one, please contact
['"[email protected]" <[email protected]>'].
