Updated Branches: refs/heads/trunk fbc1ad0e7 -> ae2237849
AMBARI-2902. Update YARN default values. (swagle) Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/ae223784 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/ae223784 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/ae223784 Branch: refs/heads/trunk Commit: ae2237849ccd973591fe163073a310d79310d286 Parents: fbc1ad0 Author: Siddharth Wagle <[email protected]> Authored: Wed Aug 14 16:57:45 2013 -0700 Committer: Siddharth Wagle <[email protected]> Committed: Wed Aug 14 16:57:45 2013 -0700 ---------------------------------------------------------------------- .../services/YARN/configuration/yarn-site.xml | 130 ++++++++++++- .../services/YARN/configuration/yarn-site.xml | 190 ++++++++++++++----- 2 files changed, 267 insertions(+), 53 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/ae223784/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/yarn-site.xml index 8c7e5f6..503cc26 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/yarn-site.xml @@ -31,31 +31,48 @@ <property> <name>yarn.resourcemanager.scheduler.address</name> <value>localhost:8030</value> + <description>The address of the scheduler interface.</description> </property> <property> <name>yarn.resourcemanager.address</name> <value>localhost:8050</value> + <description> + The address of the applications manager interface in the + RM. + </description> </property> <property> <name>yarn.resourcemanager.admin.address</name> <value>localhost:8141</value> + <description>The address of the RM admin interface.</description> </property> <property> <name>yarn.resourcemanager.scheduler.class</name> <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value> + <description>The class to use as the resource scheduler.</description> </property> <property> <name>yarn.scheduler.minimum-allocation-mb</name> - <value>1024</value> + <value>512</value> + <description> + TThe minimum allocation for every container request at the RM, + in MBs. Memory requests lower than this won't take effect, + and the specified value will get allocated at minimum. + </description> </property> <property> <name>yarn.scheduler.maximum-allocation-mb</name> - <value>8192</value> + <value>6144</value> + <description> + The maximum allocation for every container request at the RM, + in MBs. Memory requests higher than this won't take effect, + and will get capped to this value. + </description> </property> <!-- NodeManager --> @@ -63,11 +80,12 @@ <property> <name>yarn.nodemanager.address</name> <value>0.0.0.0:45454</value> + <description>The address of the container manager in the NM.</description> </property> <property> <name>yarn.nodemanager.resource.memory-mb</name> - <value>8192</value> + <value>10240</value> <description>Amount of physical memory, in MB, that can be allocated for containers.</description> </property> @@ -108,11 +126,25 @@ <property> <name>yarn.nodemanager.log-dirs</name> <value>/var/log/hadoop/yarn</value> + <description> + Where to store container logs. An application's localized log directory + will be found in ${yarn.nodemanager.log-dirs}/application_${appid}. + Individual containers' log directories will be below this, in directories + named container_{$contid}. Each container directory will contain the files + stderr, stdin, and syslog generated by that container. + </description> </property> <property> <name>yarn.nodemanager.local-dirs</name> <value></value> + <description> + List of directories to store localized files in. An + application's localized file directory will be found in: + ${yarn.nodemanager.local-dirs}/usercache/${user}/appcache/application_${appid}. + Individual containers' work directories, called container_${contid}, will + be subdirectories of this. + </description> </property> <property> @@ -124,34 +156,43 @@ <property> <name>yarn.nodemanager.container-monitor.interval-ms</name> <value>3000</value> - <description>The interval, in milliseconds, for which the node manager - waits between two cycles of monitoring its containers' memory usage. + <description> + The interval, in milliseconds, for which the node manager + waits between two cycles of monitoring its containers' memory usage. </description> </property> <property> <name>yarn.nodemanager.health-checker.script.path</name> <value>/etc/hadoop/conf/health_check</value> + <description>The health check script to run.</description> </property> <property> <name>yarn.nodemanager.health-checker.interval-ms</name> <value>135000</value> + <description>Frequency of running node health script.</description> </property> <property> <name>yarn.nodemanager.health-checker.script.timeout-ms</name> <value>60000</value> + <description>Script time out period.</description> </property> <property> <name>yarn.nodemanager.log.retain-second</name> <value>604800</value> + <description> + Time in seconds to retain user logs. Only applicable if + log aggregation is disabled. + </description> </property> <property> <name>yarn.log-aggregation-enable</name> <value>true</value> + <description>Whether to enable log aggregation</description> </property> <property> @@ -162,16 +203,89 @@ <property> <name>yarn.nodemanager.remote-app-log-dir-suffix</name> <value>logs</value> + <description> + The remote log dir will be created at + {yarn.nodemanager.remote-app-log-dir}/${user}/{thisParam}. + </description> </property> <property> <name>yarn.nodemanager.log-aggregation.compression-type</name> - <value>gz</value> + <value>gz</value> + <description> + T-file compression types used to compress aggregated logs. + </description> </property> <property> <name>yarn.nodemanager.delete.debug-delay-sec</name> - <value>36000</value> + <value>0</value> + <description> + Number of seconds after an application finishes before the nodemanager's + DeletionService will delete the application's localized file directory + and log directory. + + To diagnose Yarn application problems, set this property's value large + enough (for example, to 600 = 10 minutes) to permit examination of these + directories. After changing the property's value, you must restart the + nodemanager in order for it to have an effect. + + The roots of Yarn applications' work directories is configurable with + the yarn.nodemanager.local-dirs property (see below), and the roots + of the Yarn applications' log directories is configurable with the + yarn.nodemanager.log-dirs property (see also below). + </description> + </property> + + <property> + <name>yarn.log-aggregation.retain-seconds</name> + <value>2592000</value> + <description> + How long to keep aggregation logs before deleting them. -1 disables. + Be careful set this too small and you will spam the name node. + </description> + </property> + + <property> + <name>yarn.nodemanager.admin-env</name> + <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value> + <description> + Environment variables that should be forwarded from the NodeManager's + environment to the container's. + </description> + </property> + + <property> + <name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name> + <value>0.25</value> + <description> + The minimum fraction of number of disks to be healthy for the nodemanager + to launch new containers. This correspond to both + yarn-nodemanager.local-dirs and yarn.nodemanager.log-dirs. i.e. + If there are less number of healthy local-dirs (or log-dirs) available, + then new containers will not be launched on this node. + </description> + </property> + + <property> + <name>yarn.resourcemanager.am.max-attempts</name> + <value>2</value> + <description> + The maximum number of application attempts. It's a global + setting for all application masters. Each application master can specify + its individual maximum number of application attempts via the API, but the + individual number cannot be more than the global upper bound. If it is, + the resourcemanager will override it. The default number is set to 2, to + allow at least one retry for AM. + </description> + </property> + + <property> + <name>yarn.resourcemanager.webapp.address</name> + <value>localhost:8088</value> + <description> + The address of the RM web application. + </description> </property> - + </configuration> http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/ae223784/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/yarn-site.xml index ca49869..5491981 100644 --- a/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/yarn-site.xml @@ -21,7 +21,7 @@ <configuration xmlns:xi="http://www.w3.org/2001/XInclude";> -<!-- ResourceManager --> + <!-- ResourceManager --> <property> <name>yarn.resourcemanager.resource-tracker.address</name> @@ -31,43 +31,61 @@ <property> <name>yarn.resourcemanager.scheduler.address</name> <value>localhost:8030</value> + <description>The address of the scheduler interface.</description> </property> - + <property> <name>yarn.resourcemanager.address</name> <value>localhost:8050</value> + <description> + The address of the applications manager interface in the + RM. + </description> </property> <property> <name>yarn.resourcemanager.admin.address</name> <value>localhost:8141</value> + <description>The address of the RM admin interface.</description> </property> <property> - <name>yarn.resourcemanager.scheduler.class</name> - <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value> + <name>yarn.resourcemanager.scheduler.class</name> + <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value> + <description>The class to use as the resource scheduler.</description> </property> <property> <name>yarn.scheduler.minimum-allocation-mb</name> - <value>1024</value> + <value>512</value> + <description> + TThe minimum allocation for every container request at the RM, + in MBs. Memory requests lower than this won't take effect, + and the specified value will get allocated at minimum. + </description> </property> <property> <name>yarn.scheduler.maximum-allocation-mb</name> - <value>8192</value> + <value>6144</value> + <description> + The maximum allocation for every container request at the RM, + in MBs. Memory requests higher than this won't take effect, + and will get capped to this value. + </description> </property> -<!-- NodeManager --> + <!-- NodeManager --> <property> <name>yarn.nodemanager.address</name> <value>0.0.0.0:45454</value> + <description>The address of the container manager in the NM.</description> </property> <property> <name>yarn.nodemanager.resource.memory-mb</name> - <value>8192</value> + <value>10240</value> <description>Amount of physical memory, in MB, that can be allocated for containers.</description> </property> @@ -75,25 +93,25 @@ <property> <name>yarn.application.classpath</name> <value>/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*</value> - <description>Classpath for typical applications.</description> + <description>Classpath for typical applications.</description> </property> <property> <name>yarn.nodemanager.vmem-pmem-ratio</name> <value>2.1</value> <description>Ratio between virtual memory to physical memory when - setting memory limits for containers. Container allocations are - expressed in terms of physical memory, and virtual memory usage - is allowed to exceed this allocation by this ratio. + setting memory limits for containers. Container allocations are + expressed in terms of physical memory, and virtual memory usage + is allowed to exceed this allocation by this ratio. </description> </property> - + <property> <name>yarn.nodemanager.container-executor.class</name> <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value> <description>ContainerExecutor for launching containers</description> </property> - + <property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce.shuffle</value> @@ -108,39 +126,73 @@ <property> <name>yarn.nodemanager.log-dirs</name> <value>/var/log/hadoop/yarn</value> + <description> + Where to store container logs. An application's localized log directory + will be found in ${yarn.nodemanager.log-dirs}/application_${appid}. + Individual containers' log directories will be below this, in directories + named container_{$contid}. Each container directory will contain the files + stderr, stdin, and syslog generated by that container. + </description> + </property> + + <property> + <name>yarn.nodemanager.local-dirs</name> + <value></value> + <description> + List of directories to store localized files in. An + application's localized file directory will be found in: + ${yarn.nodemanager.local-dirs}/usercache/${user}/appcache/application_${appid}. + Individual containers' work directories, called container_${contid}, will + be subdirectories of this. + </description> + </property> + + <property> + <name>yarn.nodemanager.container-executor.class</name> + <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value> + <description>Executor(launcher) of the containers</description> </property> <property> <name>yarn.nodemanager.container-monitor.interval-ms</name> <value>3000</value> - <description>The interval, in milliseconds, for which the node manager - waits between two cycles of monitoring its containers' memory usage. + <description> + The interval, in milliseconds, for which the node manager + waits between two cycles of monitoring its containers' memory usage. </description> </property> <property> <name>yarn.nodemanager.health-checker.script.path</name> <value>/etc/hadoop/conf/health_check</value> + <description>The health check script to run.</description> </property> <property> <name>yarn.nodemanager.health-checker.interval-ms</name> <value>135000</value> + <description>Frequency of running node health script.</description> </property> <property> <name>yarn.nodemanager.health-checker.script.timeout-ms</name> <value>60000</value> + <description>Script time out period.</description> </property> <property> <name>yarn.nodemanager.log.retain-second</name> <value>604800</value> + <description> + Time in seconds to retain user logs. Only applicable if + log aggregation is disabled. + </description> </property> <property> <name>yarn.log-aggregation-enable</name> <value>true</value> + <description>Whether to enable log aggregation</description> </property> <property> @@ -151,41 +203,89 @@ <property> <name>yarn.nodemanager.remote-app-log-dir-suffix</name> <value>logs</value> + <description> + The remote log dir will be created at + {yarn.nodemanager.remote-app-log-dir}/${user}/{thisParam}. + </description> </property> <property> <name>yarn.nodemanager.log-aggregation.compression-type</name> - <value>gz</value> + <value>gz</value> + <description> + T-file compression types used to compress aggregated logs. + </description> </property> <property> <name>yarn.nodemanager.delete.debug-delay-sec</name> - <value>36000</value> - </property> - - <property> - <name>yarn.resourcemanager.history-store.class</name> - <value>org.apache.hadoop.yarn.server.resourcemanager.history.db.RMHistoryDBStore</value> - </property> - - <property> - <name>yarn.resourcemanager.history-store.db.user</name> - <value>mapred</value> - </property> - - <property> - <name>yarn.resourcemanager.history-store.db.password</name> - <value>mapred</value> - </property> - - <property> - <name>yarn.resourcemanager.history-store.db.database</name> - <value>jdbc:postgresql:ambarirca</value> - </property> - - <property> - <name>yarn.resourcemanager.history-store.db.driver</name> - <value>org.postgresql.Driver</value> - </property> - + <value>0</value> + <description> + Number of seconds after an application finishes before the nodemanager's + DeletionService will delete the application's localized file directory + and log directory. + + To diagnose Yarn application problems, set this property's value large + enough (for example, to 600 = 10 minutes) to permit examination of these + directories. After changing the property's value, you must restart the + nodemanager in order for it to have an effect. + + The roots of Yarn applications' work directories is configurable with + the yarn.nodemanager.local-dirs property (see below), and the roots + of the Yarn applications' log directories is configurable with the + yarn.nodemanager.log-dirs property (see also below). + </description> + </property> + + <property> + <name>yarn.log-aggregation.retain-seconds</name> + <value>2592000</value> + <description> + How long to keep aggregation logs before deleting them. -1 disables. + Be careful set this too small and you will spam the name node. + </description> + </property> + + <property> + <name>yarn.nodemanager.admin-env</name> + <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value> + <description> + Environment variables that should be forwarded from the NodeManager's + environment to the container's. + </description> + </property> + + <property> + <name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name> + <value>0.25</value> + <description> + The minimum fraction of number of disks to be healthy for the nodemanager + to launch new containers. This correspond to both + yarn-nodemanager.local-dirs and yarn.nodemanager.log-dirs. i.e. + If there are less number of healthy local-dirs (or log-dirs) available, + then new containers will not be launched on this node. + </description> + </property> + + <property> + <name>yarn.resourcemanager.am.max-attempts</name> + <value>2</value> + <description> + The maximum number of application attempts. It's a global + setting for all application masters. Each application master can specify + its individual maximum number of application attempts via the API, but the + individual number cannot be more than the global upper bound. If it is, + the resourcemanager will override it. The default number is set to 2, to + allow at least one retry for AM. + </description> + </property> + + <property> + <name>yarn.resourcemanager.webapp.address</name> + <value>localhost:8088</value> + <description> + The address of the RM web application. + </description> + </property> + </configuration>
