Hi All. I configured yarn to use capacity scheduler, I have for physical nodes. On first of them I run resourcemanager and nodemanager on all of them.
My capacity-scheduler.xml and yarn-site.yml are attached. When I submit a job to the "long" queue I get in resourcemanager's logfile the content of attached rm.log and in RM qui the job is in state "ACCEPTED: waiting for AM container to be allocated, launched and register with RM" and has finalstatus "UNDEFINED" At the same time I see that in RM gui in scheduler section I have four partitions (node1-4d) in which in every one there are two queues "long" and "short" available. To summarize when I run a task, samza task in my case, I only specify ( https://samza.apache.org/learn/documentation/0.10/jobs/yarn-jobs.html) yarn.queue=long or yarn.queue=short Have I missed something? BR, Rafal.
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<!-- NODEMANAGER -->
<property>
<name>yarn.nodemanager.resource.cpu-vcores</name>
<value>32</value>
</property>
<property>
<name>yarn.nodemanager.resource.memory-mb</name>
<value>51200</value>
</property>
<property>
<name>yarn.nodemanager.vmem-check-enabled</name>
<value>false</value>
</property>
<property>
<name>yarn.nodemanager.vmem-pmem-ratio</name>
<value>10</value>
</property>
<property>
<name>yarn.nodemanager.pmem-check-enabled</name>
<value>false</value>
</property>
<!-- SCHEDULER -->
<!-- VCores Allocation -->
<property>
<name>yarn.scheduler.minimum-allocation-vcores</name>
<value>1</value>
</property>
<property>
<name>yarn.scheduler.maximum-allocation-vcores</name>
<value>1</value>
</property>
<property>
<name>yarn.scheduler.increment-allocation-vcores</name>
<value>1</value>
</property>
<!-- Memory Allocation -->
<property>
<name>yarn.scheduler.minimum-allocation-mb</name>
<value>512</value>
</property>
<property>
<name>yarn.scheduler.maximum-allocation-mb</name>
<value>51200</value>
</property>
<property>
<name>yarn.scheduler.increment-allocation-mb</name>
<value>512</value>
</property>
<!-- RESOURCEMANAGER -->
<property>
<name>yarn.resourcemanager.hostname</name>
<value>node1d</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
</property>
<!-- LABELS -->
<property>
<name>yarn.node-labels.fs-store.root-dir</name>
<value>file:///data/hadoop/conf</value>
</property>
<property>
<name>yarn.node-labels.enabled</name>
<value>true</value>
</property>
<!-- LOG MANAGEMENT -->
<property>
<name>yarn.nodemanager.log.retain-seconds</name>
<value>345600</value>
</property>
<property>
<name>yarn.nodemanager.delete.debug-delay-sec</name>
<value>345600</value>
</property>
<!-- LOCALIZED FILES -->
<property>
<name>yarn.nodemanager.local-dirs</name>
<value>/data/hadoop/nm-local-dir</value>
</property>
<property>
<name>yarn.nodemanager.localizer.cache.cleanup.interval-ms</name>
<value>3600000</value> <!-- 1h -->
</property>
<property>
<name>yarn.nodemanager.localizer.cache.target-size-mb</name>
<value>20480</value> <!-- 20GB -->
</property>
</configuration>
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>yarn.scheduler.capacity.maximum-applications</name>
<value>10000</value>
</property>
<property>
<name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
<value>0.5</value>
</property>
<property>
<name>yarn.scheduler.capacity.resource-calculator</name>
<value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.queues</name>
<value>long,short</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.long.capacity</name>
<value>70</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.long.maximum-capacity</name>
<value>70</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.long.accessible-node-labels</name>
<value>node1d,node2d,node3d,node4d</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.long.accessible-node-labels.node1d.capacity</name>
<value>70</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.long.accessible-node-labels.node1d.maximum-capacity</name>
<value>70</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.long.accessible-node-labels.node2d.capacity</name>
<value>70</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.long.accessible-node-labels.node2d.maximum-capacity</name>
<value>70</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.long.accessible-node-labels.node3d.capacity</name>
<value>70</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.long.accessible-node-labels.node3d.maximum-capacity</name>
<value>70</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.long.accessible-node-labels.node4d.capacity</name>
<value>70</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.long.accessible-node-labels.node4d.maximum-capacity</name>
<value>70</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.short.capacity</name>
<value>30</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.short.maximum-capacity</name>
<value>30</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.short.accessible-node-labels.node1d.capacity</name>
<value>30</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.short.accessible-node-labels.node1d.maximum-capacity</name>
<value>30</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.short.accessible-node-labels.node2d.capacity</name>
<value>30</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.short.accessible-node-labels.node2d.maximum-capacity</name>
<value>30</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.short.accessible-node-labels.node3d.capacity</name>
<value>30</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.short.accessible-node-labels.node3d.maximum-capacity</name>
<value>30</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.short.accessible-node-labels.node4d.capacity</name>
<value>30</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.short.accessible-node-labels.node4d.maximum-capacity</name>
<value>30</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.long.user-limit-factor</name>
<value>1</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.short.user-limit-factor</name>
<value>1</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.long.state</name>
<value>RUNNING</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.short.state</name>
<value>RUNNING</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.long.acl_submit_applications</name>
<value>*</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.short.acl_submit_applications</name>
<value>*</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.long.acl_administer_queue</name>
<value>*</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.short.acl_administer_queue</name>
<value>*</value>
</property>
<property>
<name>yarn.scheduler.capacity.node-locality-delay</name>
<value>-1</value>
</property>
</configuration>
rm.log
Description: Binary data
--------------------------------------------------------------------- To unsubscribe, e-mail: user-unsubscr...@hadoop.apache.org For additional commands, e-mail: user-h...@hadoop.apache.org
