Ottomata has uploaded a new change for review.
https://gerrit.wikimedia.org/r/193109
Change subject: Update cdh module and tweak FairScheduler settings to give more
priority for essential jobs
......................................................................
Update cdh module and tweak FairScheduler settings to give more priority for
essential jobs
Change-Id: I7e84cd4d71ccd686df907efa8d98025bbdd86f1b
---
M manifests/role/analytics/hadoop.pp
M modules/cdh
M templates/hadoop/fair-scheduler.xml.erb
3 files changed, 27 insertions(+), 2 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/09/193109/1
diff --git a/manifests/role/analytics/hadoop.pp
b/manifests/role/analytics/hadoop.pp
index 72633e1..63e07da 100644
--- a/manifests/role/analytics/hadoop.pp
+++ b/manifests/role/analytics/hadoop.pp
@@ -334,6 +334,17 @@
net_topology_script_template =>
$net_topology_script_template,
# Use fair-scheduler.xml.erb to define FairScheduler queues.
fair_scheduler_template => $fair_scheduler_template,
+
+ yarn_site_extra_properties => {
+ # Enable FairScheduler preemption. This will allow the essential
queue
+ # to preempt non-essential jobs.
+ 'yarn.scheduler.fair.preemption' => true,
+ # Let YARN wait for at least 1/3 of nodes to present scheduling
+ # opportunties before scheduling a job for certain data
+ # on a node on which that data is not present.
+ 'yarn.scheduler.fair.locality.threshold.node' => '0.33',
+ }
+
gelf_logging_enabled => $gelf_logging_enabled,
gelf_logging_host => $gelf_logging_host,
gelf_logging_port => $gelf_logging_port,
diff --git a/modules/cdh b/modules/cdh
index 2d5a1d5..610d036 160000
--- a/modules/cdh
+++ b/modules/cdh
-Subproject commit 2d5a1d5e30f70e1e49560d57c9cd435102864d20
+Subproject commit 610d036ab1673517a9bab547ae8e0b1ba12dd613
diff --git a/templates/hadoop/fair-scheduler.xml.erb
b/templates/hadoop/fair-scheduler.xml.erb
index cf3592c..1d50476 100644
--- a/templates/hadoop/fair-scheduler.xml.erb
+++ b/templates/hadoop/fair-scheduler.xml.erb
@@ -3,15 +3,29 @@
<fairSharePreemptionTimeout>30</fairSharePreemptionTimeout>
<queue name="default">
+ <weight>1.0</weight>
<maxRunningApps>50</maxRunningApps>
<schedulingMode>fair</schedulingMode>
</queue>
<queue name="essential">
- <weight>2.0</weight>
+ <!--
+ Use FIFO for essential queue. We want jobs submitted
+ here to run in sequential order.
+ -->
+ <schedulingMode>fifo</schedulingMode>
<aclSubmitApps>hdfs</aclSubmitApps>
+
+ <!--
+ The essential queue has a much higher priority than default,
+ and it will start killing (preempting) jobs in other queues,
+ first after 60 seconds if it can't get its minimum share,
+ and then more after 10 minutes if it can't get its fair share.
+ -->
+ <weight>10.0</weight>
+ <minSharePreemptionTimeout>60</minSharePreemptionTimeout>
+ <fairSharePreemptionThreshold>600</fairSharePreemptionThreshold>
<maxRunningApps>50</maxRunningApps>
- <schedulingMode>fair</schedulingMode>
</queue>
</allocations>
--
To view, visit https://gerrit.wikimedia.org/r/193109
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7e84cd4d71ccd686df907efa8d98025bbdd86f1b
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
