Joal has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/349266 )
Change subject: Update restbase oozie spark job
......................................................................
Update restbase oozie spark job
This job has been created before spark had dynamic allocation.
This patch removes the fixed number of executor and enables
dynamic allocation for this job.
Bug: T163479
Change-Id: Icf5b764ce3e8906707ff1958bcdbe1a094841c4c
---
M oozie/restbase/README.md
M oozie/restbase/coordinator.properties
M oozie/restbase/coordinator.xml
M oozie/restbase/workflow.xml
4 files changed, 9 insertions(+), 8 deletions(-)
Approvals:
Mforns: Looks good to me, but someone else must approve
Ottomata: Looks good to me, but someone else must approve
Joal: Verified; Looks good to me, approved
diff --git a/oozie/restbase/README.md b/oozie/restbase/README.md
index 6f33e88..d8142f7 100644
--- a/oozie/restbase/README.md
+++ b/oozie/restbase/README.md
@@ -10,7 +10,7 @@
oozie job -run \
-config coordinator.properties \
-D
refinery_directory=hdfs://analytics-hadoop/user/madhuvishy/refinery \
- -D
spark_job_jar=hdfs://analytics-hadoop/user/madhuvishy/source/refinery-job-0.0.18-SNAPSHOT.jar
+ -D
spark_job_jar=hdfs://analytics-hadoop/user/madhuvishy/source/refinery-job-0.0.45-SNAPSHOT.jar
The results of this job can be viewed in Graphite (graphite.wikimedia.org)
under the restbase namespace,
in restbase.requests.varnish_requests.
\ No newline at end of file
diff --git a/oozie/restbase/coordinator.properties
b/oozie/restbase/coordinator.properties
index 9b5997e..08b7d93 100644
--- a/oozie/restbase/coordinator.properties
+++ b/oozie/restbase/coordinator.properties
@@ -49,9 +49,9 @@
spark_job_jar =
${artifacts_directory}/org/wikimedia/analytics/refinery/refinery-job-0.0.41.jar
spark_job_class =
org.wikimedia.analytics.refinery.job.RESTBaseMetrics
spark_job_name = restbase_metrics
-spark_number_executors = 2
-spark_executor_memory = 1G
-spark_driver_memory = 2G
+spark_executor_memory = 2G
+spark_driver_memory = 4G
+spark_max_executors = 16
graphite_namespace = restbase.requests
graphite_host = graphite-in.eqiad.wmnet
graphite_port = 2003
diff --git a/oozie/restbase/coordinator.xml b/oozie/restbase/coordinator.xml
index d2d777b..0a150e6 100644
--- a/oozie/restbase/coordinator.xml
+++ b/oozie/restbase/coordinator.xml
@@ -24,7 +24,7 @@
<property><name>spark_job_class</name></property>
<property><name>spark_executor_memory</name></property>
<property><name>spark_driver_memory</name></property>
- <property><name>spark_number_executors</name></property>
+ <property><name>spark_max_executors</name></property>
<property><name>graphite_host</name></property>
<property><name>graphite_port</name></property>
<property><name>graphite_namespace</name></property>
diff --git a/oozie/restbase/workflow.xml b/oozie/restbase/workflow.xml
index 4f74856..2ec0dd5 100644
--- a/oozie/restbase/workflow.xml
+++ b/oozie/restbase/workflow.xml
@@ -40,8 +40,8 @@
<description>Memory to allocate for spark driver
process</description>
</property>
<property>
- <name>spark_number_executors</name>
- <description>Number of executors to run in parallel</description>
+ <name>spark_max_executors</name>
+ <description>Maximum concurrent number of executors for spark
dynamic allocation</description>
</property>
<property>
<name>year</name>
@@ -104,7 +104,8 @@
<name>${spark_job_name}-${year}-${month}-${day}-${hour}</name>
<class>${spark_job_class}</class>
<jar>${spark_job_jar}</jar>
- <spark-opts>--conf spark.yarn.jar=${spark_assembly_jar}
--executor-memory ${spark_executor_memory} --driver-memory
${spark_driver_memory} --num-executors ${spark_number_executors} --queue
${queue_name} </spark-opts>
+ <spark-opts>--conf spark.yarn.jar=${spark_assembly_jar}
--executor-memory ${spark_executor_memory} --driver-memory
${spark_driver_memory} --queue ${queue_name} --conf
spark.dynamicAllocation.enabled=true --conf spark.shuffle.service.enabled=true
+ --conf
spark.dynamicAllocation.maxExecutors=${spark_max_executors}</spark-opts>
<arg>--year</arg>
<arg>${year}</arg>
<arg>--month</arg>
--
To view, visit https://gerrit.wikimedia.org/r/349266
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Icf5b764ce3e8906707ff1958bcdbe1a094841c4c
Gerrit-PatchSet: 2
Gerrit-Project: analytics/refinery
Gerrit-Branch: master
Gerrit-Owner: Joal <[email protected]>
Gerrit-Reviewer: Joal <[email protected]>
Gerrit-Reviewer: Mforns <[email protected]>
Gerrit-Reviewer: Nuria <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
