Ppchelko has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/391801 )
Change subject: [Config] Move the enabled job config to vars.yaml
..
[Config] Move the enabled job config to vars.yaml
Using regexes to subscrive to the topics is becoming unbearable,
so let's switch to using the vars.yaml.
This will completely break the metrics - we are renaming the rules
so the metrics will start over. I think it's ok, if we continue with
this pattern from now on.
Change-Id: I854c125cfd46e5f3acbe5695be1dfcc34d05deaa
---
M scap/templates/config.yaml.j2
M scap/vars.yaml
2 files changed, 40 insertions(+), 7 deletions(-)
Approvals:
Mobrovac: Looks good to me, but someone else must approve
Ppchelko: Verified; Looks good to me, approved
diff --git a/scap/templates/config.yaml.j2 b/scap/templates/config.yaml.j2
index 7411e78..842ca22 100644
--- a/scap/templates/config.yaml.j2
+++ b/scap/templates/config.yaml.j2
@@ -33,13 +33,13 @@
concurrency: <%= concurrency %>
startup_delay: 6
templates:
- job:
<%- if env == 'production' %>
-topic:
'/(?:^mediawiki\.job\.RecordLintJob$)|(?:^mediawiki\.job\.deleteLinks$)|(?:^mediawiki\.job\.MessageIndexRebuildJob$)|(?:^mediawiki\.job\.flaggedrevs_CacheUpdate$)|(?:^mediawiki\.job\.updateBetaFeaturesUserCounts$)/'
-consumer_batch_size: 10 # Temporary for testing
-<%- else %>
-topic: '/^mediawiki\.job\..*/'
-<%- endif %>
+<%- for topic, topic_config in jobs_enabled.iteritems() %>
+ <%= topic %>:
+topic: 'mediawiki.job.<%= topic %>'
+<%- for key, value in topic_config.iteritems() %>
+<%= key %>: <%= value %>
+<%- endfor %>
exec:
method: post
uri: '<%= jobrunner_uri %>'
@@ -48,6 +48,31 @@
body: '{{globals.message}}'
agentOptions:
keepAlive: true
+<%- endfor %>
+ # Temporary for double-processing to not loose jobs!
+ job:
+topic:
'/(?:^mediawiki\.job\.RecordLintJob$)|(?:^mediawiki\.job\.deleteLinks$)|(?:^mediawiki\.job\.MessageIndexRebuildJob$)|(?:^mediawiki\.job\.flaggedrevs_CacheUpdate$)|(?:^mediawiki\.job\.updateBetaFeaturesUserCounts$)/'
+exec:
+ method: post
+ uri: '<%= jobrunner_uri %>'
+ headers:
+content-type: 'application/json'
+ body: '{{globals.message}}'
+ agentOptions:
+keepAlive: true
+ # End of temporary code
+<%- else %>
+ job:
+topic: '/^mediawiki\.job\..*/'
+exec:
+ method: post
+ uri: '<%= jobrunner_uri %>'
+ headers:
+content-type: 'application/json'
+ body: '{{globals.message}}'
+ agentOptions:
+keepAlive: true
+<%- endif %>
# Number of worker processes to spawn.
diff --git a/scap/vars.yaml b/scap/vars.yaml
index 0717d29..623171f 100644
--- a/scap/vars.yaml
+++ b/scap/vars.yaml
@@ -14,8 +14,16 @@
site: datacenter1
broker_list: localhost:9092
jobrunner_uri: http://localhost/rpc/RunSingleJob.php
-concurrency: 200
+concurrency: 30
redis_path: /var/run/redis.sock
redis_pass:
env: production
kafka_max_bytes: 4194304
+jobs_enabled:
+ deleteLinks: {}
+ MessageIndexRebuildJob: {}
+ flaggedrevs_CacheUpdate: {}
+ updateBetaFeaturesUserCounts: {}
+ RecordLintJob:
+concurrency: 100
+consumer_batch_size: 10
--
To view, visit https://gerrit.wikimedia.org/r/391801
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I854c125cfd46e5f3acbe5695be1dfcc34d05deaa
Gerrit-PatchSet: 7
Gerrit-Project: mediawiki/services/change-propagation/jobqueue-deploy
Gerrit-Branch: master
Gerrit-Owner: Ppchelko
Gerrit-Reviewer: Mobrovac
Gerrit-Reviewer: Ppchelko
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
