Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/309979
Change subject: Change-Prop: Use Scap3 for config deploys ...................................................................... Change-Prop: Use Scap3 for config deploys Bug: T144595 Change-Id: Ibadfbfac4f680c56bf1d875e3e28ebf152d25fee --- M hieradata/labs/deployment-prep/common.yaml M modules/changeprop/manifests/init.pp D modules/changeprop/templates/config.yaml.erb 3 files changed, 17 insertions(+), 501 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/79/309979/1 diff --git a/hieradata/labs/deployment-prep/common.yaml b/hieradata/labs/deployment-prep/common.yaml index 5702c9d..3d489d1 100644 --- a/hieradata/labs/deployment-prep/common.yaml +++ b/hieradata/labs/deployment-prep/common.yaml @@ -18,7 +18,6 @@ citoid::zotero_host: deployment-zotero01.deployment-prep.eqiad.wmflabs citoid::zotero_port: 1969 changeprop::purge_host: deployment-cache-text04.deployment-prep.eqiad.wmflabs -changeprop::concurrency: 20 changeprop::ores_uri: http://deployment-sca03.deployment-prep.eqiad.wmflabs:8081 graphoid::allowed_domains: http: diff --git a/modules/changeprop/manifests/init.pp b/modules/changeprop/manifests/init.pp index 7b6122b..c61a7bd 100644 --- a/modules/changeprop/manifests/init.pp +++ b/modules/changeprop/manifests/init.pp @@ -18,39 +18,35 @@ # [*ores_uri*] # The host where ORES service is running. Default: http://ores.svc.eqiad.wmnet:8081 # -# [*concurrency*] -# The maximum number of tasks the service can execute at any given point in -# time. Default: 30 -# - class changeprop( $broker_list, $purge_host = '239.128.0.112', $ores_uri = 'http://ores.svc.eqiad.wmnet:8081', $purge_port = 4827, - $concurrency = 50, ) { include ::service::configuration require ::changeprop::packages - $restbase_uri = $::service::configuration::restbase_uri - $mwapi_uri = $::service::configuration::mwapi_uri - service::node { 'changeprop': - enable => true, - port => 7272, - heap_limit => 750, - config => template('changeprop/config.yaml.erb'), - no_workers => 8, # TEMP - starter_module => 'hyperswitch', - healthcheck_url => '', - has_spec => true, - deployment => 'scap3', - auto_refresh => false, - init_restart => false, - environment => { + enable => true, + port => 7272, + healthcheck_url => '', + has_spec => true, + deployment => 'scap3', + deployment_config => true, + deployment_vars => { + broker_list => $broker_list, + mwapi_uri => $::service::configuration::mwapi_uri, + restbase_uri => $::service::configuration::restbase_uri, + ores_uri => $ores_uri, + purge_host => $purge_host, + purge_port => $purge_port, + }, + auto_refresh => false, + init_restart => false, + environment => { 'UV_THREADPOOL_SIZE' => 128 }, } diff --git a/modules/changeprop/templates/config.yaml.erb b/modules/changeprop/templates/config.yaml.erb deleted file mode 100644 index 56e0cf1..0000000 --- a/modules/changeprop/templates/config.yaml.erb +++ /dev/null @@ -1,479 +0,0 @@ -user_agent: ChangePropagation/WMF -spec: - x-sub-request-filters: - - type: default - name: http - options: - allow: - - pattern: /^https?:\/\// - forward_headers: - user-agent: true - title: The Change Propagation root - paths: - /{api:sys}/purge: - x-modules: - - path: src/sys/purge.js - options: - host: <%= @purge_host %> - port: <%= @purge_port %> - /{api:sys}/links: - x-modules: - - path: src/sys/dep_updates.js - options: - templates: - mw_api: - uri: <%= @mwapi_uri %> - headers: - host: '{{message.meta.domain}}' - body: - formatversion: 2 - /{api:sys}/queue: - x-modules: - - path: src/sys/kafka.js - options: - metadata_broker_list: <%= @broker_list %> - dc_name: <%= scope.lookupvar('::site') %> - concurrency: <%= @concurrency %> - startup_delay: 60000 - templates: - summary_rerender: - topic: resource_change - match: - meta: - uri: '/^https?:\/\/[^\/]+\/api\/rest_v1\/page\/html\/([^\/]+)$/' - tags: - - restbase - match_not: - meta: - domain: '/wiktionary\.org$/' - exec: - method: get - # Don't encode title since it should be already encoded - uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/summary/{{match.meta.uri[1]}}' - query: - redirect: false - headers: - cache-control: no-cache - - definition_rerender: - topic: resource_change - match: - meta: - # These URIs are coming from RESTBase, so we know that article titles will be normalized - # and main namespace articles will not have : (uri-encoded, so %3a or %3A) - uri: '/^https?:\/\/[^\/]+\/api\/rest_v1\/page\/html\/((?:(?!%3a|%3A|\/).)+)$/' - domain: '/^en\.wiktionary\.org$/' - tags: - - restbase - exec: - method: get - # Don't encode title since it should be already encoded - uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/definition/{{match.meta.uri[1]}}' - query: - redirect: false - headers: - cache-control: no-cache - - mobile_rerender: - topic: resource_change - match: - meta: - uri: '/^https?:\/\/[^\/]+\/api\/rest_v1\/page\/html\/([^/]+)$/' - tags: - - restbase - exec: - method: get - # Don't encode title since it should be already encoded - uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/mobile-sections/{{match.meta.uri[1]}}' - query: - redirect: false - headers: - cache-control: no-cache - - purge_varnish: - topic: resource_change - match: - meta: - uri: '/^https?:\/\/[^\/]+\/api\/rest_v1\/(?<title>.+)$/' - tags: - - restbase - exec: - method: post - uri: '/sys/purge/' - body: - - meta: - uri: '//{{message.meta.domain}}/api/rest_v1/{{match.meta.uri.title}}' - - # RESTBase update jobs - mw_purge: - topic: resource_change - match: - meta: - uri: '/^https?:\/\/[^\/]+\/wiki\/(?<title>.+)$/' - tags: - - purge - exec: - method: get - uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{{match.meta.uri.title}}' - headers: - cache-control: no-cache - if-unmodified-since: '{{date(message.meta.dt)}}' - query: - redirect: false - - null_edit: - topic: resource_change - ignore: - status: - - 403 # Ignoring 403 since some of the pages with high number of null_edit events are blacklisted - - 412 - match: - meta: - uri: '/^https?:\/\/[^\/]+\/wiki\/(?<title>.+)$/' - tags: - - null_edit - exec: - method: get - uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{{match.meta.uri.title}}' - headers: - cache-control: no-cache - if-unmodified-since: '{{date(message.meta.dt)}}' - query: - redirect: false - - page_edit: - topic: mediawiki.revision_create - retry_on: - status: - - '5xx' - - 404 # Sometimes occasional 404s happen because of the mysql replication lag, so retry - exec: - method: get - uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{message.page_title}/{{message.rev_id}}' - headers: - cache-control: no-cache - x-restbase-parentrevision: '{{message.rev_parent_id}}' - if-unmodified-since: '{{date(message.meta.dt)}}' - query: - redirect: false - - revision_visibility_change: - topic: mediawiki.revision_visibility_set - ignore: - status: - - 403 # When the revision is hidden 403 will be returned by RESTBase, it's a valid situation - - 412 - exec: - method: get - uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/revision/{{message.revision_id}}' - headers: - cache-control: no-cache - query: - redirect: false - - page_delete: - topic: mediawiki.page_delete - ignore: - status: - - 404 # 404 is a normal response for page deletion - - 412 - exec: - method: get - uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/title/{message.title}' - headers: - cache-control: no-cache - query: - redirect: false - - page_restore: - topic: mediawiki.page_restore - exec: - method: get - uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{message.title}' - headers: - cache-control: no-cache - query: - redirect: false - - page_move: - topic: mediawiki.page_move - exec: - - method: get - uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{message.new_title}/{{message.new_revision_id}}' - headers: - cache-control: no-cache - if-unmodified-since: '{{date(message.meta.dt)}}' - query: - redirect: false - - method: get - uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/title/{message.old_title}' - headers: - cache-control: no-cache - query: - redirect: false - - transclusion_update: - topic: mediawiki.revision_create - exec: - method: 'post' - uri: '/sys/links/transcludes/{message.page_title}' - body: '{{globals.message}}' - - on_transclusion_update: - concurrency: <%= @concurrency.to_i * 3 %> - topic: resource_change - match: - meta: - uri: '/https?:\/\/[^\/]+\/wiki\/(?<title>.+)/' - tags: [ 'change-prop', 'transcludes' ] - exec: - method: get - uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{{match.meta.uri.title}}' - headers: - cache-control: no-cache - if-unmodified-since: '{{date(message.meta.dt)}}' - query: - redirect: false - - # ORES caching updates - ores_cache: - topic: mediawiki.revision_create - concurrency: 15 - ignore: - status: - - 503 - cases: - - match: - meta: - domain: ar.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/arwiki/' - query: - models: 'reverted' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: cs.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/cswiki/' - query: - models: 'reverted' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: en.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/enwiki/' - query: - models: 'reverted|damaging|goodfaith' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: en.wiktionary.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/enwiktionary/' - query: - models: 'reverted' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: es.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/eswiki/' - query: - models: 'reverted' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: et.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/etwiki/' - query: - models: 'reverted' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: fa.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/fawiki/' - query: - models: 'reverted|damaging|goodfaith' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: fr.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/frwiki/' - query: - models: 'reverted' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: he.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/hewiki/' - query: - models: 'reverted' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: hu.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/huwiki/' - query: - models: 'reverted' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: id.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/idwiki/' - query: - models: 'reverted' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: it.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/itwiki/' - query: - models: 'reverted' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: nl.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/nlwiki/' - query: - models: 'reverted|damaging|goodfaith' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: pl.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/plwiki/' - query: - models: 'reverted|damaging|goodfaith' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: pt.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/ptwiki/' - query: - models: 'reverted|damaging|goodfaith' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: ru.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/ruwiki/' - query: - models: 'reverted|damaging|goodfaith' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: tr.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/trwiki/' - query: - models: 'reverted|damaging|goodfaith' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: uk.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/ukwiki/' - query: - models: 'reverted' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: vi.wikipedia.org - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/viwiki/' - query: - models: 'reverted' - revids: '{{message.rev_id}}' - precache: true - - match: - meta: - domain: wikidata.org - page_namespace: 0 - rev_by_bot: false - exec: - uri: '<%= @ores_uri %>/v2/scores/wikidatawiki/' - query: - models: 'reverted|damaging|goodfaith' - revids: '{{message.rev_id}}' - precache: true - - # Re-renders summary on WikiData item update - wikidata_description: - topic: mediawiki.revision_create - match: - meta: - domain: www.wikidata.org - match_not: - page_title: /:/ - exec: - method: 'post' - uri: '/sys/links/wikidata_descriptions' - body: '{{globals.message}}' - - on_wikidata_description_change: - topic: resource_change - match: - meta: - uri: '/https?:\/\/[^\/]+\/wiki\/(?<title>.+)/' - tags: [ 'change-prop', 'wikidata' ] - exec: - - method: get - uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/summary/{{match.meta.uri.title}}' - headers: - cache-control: no-cache - query: - redirect: false - - method: get - uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/mobile-sections/{{match.meta.uri.title}}' - headers: - cache-control: no-cache - query: - redirect: false -- To view, visit https://gerrit.wikimedia.org/r/309979 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ibadfbfac4f680c56bf1d875e3e28ebf152d25fee Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Mobrovac <mobro...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
