Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/247935
Change subject: RESTBase: Set up MobileApps storage and AQS public API ...................................................................... RESTBase: Set up MobileApps storage and AQS public API This patch introduces storage capabilities for MobileApps endpoints and exposes the Analytics Query Service public API endpoints for the global domain. Bug: T114830 Bug: T102130 Change-Id: I17ae36660ebb374e7062cd1e4ad4634ffddf66a7 --- M hieradata/labs/deployment-prep/common.yaml M hieradata/role/common/restbase.yaml M modules/restbase/manifests/init.pp M modules/restbase/templates/config.labs.yaml.erb M modules/restbase/templates/config.yaml.erb 5 files changed, 325 insertions(+), 20 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/35/247935/1 diff --git a/hieradata/labs/deployment-prep/common.yaml b/hieradata/labs/deployment-prep/common.yaml index 9d63af7..32c1083 100644 --- a/hieradata/labs/deployment-prep/common.yaml +++ b/hieradata/labs/deployment-prep/common.yaml @@ -73,6 +73,7 @@ restbase::graphoid_uri: http://deployment-sca01.deployment-prep.eqiad.wmflabs:19000 restbase::mobileapps_uri: http://appservice.wmflabs.org restbase::mathoid_uri: http://mathoid-tester.wmflabs.org +restbase::aqs_uri: https://wikimedia.org/api/rest_v1/metrics "mediawiki::log_aggregator": deployment-fluorine.eqiad.wmflabs:8420 "mediawiki::forward_syslog": deployment-logstash2.deployment-prep.eqiad.wmflabs:10514 mediawiki_memcached_servers: diff --git a/hieradata/role/common/restbase.yaml b/hieradata/role/common/restbase.yaml index 48ba4a3..691a9db 100644 --- a/hieradata/role/common/restbase.yaml +++ b/hieradata/role/common/restbase.yaml @@ -13,6 +13,7 @@ restbase::graphoid_uri: http://graphoid.svc.eqiad.wmnet:19000 restbase::mobileapps_uri: http://mobileapps.svc.eqiad.wmnet:8888 restbase::mathoid_uri: http://mathoid.svc.eqiad.wmnet:10042 +restbase::aqs_uri: http://aqs.svc.eqiad.wmnet:7231/analytics.wikimedia.org/v1 restbase::cassandra_user: restbase restbase::cassandra_password: "%{passwords::cassandra::restbase}" restbase::monitoring::monitor_domain: en.wikipedia.org diff --git a/modules/restbase/manifests/init.pp b/modules/restbase/manifests/init.pp index 3c6b85d..b6473e9 100644 --- a/modules/restbase/manifests/init.pp +++ b/modules/restbase/manifests/init.pp @@ -42,6 +42,9 @@ # MobileApps service URI. Default: http://mobileapps.svc.eqiad.wmnet:8888 # [*mathoid_uri*] # Mathoid service URI. Default: http://mathoid.svc.eqiad.wmnet:10042 +# [*aqs_uri*] +# Analytics Query Service URI. Default: +# http://aqs.svc.eqiad.wmnet:7231/analytics.wikimedia.org/v1 # class restbase( $cassandra_user = 'cassandra', @@ -63,6 +66,8 @@ $graphoid_uri = 'http://graphoid.svc.eqiad.wmnet:19000', $mobileapps_uri = 'http://mobileapps.svc.eqiad.wmnet:8888', $mathoid_uri = 'http://mathoid.svc.eqiad.wmnet:10042', + $aqs_uri = + 'http://aqs.svc.eqiad.wmnet:7231/analytics.wikimedia.org/v1', ) { if $::realm == 'labs' { include restbase::deploy diff --git a/modules/restbase/templates/config.labs.yaml.erb b/modules/restbase/templates/config.labs.yaml.erb index 47dde31..5a3d9fe 100644 --- a/modules/restbase/templates/config.labs.yaml.erb +++ b/modules/restbase/templates/config.labs.yaml.erb @@ -62,6 +62,7 @@ - <%= @graphoid_uri %> - <%= @mobileapps_uri %> - <%= @mathoid_uri %> + - <%= @aqs_uri %> header_match: description: Checks client ip against one of the predefined whitelists x-error-message: This client is not allowed to use the endpoint @@ -165,34 +166,160 @@ paths: /html/{title}: get: + x-setup-handler: + - init: + method: put + uri: /{domain}/sys/key_rev_value/mobileapps.mobile-html + body: + revisionRetentionPolicy: + type: 'latest' + count: 1 + grace_ttl: 86400 + valueType: 'json' + version: 1 x-request-handler: - - get_from_backend: + - handle_req: request: - uri: <%= @mobileapps_uri %>/{domain}/v1/page/mobile-html/{title} + method: get + uri: /{domain}/sys/mobileapps/handling/rev/mobile-html/{title} + headers: + cache-control: '{cache-control}' /sections/{title}: get: + x-setup-handler: + - init: + method: put + uri: /{domain}/sys/key_rev_value/mobileapps.mobile-html-sections + body: + revisionRetentionPolicy: + type: 'latest' + count: 1 + grace_ttl: 86400 + valueType: 'json' + version: 1 x-request-handler: - - get_from_backend: + - handle_req: request: - uri: <%= @mobileapps_uri %>/{domain}/v1/page/mobile-html-sections/{title} + method: get + uri: /{domain}/sys/mobileapps/handling/rev/mobile-html-sections/{title} + headers: + cache-control: '{cache-control}' /sections-lead/{title}: get: + x-setup-handler: + - init: + method: put + uri: /{domain}/sys/key_rev_value/mobileapps.mobile-html-sections-lead + body: + revisionRetentionPolicy: + type: 'latest' + count: 1 + grace_ttl: 86400 + valueType: 'json' + version: 1 x-request-handler: - - get_from_backend: + - handle_req: request: - uri: <%= @mobileapps_uri %>/{domain}/v1/page/mobile-html-sections-lead/{title} + method: get + uri: /{domain}/sys/mobileapps/handling/rev/mobile-html-sections-lead/{title} + headers: + cache-control: '{cache-control}' /sections-remaining/{title}: get: + x-setup-handler: + - init: + method: put + uri: /{domain}/sys/key_rev_value/mobileapps.mobile-html-sections-remaining + body: + revisionRetentionPolicy: + type: 'latest' + count: 1 + grace_ttl: 86400 + valueType: 'json' + version: 1 x-request-handler: - - get_from_backend: + - handle_req: request: - uri: <%= @mobileapps_uri %>/{domain}/v1/page/mobile-html-sections-remaining/{title} + method: get + uri: /{domain}/sys/mobileapps/handling/rev/mobile-html-sections-remaining/{title} + headers: + cache-control: '{cache-control}' /text/{title}: get: + x-setup-handler: + - init: + method: put + uri: /{domain}/sys/key_rev_value/mobileapps.mobile-text + body: + revisionRetentionPolicy: + type: 'latest' + count: 1 + grace_ttl: 86400 + valueType: 'json' + version: 1 x-request-handler: - - get_from_backend: + - handle_req: request: - uri: <%= @mobileapps_uri %>/{domain}/v1/page/mobile-text/{title} + method: get + uri: /{domain}/sys/mobileapps/handling/rev/mobile-text/{title} + headers: + cache-control: '{cache-control}' + /handling/rev/{route}/{title}: + get: + x-request-handler: + - get_rev: + request: + method: get + uri: /{domain}/sys/page_revisions/page/{title} + headers: + cache-control: '{cache-control}' + - cache_branch: + request: + method: post + uri: /{domain}/sys/mobileapps/handling/content/{$$.default($.request.headers.cache-control, 'none-given')}/{route}/{title} + body: '{$.get_rev.body}' + /handling/content/no-cache/{route}/{title}: + post: + x-request-handler: + - get_mobileapps: + request: + method: get + uri: <%= @mobileapps_uri %>/{domain}/v1/page/{route}/{title} + headers: + x-restbase-etag: '{$.request.body.items[0].rev}/{$.request.body.items[0].tid}' + - store: + request: + method: put + uri: /{domain}/sys/key_rev_value/mobileapps.{route}/{title}/{$.request.body.items[0].rev}/{$.request.body.items[0].tid} + headers: + content-type: application/json + body: + headers: '{$.get_mobileapps.headers}' + body: '{$.get_mobileapps.body}' + return: + status: 200 + headers: '{$.get_mobileapps.headers}' + body: '{$.get_mobileapps.body}' + /handling/content/{cache-default}/{route}/{title}: + post: + x-request-handler: + - check_storage: + request: + method: get + uri: /{domain}/sys/key_rev_value/mobileapps.{route}/{title}/{$.request.body.items[0].rev}/{$.request.body.items[0].tid} + return_if: + status: '2xx' + return: + status: 200 + headers: '{$.check_storage.body.headers}' + body: '{$.check_storage.body.body}' + catch: + status: 404 + - render: + request: + method: post + uri: /{domain}/sys/mobileapps/handling/content/no-cache/{route}/{title} + body: '{$.request.body}' global-content: &gb/content/1.0.0 @@ -201,6 +328,10 @@ securityDefinitions: *wp/content-security/1.0.0 x-subspecs: - media/v1/mathoid + paths: + /metrics: + x-subspecs: + - analytics/v1/pageviews global-sys: &gb/sys/1.0.0 @@ -293,6 +424,24 @@ headers: '{$$.merge($.mathoid.body[$.request.params.format].headers,$$.merge($.mathoid.headers, {"x-resource-location": $.request.headers.x-resource-location}))}' body: '{$.mathoid.body[$.request.params.format].body}' + /{module:pageviews}/per-article/{+rest}: + get: + x-request-handler: + - get_from_backend: + request: + uri: <%= @aqs_uri %>/pageviews/per-article/{+rest} + /{module:pageviews}/per-project/{+rest}: + get: + x-request-handler: + - get_from_backend: + request: + uri: <%= @aqs_uri %>/pageviews/aggregate/{+rest} + /{module:pageviews}/top/{+rest}: + get: + x-request-handler: + - get_from_backend: + request: + uri: <%= @aqs_uri %>/pageviews/top/{+rest} wp-default-1.0.0: &wp/default/1.0.0 x-subspecs: diff --git a/modules/restbase/templates/config.yaml.erb b/modules/restbase/templates/config.yaml.erb index 84c3d24..88922a8 100644 --- a/modules/restbase/templates/config.yaml.erb +++ b/modules/restbase/templates/config.yaml.erb @@ -62,6 +62,7 @@ - <%= @graphoid_uri %> - <%= @mobileapps_uri %> - <%= @mathoid_uri %> + - <%= @aqs_uri %> header_match: description: Checks client ip against one of the predefined whitelists x-error-message: This client is not allowed to use the endpoint @@ -182,34 +183,160 @@ paths: /html/{title}: get: + x-setup-handler: + - init: + method: put + uri: /{domain}/sys/key_rev_value/mobileapps.mobile-html + body: + revisionRetentionPolicy: + type: 'latest' + count: 1 + grace_ttl: 86400 + valueType: 'json' + version: 1 x-request-handler: - - get_from_backend: + - handle_req: request: - uri: <%= @mobileapps_uri %>/{domain}/v1/page/mobile-html/{title} + method: get + uri: /{domain}/sys/mobileapps/handling/rev/mobile-html/{title} + headers: + cache-control: '{cache-control}' /sections/{title}: get: + x-setup-handler: + - init: + method: put + uri: /{domain}/sys/key_rev_value/mobileapps.mobile-html-sections + body: + revisionRetentionPolicy: + type: 'latest' + count: 1 + grace_ttl: 86400 + valueType: 'json' + version: 1 x-request-handler: - - get_from_backend: + - handle_req: request: - uri: <%= @mobileapps_uri %>/{domain}/v1/page/mobile-html-sections/{title} + method: get + uri: /{domain}/sys/mobileapps/handling/rev/mobile-html-sections/{title} + headers: + cache-control: '{cache-control}' /sections-lead/{title}: get: + x-setup-handler: + - init: + method: put + uri: /{domain}/sys/key_rev_value/mobileapps.mobile-html-sections-lead + body: + revisionRetentionPolicy: + type: 'latest' + count: 1 + grace_ttl: 86400 + valueType: 'json' + version: 1 x-request-handler: - - get_from_backend: + - handle_req: request: - uri: <%= @mobileapps_uri %>/{domain}/v1/page/mobile-html-sections-lead/{title} + method: get + uri: /{domain}/sys/mobileapps/handling/rev/mobile-html-sections-lead/{title} + headers: + cache-control: '{cache-control}' /sections-remaining/{title}: get: + x-setup-handler: + - init: + method: put + uri: /{domain}/sys/key_rev_value/mobileapps.mobile-html-sections-remaining + body: + revisionRetentionPolicy: + type: 'latest' + count: 1 + grace_ttl: 86400 + valueType: 'json' + version: 1 x-request-handler: - - get_from_backend: + - handle_req: request: - uri: <%= @mobileapps_uri %>/{domain}/v1/page/mobile-html-sections-remaining/{title} + method: get + uri: /{domain}/sys/mobileapps/handling/rev/mobile-html-sections-remaining/{title} + headers: + cache-control: '{cache-control}' /text/{title}: get: + x-setup-handler: + - init: + method: put + uri: /{domain}/sys/key_rev_value/mobileapps.mobile-text + body: + revisionRetentionPolicy: + type: 'latest' + count: 1 + grace_ttl: 86400 + valueType: 'json' + version: 1 x-request-handler: - - get_from_backend: + - handle_req: request: - uri: <%= @mobileapps_uri %>/{domain}/v1/page/mobile-text/{title} + method: get + uri: /{domain}/sys/mobileapps/handling/rev/mobile-text/{title} + headers: + cache-control: '{cache-control}' + /handling/rev/{route}/{title}: + get: + x-request-handler: + - get_rev: + request: + method: get + uri: /{domain}/sys/page_revisions/page/{title} + headers: + cache-control: '{cache-control}' + - cache_branch: + request: + method: post + uri: /{domain}/sys/mobileapps/handling/content/{$$.default($.request.headers.cache-control, 'none-given')}/{route}/{title} + body: '{$.get_rev.body}' + /handling/content/no-cache/{route}/{title}: + post: + x-request-handler: + - get_mobileapps: + request: + method: get + uri: <%= @mobileapps_uri %>/{domain}/v1/page/{route}/{title} + headers: + x-restbase-etag: '{$.request.body.items[0].rev}/{$.request.body.items[0].tid}' + - store: + request: + method: put + uri: /{domain}/sys/key_rev_value/mobileapps.{route}/{title}/{$.request.body.items[0].rev}/{$.request.body.items[0].tid} + headers: + content-type: application/json + body: + headers: '{$.get_mobileapps.headers}' + body: '{$.get_mobileapps.body}' + return: + status: 200 + headers: '{$.get_mobileapps.headers}' + body: '{$.get_mobileapps.body}' + /handling/content/{cache-default}/{route}/{title}: + post: + x-request-handler: + - check_storage: + request: + method: get + uri: /{domain}/sys/key_rev_value/mobileapps.{route}/{title}/{$.request.body.items[0].rev}/{$.request.body.items[0].tid} + return_if: + status: '2xx' + return: + status: 200 + headers: '{$.check_storage.body.headers}' + body: '{$.check_storage.body.body}' + catch: + status: 404 + - render: + request: + method: post + uri: /{domain}/sys/mobileapps/handling/content/no-cache/{route}/{title} + body: '{$.request.body}' global-content: &gb/content/1.0.0 @@ -218,6 +345,10 @@ securityDefinitions: *wp/content-security/1.0.0 x-subspecs: - media/v1/mathoid + paths: + /metrics: + x-subspecs: + - analytics/v1/pageviews global-sys: &gb/sys/1.0.0 @@ -310,6 +441,24 @@ headers: '{$$.merge($.mathoid.body[$.request.params.format].headers,$$.merge($.mathoid.headers, {"x-resource-location": $.request.headers.x-resource-location}))}' body: '{$.mathoid.body[$.request.params.format].body}' + /{module:pageviews}/per-article/{+rest}: + get: + x-request-handler: + - get_from_backend: + request: + uri: <%= @aqs_uri %>/pageviews/per-article/{+rest} + /{module:pageviews}/per-project/{+rest}: + get: + x-request-handler: + - get_from_backend: + request: + uri: <%= @aqs_uri %>/pageviews/aggregate/{+rest} + /{module:pageviews}/top/{+rest}: + get: + x-request-handler: + - get_from_backend: + request: + uri: <%= @aqs_uri %>/pageviews/top/{+rest} wp-default-1.0.0: &wp/default/1.0.0 x-subspecs: -- To view, visit https://gerrit.wikimedia.org/r/247935 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I17ae36660ebb374e7062cd1e4ad4634ffddf66a7 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Mobrovac <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
