Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/231047
Change subject: Switch RESTBase to use SQLite3 storage
......................................................................
Switch RESTBase to use SQLite3 storage
There is now support for the SQLite back-end storage in RESTBase, so
switch to it in MW-Vagrant as it requires much less resources than
Cassandra.
Note: the Cassandra role has been kept around. The idea is that, in a
subsequent patch, we could automatically switch RESTBase to use
Cassandra if the role has been enabled by the user.
Bug: T107447
Change-Id: I656eb8a34914f9b4e8bdb2cca6e77aab04c9f7b8
---
M puppet/hieradata/common.yaml
M puppet/modules/npm/manifests/init.pp
M puppet/modules/restbase/manifests/init.pp
M puppet/modules/restbase/templates/config.yaml.erb
M puppet/modules/role/settings/restbase.yaml
5 files changed, 114 insertions(+), 26 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant
refs/changes/47/231047/1
diff --git a/puppet/hieradata/common.yaml b/puppet/hieradata/common.yaml
index c3f40ef..e6881d4 100644
--- a/puppet/hieradata/common.yaml
+++ b/puppet/hieradata/common.yaml
@@ -244,6 +244,7 @@
restbase::port: 7231
restbase::domain: localhost
+restbase::dbdir: /var/lib/restbase
role::commons::upload_dir: "%{hiera('mwv::files_dir')}/commonsimages"
diff --git a/puppet/modules/npm/manifests/init.pp
b/puppet/modules/npm/manifests/init.pp
index af53b0c..bb7879e 100644
--- a/puppet/modules/npm/manifests/init.pp
+++ b/puppet/modules/npm/manifests/init.pp
@@ -13,6 +13,19 @@
) {
require_package('nodejs', 'npm', 'nodejs-legacy')
+ exec { 'npm_global_modules':
+ command => 'npm install -g mocha grunt node-gyp node-pre-gyp',
+ user => 'root',
+ group => 'root',
+ creates => '/usr/local/lib/node_modules/node-pre-gyp',
+ environment => [
+ "NPM_CONFIG_CACHE=${cache_dir}",
+ 'NPM_CONFIG_GLOBAL=false',
+ 'LINK=g++',
+ ],
+ require => Package['nodejs', 'npm', 'nodejs-legacy'],
+ }
+
env::var { 'NPM_CONFIG_CACHE':
value => $cache_dir,
}
diff --git a/puppet/modules/restbase/manifests/init.pp
b/puppet/modules/restbase/manifests/init.pp
index 8ff53a7..75e6e6c 100644
--- a/puppet/modules/restbase/manifests/init.pp
+++ b/puppet/modules/restbase/manifests/init.pp
@@ -1,8 +1,9 @@
# == Class: restbase
#
# RESTBase is a REST API service serving MW content from
-# a Cassandra storage, proxying requests to Parsoid in
-# case of storage misses.
+# storage (Cassandra or SQLite, here the latter), proxying
+# requests to various back-end services in case of storage
+# misses.
#
# [*port*]
# the port RESTBase will be running on
@@ -10,16 +11,34 @@
# [*domain*]
# the domain to serve
#
+# [*dbdir*]
+# the directory where to place the SQLite database file
+#
# [*log_level*]
-# the lowest level to log (trace, debug, info, warn, error, fatal)
+# the lowest level to log (trace, debug, info, warn, error, fatal)
#
class restbase (
$port,
$domain,
+ $dbdir,
$log_level = undef,
) {
- require ::cassandra
require ::mediawiki::parsoid
+
+ require_package('libsqlite3-dev')
+
+ $graphoid_port = defined(Class['graphoid']) ? {
+ true => $::graphoid::port,
+ default => 11042,
+ }
+
+ file { $dbdir:
+ ensure => directory,
+ owner => 'www-data',
+ group => 'www-data',
+ mode => '0775',
+ before => Service::Node['restbase'],
+ }
service::node { 'restbase':
port => $port,
@@ -27,6 +46,7 @@
git_remote => 'https://github.com/wikimedia/restbase.git',
log_level => $log_level,
config => template('restbase/config.yaml.erb'),
+ require => Package['libsqlite3-dev'],
}
}
diff --git a/puppet/modules/restbase/templates/config.yaml.erb
b/puppet/modules/restbase/templates/config.yaml.erb
index 61589c4..671b08b 100644
--- a/puppet/modules/restbase/templates/config.yaml.erb
+++ b/puppet/modules/restbase/templates/config.yaml.erb
@@ -8,7 +8,7 @@
version: 1.0.0-beta
title: Wikimedia REST API
description: >
- This API aims to provide coherent and low-latency access to
+ This API aims to provide straightforward and low-latency access to
Wikimedia content and services. It is currently in beta testing, so
things aren't completely locked down yet. Each entry point has
explicit stability markers to inform you about development status
@@ -16,9 +16,10 @@
policy](https://www.mediawiki.org/wiki/API_versioning).
### High-volume access
- - Don't perform more than 500 requests/s to this API.
+ - As a general rule, don't perform more than 200 requests/s to
+ this API.
- Set a unique `User-Agent` header that allows us to contact you
- quickly. Email addresses or URLs of contact pages work well.
+ quickly. Email addresses or URLs of contact pages work well.
- Consider using our [HTML
dumps](https://phabricator.wikimedia.org/T17017) once they
become available.
@@ -36,7 +37,8 @@
- user:read
x-subspecs:
- mediawiki/v1/content
- # - mediawiki/v1/mobile
+ - mediawiki_v1_graphoid
+ - mediawiki/v1/mobileapps
# - mediawiki/v1/revision-scoring
wmf-sys-1.0.0: &wp/sys/1.0.0
@@ -44,36 +46,25 @@
title: Default MediaWiki sys API module
version: 1.0.0
paths:
- /{module:table}: &wp/sys/table # Can use this anchor to share the table
+ /{module:table}:
x-modules:
# There can be multiple modules too per stanza, as long as the
# exported symbols don't conflict. The operationIds from the spec
# will be resolved against all of the modules.
- - name: restbase-mod-table-cassandra
+ - name: restbase-mod-table-sqlite
version: 1.0.0
type: npm
options: # Passed to the module constructor
conf:
- hosts: [localhost]
- keyspace: system
- username: cassandra
- password: cassandra
- defaultConsistency: one # or 'one' for single-node testing
- storage_groups:
- - name: test.group.local
- domains:
- - /test\..*\.org$/
- - /test\.local$/
- - name: default.group.local
- domains: /./
+ dbname: <%= @dbdir %>/mw-vagrant.sqlite3
- /{module:page_revisions}: &wp-page-revisions
+ /{module:page_revisions}:
x-modules:
- name: page_revisions
version: 1.0.0
type: file
- /{module:key_rev_value}: &wp/sys/key_rev_value
+ /{module:key_rev_value}:
x-modules:
- name: key_rev_value
version: 1.0.0
@@ -92,7 +83,69 @@
- name: action
type: file
options:
- apiURI: http://{domain}/w/api.php
+ apiRequest:
+ method: post
+ uri: '<%= scope['::mediawiki::server_url'] %>/w/api.php'
+ headers:
+ host: '{$.request.params.domain}'
+ body: '{$.request.body}'
+
+ /{module:graphoid}:
+ x-modules:
+ - name: simple_service
+ version: 1.0.0
+ type: file
+ options:
+ paths:
+ /v1/png/{title}/{revision}/{graph_id}:
+ get:
+ on_request:
+ - get_from_graphoid:
+ request:
+ uri: http://localhost:<%= @graphoid_port
%>/{domain}/v1/png/{title}/{revision}/{graph_id}
+
+ /{module:mobileapps}:
+ x-modules:
+ - name: simple_service
+ version: 1.0.0
+ type: file
+ options:
+ paths:
+ /v1/html/{title}:
+ get:
+ on_request:
+ - get_from_backend:
+ request:
+ uri:
http://appservice.wmflabs.org/{domain}/v1/page/mobile-html/{title}
+ /v1/sections/{title}:
+ get:
+ on_request:
+ - get_from_backend:
+ request:
+ uri:
http://appservice.wmflabs.org/{domain}/v1/page/mobile-html-sections/{title}
+ /v1/sections-lead/{title}:
+ get:
+ on_request:
+ - get_from_backend:
+ request:
+ uri:
http://appservice.wmflabs.org/{domain}/v1/page/mobile-html-sections-lead/{title}
+ /v1/sections-remaining/{title}:
+ get:
+ on_request:
+ - get_from_backend:
+ request:
+ uri:
http://appservice.wmflabs.org/{domain}/v1/page/mobile-html-sections-remaining/{title}
+ /v1/text/{title}:
+ get:
+ on_request:
+ - get_from_backend:
+ request:
+ uri:
http://appservice.wmflabs.org/{domain}/v1/page/mobile-text/{title}
+
+ /{module:page_save}:
+ x-modules:
+ - name: page_save
+ type: file
# /{module:revscore}:
# title: Simple revscore service wrapper
@@ -125,5 +178,7 @@
# Some more general RESTBase info
paths:
/{domain:<%= @domain %>}: *wp/default/1.0.0
+
salt: secret
default_page_size: 100
+user_agent: RESTBase
diff --git a/puppet/modules/role/settings/restbase.yaml
b/puppet/modules/role/settings/restbase.yaml
index 7a250fb..b260857 100644
--- a/puppet/modules/role/settings/restbase.yaml
+++ b/puppet/modules/role/settings/restbase.yaml
@@ -1,4 +1,3 @@
-vagrant_ram: 768
forward_ports:
7231: 7231
--
To view, visit https://gerrit.wikimedia.org/r/231047
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I656eb8a34914f9b4e8bdb2cca6e77aab04c9f7b8
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: master
Gerrit-Owner: Mobrovac <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
