jenkins-bot has submitted this change and it was merged.
Change subject: Switch RESTBase to use SQLite3 storage
......................................................................
Switch RESTBase to use SQLite3 storage
There is now support for the SQLite back-end storage in RESTBase, so
switch to it in MW-Vagrant as it requires much less resources than
Cassandra.
Note: the Cassandra role has been kept around. The idea is that, in a
subsequent patch, we could automatically switch RESTBase to use
Cassandra if the role has been enabled by the user.
Bug: T107447
Change-Id: I656eb8a34914f9b4e8bdb2cca6e77aab04c9f7b8
---
M puppet/hieradata/common.yaml
A puppet/modules/npm/manifests/global.pp
A puppet/modules/npm/manifests/globals.pp
M puppet/modules/restbase/manifests/init.pp
M puppet/modules/restbase/templates/config.yaml.erb
M puppet/modules/role/settings/restbase.yaml
M puppet/modules/service/manifests/gitupdate.pp
M puppet/modules/service/manifests/node.pp
8 files changed, 130 insertions(+), 28 deletions(-)
Approvals:
Dduvall: Looks good to me, approved
jenkins-bot: Verified
diff --git a/puppet/hieradata/common.yaml b/puppet/hieradata/common.yaml
index c3f40ef..e6881d4 100644
--- a/puppet/hieradata/common.yaml
+++ b/puppet/hieradata/common.yaml
@@ -244,6 +244,7 @@
restbase::port: 7231
restbase::domain: localhost
+restbase::dbdir: /var/lib/restbase
role::commons::upload_dir: "%{hiera('mwv::files_dir')}/commonsimages"
diff --git a/puppet/modules/npm/manifests/global.pp
b/puppet/modules/npm/manifests/global.pp
new file mode 100644
index 0000000..17cb153
--- /dev/null
+++ b/puppet/modules/npm/manifests/global.pp
@@ -0,0 +1,19 @@
+# == Define: npm::global
+#
+# Resource for installing node.js modules globally
+#
+define npm::global {
+ require ::npm
+
+ exec { "npm_global_${title}":
+ command => "/usr/bin/npm install -g ${title}",
+ user => 'root',
+ group => 'root',
+ creates => "/usr/local/lib/node_modules/${title}",
+ environment => [
+ "NPM_CONFIG_CACHE=${::npm::cache_dir}",
+ 'NPM_CONFIG_GLOBAL=false',
+ 'LINK=g++',
+ ],
+ }
+}
diff --git a/puppet/modules/npm/manifests/globals.pp
b/puppet/modules/npm/manifests/globals.pp
new file mode 100644
index 0000000..0ea3d06
--- /dev/null
+++ b/puppet/modules/npm/manifests/globals.pp
@@ -0,0 +1,7 @@
+# == Class: npm::globals
+#
+# Installs some commonly used NPM modules globally.
+#
+class npm::globals {
+ npm::global { ['mocha', 'grunt', 'node-gyp', 'node-pre-gyp']: }
+}
diff --git a/puppet/modules/restbase/manifests/init.pp
b/puppet/modules/restbase/manifests/init.pp
index 8ff53a7..75e6e6c 100644
--- a/puppet/modules/restbase/manifests/init.pp
+++ b/puppet/modules/restbase/manifests/init.pp
@@ -1,8 +1,9 @@
# == Class: restbase
#
# RESTBase is a REST API service serving MW content from
-# a Cassandra storage, proxying requests to Parsoid in
-# case of storage misses.
+# storage (Cassandra or SQLite, here the latter), proxying
+# requests to various back-end services in case of storage
+# misses.
#
# [*port*]
# the port RESTBase will be running on
@@ -10,16 +11,34 @@
# [*domain*]
# the domain to serve
#
+# [*dbdir*]
+# the directory where to place the SQLite database file
+#
# [*log_level*]
-# the lowest level to log (trace, debug, info, warn, error, fatal)
+# the lowest level to log (trace, debug, info, warn, error, fatal)
#
class restbase (
$port,
$domain,
+ $dbdir,
$log_level = undef,
) {
- require ::cassandra
require ::mediawiki::parsoid
+
+ require_package('libsqlite3-dev')
+
+ $graphoid_port = defined(Class['graphoid']) ? {
+ true => $::graphoid::port,
+ default => 11042,
+ }
+
+ file { $dbdir:
+ ensure => directory,
+ owner => 'www-data',
+ group => 'www-data',
+ mode => '0775',
+ before => Service::Node['restbase'],
+ }
service::node { 'restbase':
port => $port,
@@ -27,6 +46,7 @@
git_remote => 'https://github.com/wikimedia/restbase.git',
log_level => $log_level,
config => template('restbase/config.yaml.erb'),
+ require => Package['libsqlite3-dev'],
}
}
diff --git a/puppet/modules/restbase/templates/config.yaml.erb
b/puppet/modules/restbase/templates/config.yaml.erb
index 61589c4..671b08b 100644
--- a/puppet/modules/restbase/templates/config.yaml.erb
+++ b/puppet/modules/restbase/templates/config.yaml.erb
@@ -8,7 +8,7 @@
version: 1.0.0-beta
title: Wikimedia REST API
description: >
- This API aims to provide coherent and low-latency access to
+ This API aims to provide straightforward and low-latency access to
Wikimedia content and services. It is currently in beta testing, so
things aren't completely locked down yet. Each entry point has
explicit stability markers to inform you about development status
@@ -16,9 +16,10 @@
policy](https://www.mediawiki.org/wiki/API_versioning).
### High-volume access
- - Don't perform more than 500 requests/s to this API.
+ - As a general rule, don't perform more than 200 requests/s to
+ this API.
- Set a unique `User-Agent` header that allows us to contact you
- quickly. Email addresses or URLs of contact pages work well.
+ quickly. Email addresses or URLs of contact pages work well.
- Consider using our [HTML
dumps](https://phabricator.wikimedia.org/T17017) once they
become available.
@@ -36,7 +37,8 @@
- user:read
x-subspecs:
- mediawiki/v1/content
- # - mediawiki/v1/mobile
+ - mediawiki_v1_graphoid
+ - mediawiki/v1/mobileapps
# - mediawiki/v1/revision-scoring
wmf-sys-1.0.0: &wp/sys/1.0.0
@@ -44,36 +46,25 @@
title: Default MediaWiki sys API module
version: 1.0.0
paths:
- /{module:table}: &wp/sys/table # Can use this anchor to share the table
+ /{module:table}:
x-modules:
# There can be multiple modules too per stanza, as long as the
# exported symbols don't conflict. The operationIds from the spec
# will be resolved against all of the modules.
- - name: restbase-mod-table-cassandra
+ - name: restbase-mod-table-sqlite
version: 1.0.0
type: npm
options: # Passed to the module constructor
conf:
- hosts: [localhost]
- keyspace: system
- username: cassandra
- password: cassandra
- defaultConsistency: one # or 'one' for single-node testing
- storage_groups:
- - name: test.group.local
- domains:
- - /test\..*\.org$/
- - /test\.local$/
- - name: default.group.local
- domains: /./
+ dbname: <%= @dbdir %>/mw-vagrant.sqlite3
- /{module:page_revisions}: &wp-page-revisions
+ /{module:page_revisions}:
x-modules:
- name: page_revisions
version: 1.0.0
type: file
- /{module:key_rev_value}: &wp/sys/key_rev_value
+ /{module:key_rev_value}:
x-modules:
- name: key_rev_value
version: 1.0.0
@@ -92,7 +83,69 @@
- name: action
type: file
options:
- apiURI: http://{domain}/w/api.php
+ apiRequest:
+ method: post
+ uri: '<%= scope['::mediawiki::server_url'] %>/w/api.php'
+ headers:
+ host: '{$.request.params.domain}'
+ body: '{$.request.body}'
+
+ /{module:graphoid}:
+ x-modules:
+ - name: simple_service
+ version: 1.0.0
+ type: file
+ options:
+ paths:
+ /v1/png/{title}/{revision}/{graph_id}:
+ get:
+ on_request:
+ - get_from_graphoid:
+ request:
+ uri: http://localhost:<%= @graphoid_port
%>/{domain}/v1/png/{title}/{revision}/{graph_id}
+
+ /{module:mobileapps}:
+ x-modules:
+ - name: simple_service
+ version: 1.0.0
+ type: file
+ options:
+ paths:
+ /v1/html/{title}:
+ get:
+ on_request:
+ - get_from_backend:
+ request:
+ uri:
http://appservice.wmflabs.org/{domain}/v1/page/mobile-html/{title}
+ /v1/sections/{title}:
+ get:
+ on_request:
+ - get_from_backend:
+ request:
+ uri:
http://appservice.wmflabs.org/{domain}/v1/page/mobile-html-sections/{title}
+ /v1/sections-lead/{title}:
+ get:
+ on_request:
+ - get_from_backend:
+ request:
+ uri:
http://appservice.wmflabs.org/{domain}/v1/page/mobile-html-sections-lead/{title}
+ /v1/sections-remaining/{title}:
+ get:
+ on_request:
+ - get_from_backend:
+ request:
+ uri:
http://appservice.wmflabs.org/{domain}/v1/page/mobile-html-sections-remaining/{title}
+ /v1/text/{title}:
+ get:
+ on_request:
+ - get_from_backend:
+ request:
+ uri:
http://appservice.wmflabs.org/{domain}/v1/page/mobile-text/{title}
+
+ /{module:page_save}:
+ x-modules:
+ - name: page_save
+ type: file
# /{module:revscore}:
# title: Simple revscore service wrapper
@@ -125,5 +178,7 @@
# Some more general RESTBase info
paths:
/{domain:<%= @domain %>}: *wp/default/1.0.0
+
salt: secret
default_page_size: 100
+user_agent: RESTBase
diff --git a/puppet/modules/role/settings/restbase.yaml
b/puppet/modules/role/settings/restbase.yaml
index 7a250fb..b260857 100644
--- a/puppet/modules/role/settings/restbase.yaml
+++ b/puppet/modules/role/settings/restbase.yaml
@@ -1,4 +1,3 @@
-vagrant_ram: 768
forward_ports:
7231: 7231
diff --git a/puppet/modules/service/manifests/gitupdate.pp
b/puppet/modules/service/manifests/gitupdate.pp
index 0279098..949918d 100644
--- a/puppet/modules/service/manifests/gitupdate.pp
+++ b/puppet/modules/service/manifests/gitupdate.pp
@@ -75,7 +75,7 @@
# descern the update command to use
$up_cmd = $type ? {
'php' => 'composer update --no-interaction --optimize-autoloader',
- 'nodejs' => 'npm update --no-bin-links',
+ 'nodejs' => 'npm install --no-bin-links',
default => 'invalid'
}
if $update and $up_cmd == 'invalid' {
diff --git a/puppet/modules/service/manifests/node.pp
b/puppet/modules/service/manifests/node.pp
index d95fd5c..376aa09 100644
--- a/puppet/modules/service/manifests/node.pp
+++ b/puppet/modules/service/manifests/node.pp
@@ -57,7 +57,8 @@
) {
require ::service
- require_package( 'nodejs-legacy' )
+
+ require ::npm::globals
# we do not allow empty names
unless $title and size($title) > 0 {
--
To view, visit https://gerrit.wikimedia.org/r/231047
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I656eb8a34914f9b4e8bdb2cca6e77aab04c9f7b8
Gerrit-PatchSet: 4
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: master
Gerrit-Owner: Mobrovac <[email protected]>
Gerrit-Reviewer: BryanDavis <[email protected]>
Gerrit-Reviewer: Dduvall <[email protected]>
Gerrit-Reviewer: Mobrovac <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
