Smalyshev has uploaded a new change for review.
https://gerrit.wikimedia.org/r/216403
Change subject: Add definitions for WDQS
......................................................................
Add definitions for WDQS
Change-Id: I21e17e5e7d56d536f4cc34894ddb4fd0a7ea9846
---
A manifests/role/wdqs.pp
A modules/wdqs/manifests/gui.pp
A modules/wdqs/manifests/init.pp
A modules/wdqs/manifests/packages.pp
A modules/wdqs/manifests/service.pp
A modules/wdqs/templates/nginx.erb
6 files changed, 210 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/03/216403/1
diff --git a/manifests/role/wdqs.pp b/manifests/role/wdqs.pp
new file mode 100644
index 0000000..070b9df
--- /dev/null
+++ b/manifests/role/wdqs.pp
@@ -0,0 +1,27 @@
+# = Class: role::wdqs
+#
+# This class sets up Wikidata Query Service
+#
+class role::wdqs{
+
+ include standard
+# include role::labs::lvm::srv
+
+ system::role { 'role::wdqs':
+ ensure => 'present',
+ description => 'WDQS server',
+ }
+
+ # Install service
+ class { '::wdqs':
+ package_dir => '/srv',
+ username => 'blazegraph',
+ }
+
+ # Service GUI
+ class { '::wdqs::gui':
+ package_dir => '/srv/blazegraph',
+ log_aggregator => '10.68.16.147:10514',
+ require => Class['::wdqs']
+ }
+}
diff --git a/modules/wdqs/manifests/gui.pp b/modules/wdqs/manifests/gui.pp
new file mode 100644
index 0000000..6120d42
--- /dev/null
+++ b/modules/wdqs/manifests/gui.pp
@@ -0,0 +1,31 @@
+# == Class: wdqs::packages
+#
+# Provisions WDQS GUI
+#
+# == Parameters:
+# - $package_dir: Directory where the service is installed.
+# GUI files are expected to be under its gui/ directory.
+# - $log_aggregator: Where to send the logs for the service.
+class wdqs::gui(
+ $package_dir,
+ $log_aggregator
+) {
+
+ if ! defined ( Package['nginx'] ) {
+ package { 'nginx': ensure => present }
+ }
+ file { '/etc/nginx/sites-enabled/default':
+ ensure => file,
+ owner => 'root',
+ group => 'root',
+ content => template('wdqs/nginx.erb'),
+ mode => '0444',
+ require => Package['nginx'],
+ notify => Service['nginx'],
+ }
+ service { 'nginx':
+ enable => true,
+ require => File['/etc/nginx/sites-enabled/default']
+ }
+
+}
\ No newline at end of file
diff --git a/modules/wdqs/manifests/init.pp b/modules/wdqs/manifests/init.pp
new file mode 100644
index 0000000..9dcc29d
--- /dev/null
+++ b/modules/wdqs/manifests/init.pp
@@ -0,0 +1,49 @@
+# = Class: wdqs
+# Note: this does not run the service, just installs it
+# In order to run it, dump data must be loaded, which is
+# for now a manual process.
+#
+# == Parameters:
+# - $package_dir: Directory where the service should be installed.
+# Should have enough space to hold the database (>50G)
+# - $username: Username owning the service
+class wdqs($package_dir = "/srv",
+ $username = "blazegraph"
+) {
+ $version = "0.0.2" # here for now, find better place
+
+ include ::wdqs::packages
+ class { '::wdqs::service':
+ package_dir => $package_dir,
+ version => $version,
+ username => $username,
+ }
+
+ file { $package_dir:
+ path => $package_dir,
+ ensure => directory,
+ purge => false,
+ owner => $username,
+ group => 'wikidev',
+ mode => 0775,
+ require => User['blazegraph'],
+ }
+
+ user { 'blazegraph':
+ name => $username,
+ ensure => present,
+ comment => 'Blazegraph user',
+ forcelocal => true,
+ home => $package_dir,
+ managehome => no,
+ }
+
+ file { "$package_dir/blazegraph":
+ ensure => link,
+ replace => false, # do not replace existing links, version
upgrade is manual for now
+ target => "$package_dir/service-$version",
+ require => File["$package_dir/service-$version"]
+ }
+
+}
+
diff --git a/modules/wdqs/manifests/packages.pp
b/modules/wdqs/manifests/packages.pp
new file mode 100644
index 0000000..613f994
--- /dev/null
+++ b/modules/wdqs/manifests/packages.pp
@@ -0,0 +1,22 @@
+# == Class: wdqs::packages
+#
+# Provisions WDQS package and dependencies.
+#
+class wdqs::packages {
+ include ::java::tools
+
+ if ! defined ( Package['openjdk-7-jdk'] ) {
+ package { 'openjdk-7-jdk':
+ ensure => 'present',
+ }
+ }
+
+ if ! defined ( Package['curl'] ) {
+ package { 'curl': ensure => present }
+ }
+
+ if ! defined ( Package['maven'] ) {
+ package { 'maven': ensure => present }
+ }
+
+}
\ No newline at end of file
diff --git a/modules/wdqs/manifests/service.pp
b/modules/wdqs/manifests/service.pp
new file mode 100644
index 0000000..d576992
--- /dev/null
+++ b/modules/wdqs/manifests/service.pp
@@ -0,0 +1,35 @@
+# == Class: wdqs::service
+#
+# Provisions WDQS service package
+#
+# == Parameters:
+# - $package_dir: Directory where the service should be installed.
+# Should have enough space to hold the database (>50G)
+# - $version: Version of the service to install
+# - $username: Username owning the service
+class wdqs::service(
+ $package_dir,
+ $version,
+ $username
+) {
+ # Horrible hack, find better way?
+ exec { 'download_package':
+ command => "/usr/bin/mvn
org.apache.maven.plugins:maven-dependency-plugin:2.10:unpack
-Dartifact=org.wikidata.query.rdf:service:${version}:zip:dist
-DoutputDirectory=${package_dir} -Dtransitive=false -Dproject.basedir=/tmp",
+ cwd => '/tmp',
+ creates => "${package_dir}/service-${version}",
+ require => Package['maven'],
+ before => File['/tmp/target'],
+ }
+
+ file { '/tmp/target':
+ ensure => absent,
+ force => true,
+ }
+
+ file { "${package_dir}/service-${version}":
+ owner => $username,
+ group => 'wikidev',
+ mode => 0775,
+ require => Exec['download_package']
+ }
+}
\ No newline at end of file
diff --git a/modules/wdqs/templates/nginx.erb b/modules/wdqs/templates/nginx.erb
new file mode 100644
index 0000000..6617303
--- /dev/null
+++ b/modules/wdqs/templates/nginx.erb
@@ -0,0 +1,46 @@
+server {
+ listen 80 default_server;
+ listen [::]:80 default_server;
+
+ log_format logstash
+ '[$time_local] '
+ '"$request" $status $body_bytes_sent '
+ '"$http_referer" "$http_user_agent" '
+ '$request_time '
+ '$upstream_response_time';
+
+ location / {
+ root <%= @package_dir %>/gui;
+ }
+
+ location /bigdata {
+ proxy_pass http://localhost:9999;
+
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_set_header X-BIGDATA-MAX-QUERY-MILLIS 30000;
+
+ add_header Access-Control-Allow-Origin *;
+
+ client_max_body_size 10M;
+ client_body_buffer_size 1m;
+ proxy_intercept_errors on;
+ proxy_buffering on;
+ proxy_buffer_size 128k;
+ proxy_buffers 256 16k;
+ proxy_busy_buffers_size 256k;
+ proxy_temp_file_write_size 256k;
+ proxy_max_temp_file_size 0;
+ proxy_read_timeout 300;
+
+ limit_except GET OPTIONS {
+ deny all;
+ }
+
+ <% if @log_aggregator -%>
+ # access_log syslog:server=<%= @log_aggregator
%>,facility=local7,tag=wdqs,severity=info logstash;
+ <% end -%>
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/216403
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I21e17e5e7d56d536f4cc34894ddb4fd0a7ea9846
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Smalyshev <smalys...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
