Ottomata has uploaded a new change for review.
https://gerrit.wikimedia.org/r/288099
Change subject: [WIP] Druid module
......................................................................
[WIP] Druid module
Bug: T131974
Change-Id: I65cb169ac2701e754cd6a68e31903bcb25644290
---
A modules/druid/manifests/broker.pp
A modules/druid/manifests/coordinator.pp
A modules/druid/manifests/historical.pp
A modules/druid/manifests/init.pp
A modules/druid/manifests/middlemanager.pp
A modules/druid/manifests/overlord.pp
A modules/druid/templates/env.sh.erb
A modules/druid/templates/log4j2.xml.erb
A modules/druid/templates/runtime.properties.erb
9 files changed, 378 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/99/288099/1
diff --git a/modules/druid/manifests/broker.pp
b/modules/druid/manifests/broker.pp
new file mode 100644
index 0000000..4e79040
--- /dev/null
+++ b/modules/druid/manifests/broker.pp
@@ -0,0 +1,72 @@
+# == Class druid::broker
+#
+
+# [*druid.service*]
+
+# Default: druid/broker
+# [*druid.port*]
+
+# Default: 8082
+
+# HTTP server threads
+# [*druid.broker.http.numConnections*]
+
+# Default: 4
+# [*druid.server.http.numThreads*]
+
+# Default: 4
+
+# Processing threads and buffers
+# [*druid.processing.buffer.sizeBytes*]
+
+# Default: 134217728
+# [*druid.processing.numThreads*]
+
+# Default: 1
+
+# Query cache (we use a small local cache)
+# [*druid.broker.cache.useCache*]
+
+# Default: true
+# [*druid.broker.cache.populateCache*]
+
+# Default: true
+# [*druid.cache.type*]
+
+# Default: local
+# [*druid.cache.sizeInBytes*]
+
+# Default: 10000000
+class druid::broker(
+ $properties = {},
+ $env = {},
+)
+{
+ require druid
+
+ $default_properties = {
+ 'druid.port' => 8082,
+ # HTTP server threads,
+ 'druid.broker.http.numConnections' => '4',
+ 'druid.server.http.numThreads' => '4',
+ # Processing threads and buffers,
+ 'druid.processing.buffer.sizeBytes' => '134217728',
+ 'druid.processing.numThreads' => '1',
+ # Query cache (we use a small local cache),
+ 'druid.broker.cache.useCache' => 'true',
+ 'druid.broker.cache.populateCache' => 'true',
+ 'druid.cache.type' => 'local',
+ 'druid.cache.sizeInBytes' => '10000000'
+ }
+
+ $default_env = {
+ 'JMX_PORT' => 9661,
+ 'DRUID_HEAP_OPTS' => '-Xmx256m -Xms256m',
+ 'DRUID_EXTRA_JVM_OPTS' => '-XX:MaxDirectMemorySize=256m',
+ }
+
+ druid::service { 'broker':
+ runtime_properties => merge($default_properties, $properties),
+ env => merge($default_env, $env),
+ }
+}
diff --git a/modules/druid/manifests/coordinator.pp
b/modules/druid/manifests/coordinator.pp
new file mode 100644
index 0000000..888c372
--- /dev/null
+++ b/modules/druid/manifests/coordinator.pp
@@ -0,0 +1,26 @@
+# == Class druid::coordinator
+#
+class druid::coordinator(
+ $properties = {},
+ $env = {},
+)
+{
+ require druid
+
+ $default_properties = {
+ 'druid.port' => 8081,
+ 'druid.coordinator.startDelay' => 'PT30S',
+ 'druid.coordinator.period' => 'PT30S',
+ }
+
+ $default_env = {
+ 'JMX_PORT' => 9662,
+ 'DRUID_HEAP_OPTS' => '-Xmx128m -Xms128m',
+ 'DRUID_EXTRA_JVM_OPTS' =>
'-Dderby.stream.error.file=/var/log/druid/coordinator-derby.log',
+ }
+
+ druid::service { 'coordinator':
+ runtime_properties => merge($default_properties, $properties),
+ env => merge($default_env, $env),
+ }
+}
diff --git a/modules/druid/manifests/historical.pp
b/modules/druid/manifests/historical.pp
new file mode 100644
index 0000000..26cdfd7
--- /dev/null
+++ b/modules/druid/manifests/historical.pp
@@ -0,0 +1,34 @@
+# == Class druid::historical
+#
+class druid::historical(
+ $properties = {},
+ $env = {},
+)
+{
+ require druid
+
+ $default_properties = {
+ 'druid.port' => 8083,
+ # HTTP server threads
+ 'druid.server.http.numThreads' => 4,
+
+ # Processing threads and buffers
+ 'druid.processing.buffer.sizeBytes' => 134217728,
+ 'druid.processing.numThreads' => 1,
+
+ # Segment storage
+ 'druid.segmentCache.locations' =>
'[{"path":"var/lib/druid/segment-cache","maxSize"\:2147483648}]',
+ 'druid.server.maxSize' => 2147483648,
+ }
+
+ $default_env = {
+ 'JMX_PORT' => 9663,
+ 'DRUID_HEAP_OPTS' => '-Xmx256m -Xms256m',
+ 'DRUID_EXTRA_JVM_OPTS' => '-XX:MaxDirectMemorySize=256m',
+ }
+
+ druid::service { 'historical':
+ runtime_properties => merge($default_properties, $properties),
+ env => merge($default_env, $env),
+ }
+}
diff --git a/modules/druid/manifests/init.pp b/modules/druid/manifests/init.pp
new file mode 100644
index 0000000..5d85243
--- /dev/null
+++ b/modules/druid/manifests/init.pp
@@ -0,0 +1,150 @@
+# == Class druid
+#
+# Installs druid-common and configures common runtime properties.
+#
+# == Properties
+# Note that the properties listed here are only the defaults.
+# See http://druid.io/docs/latest/configuration/ for a
+# list of available properties.
+#
+#
+# [*druid.extensions.directory*]
+# Druid extensions are installed here. Only extensions listed in
+# druid.extensions.loadList will be automatically loaded into the classpath.
+# Default: /usr/share/druid/extensions
+#
+# [*druid.extensions.loadList*]
+# List extensions to load. Directories matching these names must exist
+# in druid.extensions.directory.
+# Default: ["druid-histogram", "druid-datasketches",
"druid-namespace-lookup"]
+#
+# [*druid.extensions.hadoopDependenciesDir*]
+# If you have a different version of Hadoop, place your Hadoop client jar
+# files in your hadoop-dependencies directory and uncomment the line below to
+# point to your directory. Or you may manually include them in
+# DRUID_CLASSPATH.
+# Default: /usr/share/druid/hadoop-dependencies
+#
+# [*druid.startup.logging.logProperties*]
+# Log all runtime properties on startup. Disable to avoid logging properties
+# on startup. Default: true
+#
+# [*druid.zk.service.host*]
+# Zookeeper hostnames. Default: localhost:2181
+#
+# [*druid.zk.paths.base*]
+# Chroot to druid in zookeeper. Default: /druid
+#
+# [*druid.metadata.storage.type*]
+# For Derby server on your Druid Coordinator (only viable in a cluster with
+# single Coordinator, no fail-over). Default: derby
+#
+# [*druid.metadata.storage.connector.connectURI*]
+# Default: jdbc:derby://localhost:1527/var/lib/druid/metadata.db;create=true
+#
+# [*druid.metadata.storage.connector.host*]
+# Default: localhost
+#
+# [*druid.metadata.storage.connector.port*]
+# Default: 1527
+#
+# [*druid.storage.type*]
+# Default: noop
+#
+# [*druid.indexer.logs.type*]
+# This property must be set for both overlord and middlemanager, hence
+# it is present in common.runtime.properties.
+# Default: file
+#
+# [*druid.indexer.logs.directory*]
+# This property must be set for both overlord and middlemanager, hence
+# it is present in common.runtime.properties.
+# Default: /var/lib/druid/indexing-logs
+#
+# [*druid.monitoring.monitors*]
+# Default: ["com.metamx.metrics.JvmMonitor"]
+#
+# [*druid.emitter*]
+# Default: logging
+#
+# [*druid.emitter.logging.logLevel*]
+# Default: info
+#
+class druid(
+ $properties = {},
+)
+{
+ $default_properties = {
+ 'druid.extensions.directory' =>
'/usr/share/druid/extensions',
+ 'druid.extensions.loadList' => [
+ 'druid-histogram',
+ 'druid-datasketches',
+ 'druid-namespace-lookup'
+ ],
+ 'druid.extensions.hadoopDependenciesDir' =>
'/usr/share/druid/hadoop-dependencies',
+ 'druid.startup.logging.logProperties' => true,
+ 'druid.zk.service.host' =>
'localhost:2181',
+ 'druid.zk.paths.base' => '/druid',
+ 'druid.metadata.storage.type' => 'derby',
+ 'druid.metadata.storage.connector.connectURI' =>
'jdbc:derby://localhost:1527/var/lib/druid/metadata.db;create=true',
+ 'druid.metadata.storage.connector.host' => 'localhost',
+ 'druid.metadata.storage.connector.port' => 1527,
+ 'druid.storage.type' => 'noop',
+ 'druid.monitoring.monitors' =>
['com.metamx.metrics.JvmMonitor'],
+ 'druid.emitter' => 'logging',
+ 'druid.emitter.logging.logLevel' => 'info'
+ }
+ $runtime_properties = merge($default_properties, $properties)
+
+ require_package('druid-common')
+
+ file { '/etc/druid/common.runtime.properties':
+ content => template('druid/runtime.properties.erb')
+ }
+}
+
+# # TODO: This won't work from manifests/service.pp on mediawiki vagrant, hmmm.
+# # == Define druid::service
+# # == Parameters
+# # arguments
+# #
+# define druid::service(
+# $runtime_properties,
+# $service = $title,
+# $env = undef,
+# $enable = true,
+# )
+# {
+# file { "/etc/druid/${service}/runtime.properties":
+# content => template('druid/runtime.properties.erb'),
+# }
+#
+# $env_ensure = $env ? {
+# undef => 'absent',
+# default => 'present',
+# }
+#
+# file { "/etc/druid/${service}/env.sh":
+# ensure => $env_ensure,
+# content => template('druid/env.sh.erb'),
+# }
+#
+# file { "/etc/druid/${service}/log4j2.xml":
+# content => template('druid/log4j2.xml.erb'),
+# }
+#
+# $service_ensure = $enabled ? {
+# false => 'stopped',
+# default => 'running',
+# }
+# service { "druid-${service}":
+# ensure => $service_ensure,
+# enable => $enabled,
+# hasrestart => true,
+# require => [
+# File["/etc/druid/${service}/runtime.properties"],
+# File["/etc/druid/${service}/env.sh"],
+# File["/etc/druid/${service}/log4j2.xml"],
+# ]
+# }
+# }
\ No newline at end of file
diff --git a/modules/druid/manifests/middlemanager.pp
b/modules/druid/manifests/middlemanager.pp
new file mode 100644
index 0000000..a8716e0
--- /dev/null
+++ b/modules/druid/manifests/middlemanager.pp
@@ -0,0 +1,36 @@
+# == Class druid::middlemanager
+#
+class druid::middlemanager(
+ $properties = {},
+ $env = {},
+)
+{
+ require druid
+
+ $default_properties = {
+ 'druid.port' => 8091,
+ # Number of tasks per middleManager
+ 'druid.worker.capacity' => 3,
+
+ # Task launch parameters
+ 'druid.indexer.runner.javaOpts' => '-server -Xmx128m
-Duser.timezone=UTC -Dfile.encoding=UTF-8
-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager',
+ 'druid.indexer.task.baseTaskDir' => '/var/lib/druid/task',
+
+ # HTTP server threads
+ 'druid.server.http.numThreads' => 4,
+
+ # Processing threads and buffers
+ 'druid.processing.buffer.sizeBytes' => 134217728,
+ 'druid.processing.numThreads' => 1,
+ }
+
+ $default_env = {
+ 'JMX_PORT' => 9664,
+ 'DRUID_HEAP_OPTS' => '-Xmx64m -Xms64m',
+ }
+
+ druid::service { 'middlemanager':
+ runtime_properties => merge($default_properties, $properties),
+ env => merge($default_env, $env),
+ }
+}
diff --git a/modules/druid/manifests/overlord.pp
b/modules/druid/manifests/overlord.pp
new file mode 100644
index 0000000..cee5387
--- /dev/null
+++ b/modules/druid/manifests/overlord.pp
@@ -0,0 +1,27 @@
+# == Class druid::overlord
+#
+class druid::overlord(
+ $properties = {},
+ $env = {},
+)
+{
+ require druid
+
+ $default_properties = {
+ 'druid.port' => 8090,
+ 'druid.indexer.queue.startDelay' => 'PT5S',
+
+ 'druid.indexer.runner.type' => 'remote',
+ 'druid.indexer.storage.type' => 'metadata',
+ }
+
+ $default_env = {
+ 'JMX_PORT' => 9665,
+ 'DRUID_HEAP_OPTS' => '-Xmx128m -Xms128m',
+ }
+
+ druid::service { 'overlord':
+ runtime_properties => merge($default_properties, $properties),
+ env => merge($default_env, $env),
+ }
+}
diff --git a/modules/druid/templates/env.sh.erb
b/modules/druid/templates/env.sh.erb
new file mode 100644
index 0000000..0f34a7c
--- /dev/null
+++ b/modules/druid/templates/env.sh.erb
@@ -0,0 +1,5 @@
+# NOTE: This file is managed by Puppet.
+
+<% @env.keys.sort.each do |key| -%>
+<%= key %>="<%= @env[key] %>"
+<% end -%>
diff --git a/modules/druid/templates/log4j2.xml.erb
b/modules/druid/templates/log4j2.xml.erb
new file mode 100644
index 0000000..165c456
--- /dev/null
+++ b/modules/druid/templates/log4j2.xml.erb
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+<Configuration status="warn">
+ <Appenders>
+ <RollingFile name="RollingFile"
+ fileName="/var/log/druid/<%= @service %>.log"
+ filePattern="/var/log/druid/<%= @service %>.%i.log">
+ <PatternLayout>
+ <Pattern>%d{ISO8601} %p %c: %m%n</Pattern>
+ </PatternLayout>
+ <Policies>
+ <SizeBasedTriggeringPolicy size="128 MB"/>
+ </Policies>
+ <DefaultRolloverStrategy max="4"/>
+ </RollingFile>
+ </Appenders>
+
+ <Loggers>
+ <Root level="info">
+ <AppenderRef ref="RollingFile"/>
+ </Root>
+ </Loggers>
+</Configuration>
diff --git a/modules/druid/templates/runtime.properties.erb
b/modules/druid/templates/runtime.properties.erb
new file mode 100644
index 0000000..09c80b3
--- /dev/null
+++ b/modules/druid/templates/runtime.properties.erb
@@ -0,0 +1,5 @@
+# NOTE: This file is managed by Puppet.
+
+<% @runtime_properties.keys.sort.each do |key| -%>
+<%= key %>=<%= @runtime_properties[key] %>
+<% end -%>
--
To view, visit https://gerrit.wikimedia.org/r/288099
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I65cb169ac2701e754cd6a68e31903bcb25644290
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
