Ottomata has uploaded a new change for review.
https://gerrit.wikimedia.org/r/282166
Change subject: Create new eventlogging::analytics role in modules/role, use
scap for deployment
......................................................................
Create new eventlogging::analytics role in modules/role, use scap for deployment
This is only the deployment of a new cloned repo to eventlog1001, it does not
affect
the running deployment or daemons.
Bug: T118772
Change-Id: Ib37ec5a3aba1a9e018a700d4271b4930d4dc5472
---
M hieradata/common/scap/server.yaml
M manifests/role/eventlogging.pp
M manifests/site.pp
A modules/eventlogging/manifests/dependencies.pp
M modules/eventlogging/manifests/deployment/target.pp
M modules/eventlogging/manifests/init.pp
M modules/eventlogging/manifests/server.pp
M modules/role/manifests/eventbus/eventbus.pp
A modules/role/manifests/eventlogging/README.md
A modules/role/manifests/eventlogging/analytics.pp
10 files changed, 97 insertions(+), 30 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/66/282166/1
diff --git a/hieradata/common/scap/server.yaml
b/hieradata/common/scap/server.yaml
index 6554a5c..c10d380 100644
--- a/hieradata/common/scap/server.yaml
+++ b/hieradata/common/scap/server.yaml
@@ -49,4 +49,12 @@
# eventlogging/analytics is the original EventLogging server deployment.
eventlogging/analytics:
repository: eventlogging
- scap_repository: eventlogging/scap/analytics
\ No newline at end of file
+ scap_repository: eventlogging/scap/analytics
+
+ # eventlogging/common is a default deployment of eventlogging to
+ # places that don't need service management or regular updates.
+ # This is meant to be used for CLI uses of eventlogging. webperf
+ # and stat1002 and others use this.
+ eventlogging/common:
+ repository: eventlogging
+ scap_repository: eventlogging/scap/common
diff --git a/manifests/role/eventlogging.pp b/manifests/role/eventlogging.pp
index 08b3e09..6051696 100644
--- a/manifests/role/eventlogging.pp
+++ b/manifests/role/eventlogging.pp
@@ -76,7 +76,7 @@
$kafka_server_side_raw_uri =
"${kafka_base_uri}?topic=eventlogging-server-side"
$kafka_client_side_raw_uri =
"${kafka_base_uri}?topic=eventlogging-client-side"
- class { '::eventlogging': }
+ include ::eventlogging
# This check was written for eventlog1001, so only include it there.,
if $::hostname == 'eventlog1001' {
diff --git a/manifests/site.pp b/manifests/site.pp
index 64bad1a..f38de3e 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -993,7 +993,12 @@
eventlogging::processor,
eventlogging::consumer::mysql,
eventlogging::consumer::files,
- logging::mediawiki::errors
+ logging::mediawiki::errors,
+ # TODO: This is in trial mode while we migrate away
+ # from manifests/role classes into module/role, and
+ # use scap3 for deployment.
+ eventlogging::analytics
+
include standard
include base::firewall
diff --git a/modules/eventlogging/manifests/dependencies.pp
b/modules/eventlogging/manifests/dependencies.pp
new file mode 100644
index 0000000..3580504
--- /dev/null
+++ b/modules/eventlogging/manifests/dependencies.pp
@@ -0,0 +1,31 @@
+# == Class: eventlogging::dependencies
+#
+# EventLogging is a platform for modeling, logging and processing
+# arbitrary schemaed JSON data.
+#
+# This class only installs dependencies. To set up a deployment target, use
+# the eventlogging::deployment::target define. If you want to configure an
+# eventlogging server that will run eventlogging service daemons, include the
+# eventlogging::server class, or use one or more of the
+# eventlogging::service::* defines.
+#
+# If you want just a unmanaged clone of eventlogging source code to use,
+# use the eventlogging class.
+#
+class eventlogging::dependencies {
+ # Install all eventlogging dependencies from .debs.
+ require_package([
+ 'python-dateutil',
+ 'python-jsonschema',
+ 'python-kafka',
+ 'python-mysqldb',
+ 'python-pygments',
+ 'python-pykafka',
+ 'python-pymongo',
+ 'python-six',
+ 'python-sqlalchemy',
+ 'python-statsd',
+ 'python-yaml',
+ 'python-zmq',
+ ])
+}
diff --git a/modules/eventlogging/manifests/deployment/target.pp
b/modules/eventlogging/manifests/deployment/target.pp
index ac489fa..07a46ea 100644
--- a/modules/eventlogging/manifests/deployment/target.pp
+++ b/modules/eventlogging/manifests/deployment/target.pp
@@ -8,6 +8,9 @@
#
# == Parameters
#
+# [*package_name*]
+# The name of the scap3 deployment package Default: eventlogging/$title
+#
# [*service_name*]
# service_name to pass to scap::target for sudo rules. Default: undef
#
@@ -30,16 +33,15 @@
# }
#
define eventlogging::deployment::target(
+ $package_name = "eventlogging/${title}",
$service_name = undef,
$sudo_rules = undef,
) {
# Install eventlogging dependencies from .deb packages.
- include eventlogging
+ include eventlogging::dependencies
- # eventlogging code for eventbus is configured to deploy
- # from the eventlogging/eventbus deploy target
- # via scap/scap.cfg on the deployment host.
scap::target { "eventlogging/${title}":
+ package_name => $package_name,
deploy_user => 'eventlogging',
public_key_source =>
"puppet:///modules/eventlogging/deployment/eventlogging_rsa.pub.${::realm}",
service_name => $service_name,
diff --git a/modules/eventlogging/manifests/init.pp
b/modules/eventlogging/manifests/init.pp
index ca7eb4b..0d62e55 100644
--- a/modules/eventlogging/manifests/init.pp
+++ b/modules/eventlogging/manifests/init.pp
@@ -1,29 +1,14 @@
-# == Class: eventlogging
+# == Class eventlogging
#
-# EventLogging is a platform for modeling, logging and processing
-# arbitrary schemaed JSON data.
+# TODO: Implement the following as part of T118772:
#
-# This class only installs dependencies and sets up eventlogging code
-# deployment. If you want to configure an eventlogging server that
-# will run eventlogging service daemons, include the eventlogging::server
-# class, or use one or more of the eventlogging::service::* defines.
+# Currently, this class only installs EventLogging dependencies
+# and ensures that an unmanaged cloned at /usr/local/src/eventlogging
+# exists. If you want to update eventlogging code, you must manually
+# pull it.
#
class eventlogging {
- # Install all eventlogging dependencies from .debs.
- require_package([
- 'python-dateutil',
- 'python-jsonschema',
- 'python-kafka',
- 'python-mysqldb',
- 'python-pygments',
- 'python-pykafka',
- 'python-pymongo',
- 'python-six',
- 'python-sqlalchemy',
- 'python-statsd',
- 'python-yaml',
- 'python-zmq',
- ])
+ include eventlogging::dependencies
# TEMPORARY HACK!!!
# TODO: use scap everywhere: T118772
diff --git a/modules/eventlogging/manifests/server.pp
b/modules/eventlogging/manifests/server.pp
index 234c716..34bd6ec 100644
--- a/modules/eventlogging/manifests/server.pp
+++ b/modules/eventlogging/manifests/server.pp
@@ -54,7 +54,7 @@
$eventlogging_path = '/srv/deployment/eventlogging/eventlogging',
)
{
- require ::eventlogging
+ require ::eventlogging::dependencies
group { 'eventlogging':
ensure => present,
diff --git a/modules/role/manifests/eventbus/eventbus.pp
b/modules/role/manifests/eventbus/eventbus.pp
index 0d4c766..ed31252 100644
--- a/modules/role/manifests/eventbus/eventbus.pp
+++ b/modules/role/manifests/eventbus/eventbus.pp
@@ -12,6 +12,9 @@
require ::eventschemas
require ::role::kafka::main::config
+ # eventlogging code for eventbus is configured to deploy
+ # from the eventlogging/eventbus deploy target
+ # via scap/scap.cfg on the deployment host.
eventlogging::deployment::target { 'eventbus':
service_name => 'eventlogging-service-eventbus',
}
diff --git a/modules/role/manifests/eventlogging/README.md
b/modules/role/manifests/eventlogging/README.md
new file mode 100644
index 0000000..2840649
--- /dev/null
+++ b/modules/role/manifests/eventlogging/README.md
@@ -0,0 +1,18 @@
+# EventLogging role classes
+
+At the very least, each of these classes sets up a distinct scap deployment
+target for deploying the EventLogging python codebase. The
+eventlogging::deployment::target define in the eventlogging module
+sets up EventLoggging dependencies via the eventlogging::dependencies
+class, and also a scap::target for deploying the eventlogging code.
+
+- analytics: The original Analytics deployment of EventLogging. This
+ role sets up and manages EventLogging daemon processes for processing
+ Analytics events.
+
+- common: (TODO) A minimal role that only installs dependencies and the
+ codebase. For historical reasons, this is symlinked to
+ /srv/deployment/eventlogging/eventlogging on targets.
+
+TODO: Move eventbus/eventbus.pp role here to role::eventlogging::eventbus.
+
diff --git a/modules/role/manifests/eventlogging/analytics.pp
b/modules/role/manifests/eventlogging/analytics.pp
new file mode 100644
index 0000000..c269b56
--- /dev/null
+++ b/modules/role/manifests/eventlogging/analytics.pp
@@ -0,0 +1,15 @@
+# == Class role::eventlogging::analytics
+#
+class role::eventlogging::analytics {
+ system::role { 'role::eventlogging::analytics':
+ description => 'EventLogging Analytics Processor and Consumer',
+ }
+
+ eventlogging::deployment::target { 'analytics':
+ # TODO: Do we need this sudo rule for 'eventlogging' user here?
+ # Allow eventlogging user to run eventloggingctl as root.
+ # sudo_rules => ['ALL=(root) NOPASSWD: /sbin/eventloggingctl *']
+ }
+
+ # TODO: Move manifests/role/eventlogging.pp classes here.
+}
--
To view, visit https://gerrit.wikimedia.org/r/282166
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib37ec5a3aba1a9e018a700d4271b4930d4dc5472
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
