Gehel has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/389550 )
Change subject: Deploy MjoLniR with new deploy repository ...................................................................... Deploy MjoLniR with new deploy repository While we were able to deploy mjolnir relatively naively onto relforge, as it only needed the kafka deps, deploying to the analytics cluster needs to be a bit more involved. A deploy repo mimicing ORES deployment was setup and this deploys it. This depends on I420c894250 being merged before this can be deployed. Change-Id: Idff10716e370462b8cecdbf3298ae9b64566fc25 --- M hieradata/role/common/deployment_server.yaml A modules/mjolnir/manifests/init.pp M modules/profile/manifests/mjolnir/kafka_daemon.pp M modules/profile/templates/mjolnir/kafka-daemon.service.erb M modules/role/manifests/elasticsearch/analytics.pp 5 files changed, 21 insertions(+), 14 deletions(-) Approvals: Alexandros Kosiaris: Looks good to me, but someone else must approve jenkins-bot: Verified Gehel: Looks good to me, approved diff --git a/hieradata/role/common/deployment_server.yaml b/hieradata/role/common/deployment_server.yaml index 8eac2e3..9d82708 100644 --- a/hieradata/role/common/deployment_server.yaml +++ b/hieradata/role/common/deployment_server.yaml @@ -173,8 +173,8 @@ # Librenms software librenms/librenms: repository: operations/software/librenms - relforge/mjolnir: - repository: search/MjoLniR + search/mjolnir/deploy: + repository: search/MjoLniR/deploy statsv/statsv: repository: analytics/statsv "docker-pkg/deploy": diff --git a/modules/mjolnir/manifests/init.pp b/modules/mjolnir/manifests/init.pp new file mode 100644 index 0000000..5a3cbbd --- /dev/null +++ b/modules/mjolnir/manifests/init.pp @@ -0,0 +1,14 @@ +# = Class: mjolnir +# +# This class installs the MjoLniR (Machine Learned Ranking) data +# processing package. +# +class mjolnir { + require_package('virtualenv', 'zip') + + scap::target { 'search/mjolnir/deploy': + deploy_user => 'deploy-service', + } +} + + diff --git a/modules/profile/manifests/mjolnir/kafka_daemon.pp b/modules/profile/manifests/mjolnir/kafka_daemon.pp index c1dcaf9..479187c 100644 --- a/modules/profile/manifests/mjolnir/kafka_daemon.pp +++ b/modules/profile/manifests/mjolnir/kafka_daemon.pp @@ -9,18 +9,11 @@ # it is named just 'eqiad'. $kafka_config = kafka_config('eqiad'), ) { - scap::target { 'relforge/mjolnir': - deploy_user => 'deploy-service', - } - - # This is a limited subset of what the full mjolnir package requires because - # the daemon is a small part of the overall application. The daemon only needs - # to read/write kafka topics and send requests to localhost. - require_package('python-kafka', 'python-requests') + class { 'mjolnir': } systemd::service { 'mjolnir-kafka-daemon': content => template('profile/mjolnir/kafka-daemon.service.erb'), - require => Scap::Target['relforge/mjolnir'], + require => Scap::Target['search/mjolnir/deploy'], } } diff --git a/modules/profile/templates/mjolnir/kafka-daemon.service.erb b/modules/profile/templates/mjolnir/kafka-daemon.service.erb index b6947ba..7e40bd8 100644 --- a/modules/profile/templates/mjolnir/kafka-daemon.service.erb +++ b/modules/profile/templates/mjolnir/kafka-daemon.service.erb @@ -5,9 +5,7 @@ [Service] User=nobody Group=nogroup -WorkingDirectory=/srv/deployment/relforge/mjolnir -Environment=PYTHONPATH=/srv/deployment/relforge/mjolnir -ExecStart=/usr/bin/python2 /srv/deployment/relforge/mjolnir/mjolnir/cli/kafka_daemon.py --brokers <%= @kafka_config['brokers']['string'] %> +ExecStart=/srv/deployment/search/mjolnir/venv/bin/mjolnir-utilities.py kafka_daemon --brokers <%= @kafka_config['brokers']['string'] %> StandardInput=null StandardOutput=journal StandardError=journal diff --git a/modules/role/manifests/elasticsearch/analytics.pp b/modules/role/manifests/elasticsearch/analytics.pp index a46391e..66d7789 100644 --- a/modules/role/manifests/elasticsearch/analytics.pp +++ b/modules/role/manifests/elasticsearch/analytics.pp @@ -1,5 +1,7 @@ # Supports CirrusSearch usage on the analytics cluster class role::elasticsearch::analytics { + class { 'mjolnir': } + # wikimedia/discovery/analytics will be deployed to this node scap::target { 'wikimedia/discovery/analytics': deploy_user => 'deploy-service', -- To view, visit https://gerrit.wikimedia.org/r/389550 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Idff10716e370462b8cecdbf3298ae9b64566fc25 Gerrit-PatchSet: 9 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: EBernhardson <ebernhard...@wikimedia.org> Gerrit-Reviewer: Alexandros Kosiaris <akosia...@wikimedia.org> Gerrit-Reviewer: DCausse <dcau...@wikimedia.org> Gerrit-Reviewer: EBernhardson <ebernhard...@wikimedia.org> Gerrit-Reviewer: Gehel <guillaume.leder...@wikimedia.org> Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits