Ottomata has submitted this change and it was merged. Change subject: Add new scap::source define to ease bootstrapping of repositories on deploy servers ......................................................................
Add new scap::source define to ease bootstrapping of repositories on deploy servers scap::source will clone your source repo, and if scap_repository is set, it will clone that repo at /srv/deployment/$title/scap. This allows for scap/ directories to be separated from source repositories, and allows scap repos to bootstrap themselves on deploy servers, instead of relying on trebuchet. 'scap::sources' is a hiera variable that contains resource declarations for scap::source that will be dynamically by scap::server. eventlogging/eventbus is the guinea pig here, so this is applied to it. It is declared in hieradata/role/common/deployment/server.yaml to clone from the eventlogging repository. This shouldn't conflict with trebuchet's deployment.yaml clones, as the git::clone will only execute if .git/config doesn't yet exist. This also moves scap::server's directy hiera_hash lookups to class parameter based lookups via hiera classpath. See scap/server.yaml files. TODO: Can we use :expand_path nuyaml config in labs? Ask Guiseppe. Bug: T118772 Change-Id: I32bd25a84b182b52db7db81404734f1259b623e6 --- A hieradata/common/scap/server.yaml M hieradata/labs/deployment-prep/common.yaml M hieradata/role/common/deployment/server.yaml A modules/eventlogging/manifests/deployment/target.pp M modules/role/manifests/deployment/server.pp M modules/role/manifests/eventbus/eventbus.pp M modules/scap/manifests/server.pp A modules/scap/manifests/source.pp M modules/scap/manifests/target.pp 9 files changed, 320 insertions(+), 39 deletions(-) Approvals: Ottomata: Looks good to me, approved jenkins-bot: Verified diff --git a/hieradata/common/scap/server.yaml b/hieradata/common/scap/server.yaml new file mode 100644 index 0000000..6554a5c --- /dev/null +++ b/hieradata/common/scap/server.yaml @@ -0,0 +1,52 @@ +# Default scap::server configuration. This is used in production. +# If you are setting up scap::server in labs, these will be used +# unless you override them for your labs project. +# See hieradata/labs/deployment-prep/scap/server.yaml as an example. + +# keyholder::agent declarations. These are created +# by the scap::server class. Each agent listed here +# will be present and useable by scap on the scap deploy server. +# NOTE: since labs +keyholder_agents: + + phabricator: + trusted_group: deploy-phabricator + key_fingerprint: 39:b3:2c:a7:b2:80:65:ff:0c:97:e1:22:88:6c:59:10 + key_secret: phabricator/phab_deploy_private_key + + eventlogging: + trusted_group: eventlogging-admins + key_fingerprint: b6:4e:1a:1b:4b:70:ef:91:31:cd:a3:18:9a:ca:41:44 + + deploy-service: + trusted_group: + - deploy-service + - aqs-admins + key_fingerprint: 6d:54:92:8b:39:10:f5:9b:84:40:36:ef:3c:9a:6d:d8 + key_file: servicedeploy_rsa + + dumpsdeploy: + trusted_group: ops + key_fingerprint: 86:c9:17:ab:b7:00:79:b5:8a:c5:b5:ee:29:24:c9:2f + + +# scap::source declarations. These are created +# by the scap::server class. Each source listed here +# will be cloned on the scap deploy server. +sources: + + # Source code for taking XML dumps and snapshots of Mediawiki. + dumps/dumps: + repository: operations/dumps + scap_repository: operations/dumps/scap + + # eventbus runs eventlogging-service. + # See: https://wikitech.wikimedia.org/wiki/EventBus + eventlogging/eventbus: + repository: eventlogging + scap_repository: eventlogging/scap/eventbus + + # eventlogging/analytics is the original EventLogging server deployment. + eventlogging/analytics: + repository: eventlogging + scap_repository: eventlogging/scap/analytics \ No newline at end of file diff --git a/hieradata/labs/deployment-prep/common.yaml b/hieradata/labs/deployment-prep/common.yaml index 7ca3da6..dadcf2c 100644 --- a/hieradata/labs/deployment-prep/common.yaml +++ b/hieradata/labs/deployment-prep/common.yaml @@ -212,16 +212,47 @@ deployment-kafka02.deployment-prep.eqiad.wmflabs: id: 1 -keyholder::agents: + +# NOTE: The production scap::server $keyholder_agents and $sources used by +# The scap::server class are in common/scap/server.yaml. +# These override the production ones for deployment-prep in labs. +# These would be in $classpath based lookup somewhere for deployment-prep, +# (like labs/deployment-prep/scap/server.yaml), but :expand_path doesn't +# seem to work the same way in labs as it does in production for the nuyaml +# backend. + +# deployment-prep keyholder::agent declarations. These are created +# by the scap::server class. Each agent listed here +# will be present and useable by scap on the scap deploy server. +scap::server::keyholder_agents: + phabricator: trusted_group: project-%{::labsproject} key_fingerprint: 39:b3:2c:a7:b2:80:65:ff:0c:97:e1:22:88:6c:59:10 key_secret: phabricator/phab_deploy_private_key + eventlogging: trusted_group: project-%{::labsproject} key_fingerprint: 02:9b:99:e2:f0:16:70:a3:d2:5a:e6:02:a3:73:0e:b0 - key_file: eventlogging_rsa + deploy-service: trusted_group: deploy-service key_fingerprint: 6d:54:92:8b:39:10:f5:9b:84:40:36:ef:3c:9a:6d:d8 key_file: servicedeploy_rsa + + +# deployment-prep scap::source declarations. These are created +# by the scap::server class. Each source listed here +# will be cloned on the scap deploy server. +scap::server::sources: + + # eventbus runs eventlogging-service. + # See: https://wikitech.wikimedia.org/wiki/EventBus + eventlogging/eventbus: + repository: eventlogging + scap_repository: eventlogging/scap/eventbus + + # eventlogging/analytics is the original EventLogging server deployment. + eventlogging/analytics: + repository: eventlogging + scap_repository: eventlogging/scap/analytics diff --git a/hieradata/role/common/deployment/server.yaml b/hieradata/role/common/deployment/server.yaml index 79b3dfa..a1c3efe 100644 --- a/hieradata/role/common/deployment/server.yaml +++ b/hieradata/role/common/deployment/server.yaml @@ -22,17 +22,3 @@ server: light_process_count: 0 light_process_file_prefix: - -keyholder::agents: - eventlogging: - trusted_group: eventlogging-admins - key_fingerprint: b6:4e:1a:1b:4b:70:ef:91:31:cd:a3:18:9a:ca:41:44 - deploy-service: - trusted_group: - - deploy-service - - aqs-admins - key_fingerprint: 6d:54:92:8b:39:10:f5:9b:84:40:36:ef:3c:9a:6d:d8 - key_file: servicedeploy_rsa - dumpsdeploy: - trusted_group: ops - key_fingerprint: 86:c9:17:ab:b7:00:79:b5:8a:c5:b5:ee:29:24:c9:2f' diff --git a/modules/eventlogging/manifests/deployment/target.pp b/modules/eventlogging/manifests/deployment/target.pp new file mode 100644 index 0000000..ac489fa --- /dev/null +++ b/modules/eventlogging/manifests/deployment/target.pp @@ -0,0 +1,49 @@ +# == Define eventlogging::deployment::target +# +# Abstracts use of scap::target for multiple eventlogging deployment targets. +# A corresponding 'eventlogging/$title' scap::source in the scap::sources +# hiera variable must be declared. +# See: hieradata/role/common/deployment/server.yaml and +# modules/scap/manifests/deploy +# +# == Parameters +# +# [*service_name*] +# service_name to pass to scap::target for sudo rules. Default: undef +# +# [*sudo_rules*] +# Array of extra sudo rules to pass to scap::target. +# Default: undef +# +# == Usage +# +# # Deploy eventlogging/eventbus here, and allow +# # eventlogging user to restart eventlogging-service-eventbus. +# eventlogging::deployment::target { 'eventbus': +# service_name => 'eventlogging-service-eventbus', +# } +# +# # Deploy eventlogging/eventlogging here, and allow +# # eventlogging user to run eventloggingctl as root. +# eventlogging::deployment::target { 'eventlogging': +# sudo_rules => ['ALL=(root) NOPASSWD: /sbin/eventloggingctl *'] +# } +# +define eventlogging::deployment::target( + $service_name = undef, + $sudo_rules = undef, +) { + # Install eventlogging dependencies from .deb packages. + include eventlogging + + # eventlogging code for eventbus is configured to deploy + # from the eventlogging/eventbus deploy target + # via scap/scap.cfg on the deployment host. + scap::target { "eventlogging/${title}": + deploy_user => 'eventlogging', + public_key_source => "puppet:///modules/eventlogging/deployment/eventlogging_rsa.pub.${::realm}", + service_name => $service_name, + sudo_rules => $sudo_rules, + manage_user => false, + } +} diff --git a/modules/role/manifests/deployment/server.pp b/modules/role/manifests/deployment/server.pp index 322bdb3..34c7b51 100644 --- a/modules/role/manifests/deployment/server.pp +++ b/modules/role/manifests/deployment/server.pp @@ -8,7 +8,7 @@ # include misc::deployment::scripts include role::deployment::mediawiki - # scap::server will ensure that all keyholder::agents + # scap::server will ensure that all keyholder::agents and scap::sources # declared in hiera will exist. scap::server is # for generic repository deployment and does not have # anything to do with Mediawiki. diff --git a/modules/role/manifests/eventbus/eventbus.pp b/modules/role/manifests/eventbus/eventbus.pp index 109885e..0d4c766 100644 --- a/modules/role/manifests/eventbus/eventbus.pp +++ b/modules/role/manifests/eventbus/eventbus.pp @@ -12,15 +12,12 @@ require ::eventschemas require ::role::kafka::main::config - # eventlogging code for eventbus is configured to deploy - # from the eventlogging/eventbus deploy target - # via scap/scap.cfg on the deployment host. - scap::target { 'eventlogging/eventbus': - deploy_user => 'eventlogging', - public_key_source => "puppet:///modules/eventlogging/deployment/eventlogging_rsa.pub.${::realm}", - service_name => 'eventlogging-service-eventbus', - manage_user => false, + eventlogging::deployment::target { 'eventbus': + service_name => 'eventlogging-service-eventbus', } + # eventlogging::deployment::target { 'eventbus': + # Will deploy eventlogging code here. + $eventlogging_path = '/srv/deployment/eventlogging/eventbus' $kafka_brokers_array = $role::kafka::main::config::brokers_array $kafka_base_uri = inline_template('kafka:///<%= @kafka_brokers_array.join(":9092,") + ":9092" %>') @@ -36,7 +33,6 @@ "${kafka_base_uri}?async=False&topic=${::site}.{meta[topic]}" ] - $eventlogging_path = '/srv/deployment/eventlogging/eventbus' # TODO: Allow configuration of more than one service daemon process? eventlogging::service::service { 'eventbus': eventlogging_path => $eventlogging_path, diff --git a/modules/scap/manifests/server.pp b/modules/scap/manifests/server.pp index 543a182..5210cc1 100644 --- a/modules/scap/manifests/server.pp +++ b/modules/scap/manifests/server.pp @@ -3,24 +3,67 @@ # Configures dependencies for a scap3 deployment server. This includes # setting up ssh agent keys and repositories configured for deployment. # -# This class creates keyholder::agent resources based on -# the contents of the 'keyholder::agents' hiera variable. +# This class creates keyholder::agent and scap::source resources based on +# the contents of the 'keyholder::agents' and 'scap::sources' hiera variables. +# These would be class parameters instead of hiera lookups, if it were possible +# to do a hiera hash merge using class parameters. Since hash merge doesn't +# work with class paramaters, these are looked up via hiera_hash and +# must be defined as noted above. # # Legacy scap and mediawiki deployment dependencies are in # scap::master. # -class scap::server { +# == Parameters +# +# [*keyholder_agents*] +# Hash of keyholder::agent resource declarations to be passed to +# the create_resources() function. Default: {} +# +# keyholder is an ssh agent proxy that allows members of select groups to +# connect using ssh keys shared with the group. This facilitates multiple +# deployers to deploy over ssh to corresponding scap::target instances. +# See keyholder::agent for more information. +# +# $keyholder_agents lists the details of each ssh key. +# Actual keys are stored in the `secret` module +# which is kept in a private location in the puppet modulepath. +# +# [*sources*] +# Hash of scap::source resource declarations to be passed to +# the create_resources() function. Default: {} +# +# Each repository listed will be cloned via declaration of the +# scap::source define. You should use scap::target directly on your +# target hosts that are declared with $package_name matching the keys in +# this hash. +# See scap::source for more information. +# +# == Usage +# +# class { 'scap::server': +# keyholder_agents => { +# 'deploy-service' => { +# 'trusted_group' => 'deploy-service', +# 'key_fingerprint' => 'xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx', +# 'key_file' => 'servicedeploy_rsa', +# }, +# }, +# sources => { +# 'myrepo/instance' => { +# 'repository' => 'myrepo', +# }, +# }, +# } +# +class scap::server( + $keyholder_agents = {}, + $sources = {}, +) { require ::scap - # keyholder is an ssh agent proxy that allows members of select groups to - # connect using ssh keys shared with the group. This facilitates multiple - # deployers to deploy over ssh to corresponding scap::target instances. + # Create an instance of $keyholder_agents for each of the key specs. + create_resources('keyholder::agent', $keyholder_agents) - # For a given deployment server, we list the details of each key in hiera - # under keyholder::agents, actual keys are stored in the `secret` module - # which is kept in a private location in the puppet modulepath. - $agent_keys = hiera_hash('keyholder::agents', {}) - - # Create an instance of keyholder::agent for each of the key specs in hiera: - create_resources('keyholder::agent', $agent_keys) + # Create an instance of scap::source for each of the key specs in hiera. + create_resources('scap::source', $sources) } diff --git a/modules/scap/manifests/source.pp b/modules/scap/manifests/source.pp new file mode 100644 index 0000000..6f8e436 --- /dev/null +++ b/modules/scap/manifests/source.pp @@ -0,0 +1,123 @@ +# == Define scap::source +# +# Sets up scap3 deployment source on a deploy server. +# This will clone $repository at /srv/deployment/$title. +# If $scap_repository is set it will clone it at +# /srv/deployment/$title/scap. If you set $scap_repository to true, +# this will assume that your scap repository is named $title/scap. +# +# To use this in conjunction with scap::target, make sure the $title here +# matches a scap::target's $title on your target hosts, or at least matches +# the $package_name provided to scap::target (which defaults to $title). +# +# NOTE: This define is compatible with trebuchet's deployment.yaml file. +# If trebuchet has already cloned a source repository in /srv/deployment, +# this clone will do nothing, as it only executes if .git/config +# doesn't already exist. +# +# == Parameters +# +# [*repository*] +# Repository name in gerrit. Default: $title +# +# [*scap_repository*] +# String or boolean. +# +# If you set this to a string, it will be assumed to be a repository name +# This scap repository will then be cloned into /srv/deployment/$title/scap. +# If this is set to true your scap_repository will be assumed to +# live at $title/scap in gerrit. +# +# You can use this keep your scap configs separate from your source +# repositories. +# +# Default: false. +# +# [*owner*] +# Owner of cloned repository, +# Default: trebuchet +# +# [*group*] +# Group owner of cloned repository. +# Default: wikidev +# +# == Usage +# +# # Clones the 'repo/without/external/scap' repsitory into +# # /srv/deployment/repo/without/external/scap. +# +# scap::source { 'repo/without/external/scap': } +# +# +# # Clones the 'eventlogging' repository into +# # /srv/deployment/eventlogging/eventbus and +# # clones the 'eventlogging/eventbus/scap' repository +# # into /srv/deployment/eventlogging/eventbus/scap +# +# scap::source { 'eventlogging/eventbus': +# repository => 'eventlogging', +# scap_repository => true, +# } +# +# +# # Clones the 'myproject/myrepo' repository into +# # /srv/deployment/myproject/myrepo, and +# # clones the custom scap repository at +# # 'my/custom/scap/repo' from gerrit into +# # /srv/deployment/myproject/myrepo/scap +# +# scap::source { 'myproject/myrepo': +# scap_repository => 'my/custom/scap/repo', +# } +# +define scap::source( + $repository = $title, + $scap_repository = false, + # TODO: change scap repo owner when scap figures out + # how to bootstrap itself properly without trebuchet. + $owner = 'trebuchet', + $group = 'wikidev', +) { + # Path at which $repository should be cloned. + $path = "/srv/deployment/${title}" + + # We can't rely on puppet to manage arbitrary subdirectories. + # Use an exec to just make sure that $path's parent directories exist. + exec { "mkdir_scap_source_path_${title}": + command => "mkdir -p $(dirname ${path}) && chmod 775 $(dirname ${path}) && chown ${owner}:${group} $(dirname ${path})", + path => '/bin:/usr/bin', + unless => "test -d $(dirname ${path})", + user => 'root', + } + + # Clone the source repository at $path. + git::clone { "scap::source ${repository} for ${title}": + # Since usage of this define might result in multiple clones of the + # same $repository, it is necessary to title the git::clones with + # unique names. If we aren't using the repository name as the $title + # of git::clone, then we need to set $origin, and a $origin + # must be a full git URL. This means we can't yet use phabricator + # git URLs. TODO: Fix git::clone to support custom repository names + # without specificing full git $origin URLs. + origin => "https://gerrit.wikimedia.org/r/p/${repository}.git";, + directory => $path, + owner => $owner, + group => $group, + shared => true, + recurse_submodules => true, + require => Exec["mkdir_scap_source_path_${title}"], + } + + if $scap_repository { + # Clone the scap repository at $path/scap + git::clone { "scap::source ${scap_repository} for ${title}": + origin => "https://gerrit.wikimedia.org/r/p/${scap_repository}.git";, + directory => "${path}/scap", + owner => $owner, + group => $group, + shared => true, + recurse_submodules => true, + require => Git::Clone["scap::source ${repository} for ${title}"], + } + } +} diff --git a/modules/scap/manifests/target.pp b/modules/scap/manifests/target.pp index 36d32e9..234c6ac 100644 --- a/modules/scap/manifests/target.pp +++ b/modules/scap/manifests/target.pp @@ -40,6 +40,7 @@ # scap::target { 'eventlogging/eventlogging': # deploy_user => 'eventlogging', # public_key_source => "puppet:///modules/eventlogging/deployment/eventlogging_rsa.pub.${::realm}", +# manage_user => false, # } # define scap::target( -- To view, visit https://gerrit.wikimedia.org/r/280730 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I32bd25a84b182b52db7db81404734f1259b623e6 Gerrit-PatchSet: 29 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Ottomata <[email protected]> Gerrit-Reviewer: 20after4 <[email protected]> Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]> Gerrit-Reviewer: Ottomata <[email protected]> Gerrit-Reviewer: Thcipriani <[email protected]> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
