Rush has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/368938 )
Change subject: openstack: move novaobserver to a profile
......................................................................
openstack: move novaobserver to a profile
* dupe entries in hieradata/labs.yaml as there
no shared portion of the tree here.
* change modules/graphite/files/archive-instances,
and shinkengen to gather creds from authoritative local file
* no longer include ::openstack::observerenv in
openstack::clientlib as it is a profile (clientlib
to be moved up next)
* remove obsolete observerenv.pp
* base and deployment versions for observerenv profile
* novaobserver.yaml.erb no longer looks in the giant
dict of novaconfig
* Add observerenv to roles now that it is no longer
part of openstack::clientlib
Bug: T171494
Change-Id: I3b2ab05096095c4be369edeaadeffc64c7cb045f
---
A hieradata/common/profile/openstack/base.yaml
M hieradata/labs.yaml
M modules/graphite/files/archive-instances
M modules/graphite/manifests/labs/archiver.pp
M modules/openstack/manifests/clientlib.pp
D modules/openstack/manifests/observerenv.pp
A modules/profile/files/openstack/base/novaobserver/observerenv.sh
A modules/profile/manifests/openstack/base/observerenv.pp
A modules/profile/manifests/openstack/labtest/observerenv.pp
A modules/profile/manifests/openstack/labtestn/observerenv.pp
A modules/profile/manifests/openstack/main/observerenv.pp
A modules/profile/templates/openstack/base/novaobserver/novaobserver.yaml.erb
M modules/role/manifests/labs/graphite.pp
M modules/role/manifests/labs/instance.pp
M modules/role/manifests/labs/nfs/secondary.pp
M modules/role/manifests/wmcs/openstack/labtest/control.pp
M modules/role/manifests/wmcs/openstack/labtest/web.pp
M modules/role/manifests/wmcs/openstack/labtestn/control.pp
M modules/role/manifests/wmcs/openstack/labtestn/web.pp
M modules/role/manifests/wmcs/openstack/main/control.pp
M modules/role/manifests/wmcs/openstack/main/horizon.pp
M modules/role/manifests/wmcs/openstack/main/net.pp
M modules/role/manifests/wmcs/openstack/main/web.pp
M modules/shinken/files/shinkengen
M modules/shinken/manifests/shinkengen.pp
M modules/shinken/templates/shinkengen.yaml.erb
26 files changed, 120 insertions(+), 39 deletions(-)
Approvals:
Rush: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/common/profile/openstack/base.yaml
b/hieradata/common/profile/openstack/base.yaml
new file mode 100644
index 0000000..f4a5a71
--- /dev/null
+++ b/hieradata/common/profile/openstack/base.yaml
@@ -0,0 +1,2 @@
+profile::openstack::base::region: "%{::site}"
+profile::openstack::base::observer_user: 'novaobserver'
diff --git a/hieradata/labs.yaml b/hieradata/labs.yaml
index 71d4919..0d265da 100644
--- a/hieradata/labs.yaml
+++ b/hieradata/labs.yaml
@@ -17,6 +17,12 @@
recursor: 'labs-recursor0.wikimedia.org'
recursor_secondary: 'labs-recursor1.wikimedia.org'
+profile::openstack::base::region: "%{::site}"
+profile::openstack::main::nova_controller: 'labcontrol1001.wikimedia.org'
+profile::openstack::base::observer_user: 'novaobserver'
+# publicly available read-only credentials
+profile::openstack::main::observer_password: 'Fs6Dq2RtG8KwmM2Z'
+
# Additional base overrides
standard::has_admin: false
profile::base::remote_syslog: false
diff --git a/modules/graphite/files/archive-instances
b/modules/graphite/files/archive-instances
index 7b7f0d0..eb20db0 100755
--- a/modules/graphite/files/archive-instances
+++ b/modules/graphite/files/archive-instances
@@ -31,10 +31,15 @@
def get_keystone_session(project_name):
+
+ with open('/etc/novaobserver.yaml') as n:
+ nova_observer = yaml.safe_load(n)
+ observer_pass = nova_observer['OS_PASSWORD']
+
return KeystoneSession(auth=KeystonePassword(
auth_url="http://labcontrol1001.wikimedia.org:5000/v3";,
username="novaobserver",
- password=sys.argv[1],
+ password=observer_pass,
project_name=project_name,
user_domain_name='default',
project_domain_name='default'
diff --git a/modules/graphite/manifests/labs/archiver.pp
b/modules/graphite/manifests/labs/archiver.pp
index cfab374..606b9cf 100644
--- a/modules/graphite/manifests/labs/archiver.pp
+++ b/modules/graphite/manifests/labs/archiver.pp
@@ -3,6 +3,7 @@
# Sets up a cron job that clears metrics from killed instances every
# hour
class graphite::labs::archiver {
+
file { '/usr/local/bin/archive-instances':
source => 'puppet:///modules/graphite/archive-instances',
owner => '_graphite',
@@ -10,12 +11,9 @@
mode => '0700',
}
- $novaconfig = hiera_hash('novaconfig', {})
- $observer_pass = $novaconfig['observer_password']
- include ::openstack::clientlib
cron { 'archive-deleted-instances':
ensure => present,
- command => "/usr/local/bin/archive-instances ${observer_pass}",
+ command => '/usr/local/bin/archive-instances',
user => '_graphite',
minute => 0,
hour => 13,
diff --git a/modules/openstack/manifests/clientlib.pp
b/modules/openstack/manifests/clientlib.pp
index 1827790..d202019 100644
--- a/modules/openstack/manifests/clientlib.pp
+++ b/modules/openstack/manifests/clientlib.pp
@@ -1,6 +1,5 @@
# Utilities for querying openstack
class openstack::clientlib {
- include ::openstack::observerenv
include ::openstack
require openstack2::cloudrepo
diff --git a/modules/openstack/manifests/observerenv.pp
b/modules/openstack/manifests/observerenv.pp
deleted file mode 100644
index f7a6b37..0000000
--- a/modules/openstack/manifests/observerenv.pp
+++ /dev/null
@@ -1,23 +0,0 @@
-# Access credentials for the keystone 'novaobserver' account
-class openstack::observerenv {
-
- # We don't need all the extras that role::labs::openstack::nova::common
- # includes... the simple config straight from hiera should do the trick.
- $novaconfig = hiera_hash('novaconfig', {})
- $nova_region = $::site
-
- # Keystone credentials for novaobserver
- file { '/etc/novaobserver.yaml':
- content => template('openstack/novaobserver.yaml.erb'),
- mode => '0444',
- owner => 'root',
- group => 'root',
- }
-
- file { '/usr/local/bin/observerenv.sh':
- source => 'puppet:///modules/openstack/observerenv.sh',
- mode => '0555',
- owner => 'root',
- group => 'root',
- }
-}
diff --git a/modules/profile/files/openstack/base/novaobserver/observerenv.sh
b/modules/profile/files/openstack/base/novaobserver/observerenv.sh
new file mode 100644
index 0000000..b57080b
--- /dev/null
+++ b/modules/profile/files/openstack/base/novaobserver/observerenv.sh
@@ -0,0 +1,21 @@
+# Largely cribbed from https://gist.github.com/pkuczynski/8665367
+parse_yaml() {
+ file='/etc/novaobserver.yaml'
+ s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034')
+ sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \
+ -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $file |
+ awk -F$fs '{
+ indent = length($file)/2;
+ vname[indent] = $2;
+ for (i in vname) {if (i > indent) {delete vname[i]}}
+ if (length($3) > 0) {
+ vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
+ printf("%s=%s ",$2,$3)
+ }
+ }'
+}
+
+values=`parse_yaml`
+for entry in $values; do
+ export $entry
+done
diff --git a/modules/profile/manifests/openstack/base/observerenv.pp
b/modules/profile/manifests/openstack/base/observerenv.pp
new file mode 100644
index 0000000..b05a9e8
--- /dev/null
+++ b/modules/profile/manifests/openstack/base/observerenv.pp
@@ -0,0 +1,23 @@
+# Access credentials for the keystone 'novaobserver' account
+class profile::openstack::base::observerenv(
+ $region = hiera('profile::openstack::base::region'),
+ $nova_controller = hiera('profile::openstack::base::nova_controller'),
+ $observer_user = hiera('profile::openstack::base::observer_user'),
+ $observer_password = hiera('profile::openstack::base::observer_password'),
+ ) {
+
+ # Keystone credentials for novaobserver
+ file { '/etc/novaobserver.yaml':
+ content =>
template('profile/openstack/base/novaobserver/novaobserver.yaml.erb'),
+ mode => '0444',
+ owner => 'root',
+ group => 'root',
+ }
+
+ file { '/usr/local/bin/observerenv.sh':
+ source =>
'puppet:///modules/profile/openstack/base/novaobserver/observerenv.sh',
+ mode => '0555',
+ owner => 'root',
+ group => 'root',
+ }
+}
diff --git a/modules/profile/manifests/openstack/labtest/observerenv.pp
b/modules/profile/manifests/openstack/labtest/observerenv.pp
new file mode 100644
index 0000000..c560407
--- /dev/null
+++ b/modules/profile/manifests/openstack/labtest/observerenv.pp
@@ -0,0 +1,10 @@
+class profile::openstack::labtest::observerenv(
+ $nova_controller = hiera('profile::openstack::labtest::nova_controller'),
+ $observer_password =
hiera('profile::openstack::labtest::observer_password'),
+ ) {
+
+ class {'profile::openstack::base::observerenv':
+ nova_controller => $nova_controller ,
+ observer_password => $observer_password,
+ }
+}
diff --git a/modules/profile/manifests/openstack/labtestn/observerenv.pp
b/modules/profile/manifests/openstack/labtestn/observerenv.pp
new file mode 100644
index 0000000..c72dbf2
--- /dev/null
+++ b/modules/profile/manifests/openstack/labtestn/observerenv.pp
@@ -0,0 +1,10 @@
+class profile::openstack::labtestn::observerenv(
+ $nova_controller = hiera('profile::openstack::labtestn::nova_controller'),
+ $observer_password =
hiera('profile::openstack::labtestn::observer_password'),
+ ) {
+
+ class {'profile::openstack::base::observerenv':
+ nova_controller => $nova_controller,
+ observer_password => $observer_password,
+ }
+}
diff --git a/modules/profile/manifests/openstack/main/observerenv.pp
b/modules/profile/manifests/openstack/main/observerenv.pp
new file mode 100644
index 0000000..368c258
--- /dev/null
+++ b/modules/profile/manifests/openstack/main/observerenv.pp
@@ -0,0 +1,10 @@
+class profile::openstack::main::observerenv(
+ $nova_controller = hiera('profile::openstack::main::nova_controller'),
+ $observer_password = hiera('profile::openstack::main::observer_password'),
+ ) {
+
+ class {'profile::openstack::base::observerenv':
+ nova_controller => $nova_controller ,
+ observer_password => $observer_password,
+ }
+}
diff --git
a/modules/profile/templates/openstack/base/novaobserver/novaobserver.yaml.erb
b/modules/profile/templates/openstack/base/novaobserver/novaobserver.yaml.erb
new file mode 100644
index 0000000..69a9058
--- /dev/null
+++
b/modules/profile/templates/openstack/base/novaobserver/novaobserver.yaml.erb
@@ -0,0 +1,10 @@
+OS_USERNAME: "<%= @observer_user %>"
+OS_PROJECT_DOMAIN_ID: "default"
+OS_USER_DOMAIN_ID: "default"
+OS_PASSWORD: "<%= @observer_password %>"
+OS_AUTH_URL: "http://<%= @nova_controller %>:5000/v3"
+OS_REGION_NAME: "<%= @region %>"
+OS_TENANT_NAME: "observer"
+OS_NO_CACHE: 1
+OS_IDENTITY_API_VERSION: 3
+OS_INTERFACE: "public"
diff --git a/modules/role/manifests/labs/graphite.pp
b/modules/role/manifests/labs/graphite.pp
index 0b57d23..b93863a 100644
--- a/modules/role/manifests/labs/graphite.pp
+++ b/modules/role/manifests/labs/graphite.pp
@@ -3,14 +3,16 @@
# Instance is open to all, no password required to see metrics
class role::labs::graphite {
+ require ::profile::openstack::main::observerenv
+ include graphite::labs::archiver
+ include role::statsite
+
class { 'role::graphite::base':
storage_dir => '/srv/carbon',
auth => false,
hostname => 'graphite-labs.wikimedia.org',
cors_origins => [
'https?://(grafana-labs|grafana-labs-admin).wikimedia.org' ],
}
-
- include graphite::labs::archiver
file { '/var/lib/carbon':
ensure => link,
@@ -19,8 +21,6 @@
group => '_graphite',
require => Class['role::graphite::base']
}
-
- include role::statsite
ferm::service { 'carbon_c_relay-local_relay_udp':
proto => 'udp',
diff --git a/modules/role/manifests/labs/instance.pp
b/modules/role/manifests/labs/instance.pp
index e43db4f..12bf41d 100644
--- a/modules/role/manifests/labs/instance.pp
+++ b/modules/role/manifests/labs/instance.pp
@@ -4,7 +4,7 @@
include ::profile::base::labs
include sudo
include ::base::instance_upstarts
- include ::openstack::observerenv
+ include ::profile::openstack::main::observerenv
sudo::group { 'ops':
privileges => ['ALL=(ALL) NOPASSWD: ALL'],
diff --git a/modules/role/manifests/labs/nfs/secondary.pp
b/modules/role/manifests/labs/nfs/secondary.pp
index e37b4ee..c9c2942 100644
--- a/modules/role/manifests/labs/nfs/secondary.pp
+++ b/modules/role/manifests/labs/nfs/secondary.pp
@@ -7,6 +7,7 @@
description => 'NFS secondary share cluster',
}
+ require ::profile::openstack::main::observerenv
include labstore::fileserver::exports
include labstore::fileserver::secondary
include labstore::backup_keys
diff --git a/modules/role/manifests/wmcs/openstack/labtest/control.pp
b/modules/role/manifests/wmcs/openstack/labtest/control.pp
index b894b4e..58ba38a 100644
--- a/modules/role/manifests/wmcs/openstack/labtest/control.pp
+++ b/modules/role/manifests/wmcs/openstack/labtest/control.pp
@@ -1,4 +1,5 @@
class role::wmcs::openstack::labtest::control {
include profile::openstack::labtest::cloudrepo
+ include profile::openstack::labtest::observerenv
include profile::openstack::labtest::rabbitmq
}
diff --git a/modules/role/manifests/wmcs/openstack/labtest/web.pp
b/modules/role/manifests/wmcs/openstack/labtest/web.pp
index a742a98..afe28cf 100644
--- a/modules/role/manifests/wmcs/openstack/labtest/web.pp
+++ b/modules/role/manifests/wmcs/openstack/labtest/web.pp
@@ -1,3 +1,4 @@
class role::wmcs::openstack::labtest::web {
+ include profile::openstack::labtest::observerenv
include profile::openstack::labtest::cloudrepo
}
diff --git a/modules/role/manifests/wmcs/openstack/labtestn/control.pp
b/modules/role/manifests/wmcs/openstack/labtestn/control.pp
index d175474..82cb4f5 100644
--- a/modules/role/manifests/wmcs/openstack/labtestn/control.pp
+++ b/modules/role/manifests/wmcs/openstack/labtestn/control.pp
@@ -1,4 +1,5 @@
class role::wmcs::openstack::labtestn::control {
include profile::openstack::labtestn::cloudrepo
+ include profile::openstack::labtestn::observerenv
include profile::openstack::labtestn::rabbitmq
}
diff --git a/modules/role/manifests/wmcs/openstack/labtestn/web.pp
b/modules/role/manifests/wmcs/openstack/labtestn/web.pp
index 70b727b..f20496d 100644
--- a/modules/role/manifests/wmcs/openstack/labtestn/web.pp
+++ b/modules/role/manifests/wmcs/openstack/labtestn/web.pp
@@ -1,3 +1,4 @@
class role::wmcs::openstack::labtestn::web {
+ include profile::openstack::labtestn::observerenv
include profile::openstack::labtestn::cloudrepo
}
diff --git a/modules/role/manifests/wmcs/openstack/main/control.pp
b/modules/role/manifests/wmcs/openstack/main/control.pp
index 8476b46..332093b 100644
--- a/modules/role/manifests/wmcs/openstack/main/control.pp
+++ b/modules/role/manifests/wmcs/openstack/main/control.pp
@@ -1,4 +1,5 @@
class role::wmcs::openstack::main::control {
include profile::openstack::main::cloudrepo
+ include profile::openstack::main::observerenv
include profile::openstack::main::rabbitmq
}
diff --git a/modules/role/manifests/wmcs/openstack/main/horizon.pp
b/modules/role/manifests/wmcs/openstack/main/horizon.pp
index ca4c042..5ffc0f2 100644
--- a/modules/role/manifests/wmcs/openstack/main/horizon.pp
+++ b/modules/role/manifests/wmcs/openstack/main/horizon.pp
@@ -2,4 +2,5 @@
# role::wmcs::openstack::main::web when labweb* is finished
class role::wmcs::openstack::main::horizon {
include profile::openstack::main::cloudrepo
+ include profile::openstack::main::observerenv
}
diff --git a/modules/role/manifests/wmcs/openstack/main/net.pp
b/modules/role/manifests/wmcs/openstack/main/net.pp
index 8e5d489..fb2aa96 100644
--- a/modules/role/manifests/wmcs/openstack/main/net.pp
+++ b/modules/role/manifests/wmcs/openstack/main/net.pp
@@ -1,3 +1,5 @@
class role::wmcs::openstack::main::net {
include profile::openstack::main::cloudrepo
+ # for keystone checks which should move to control
+ include profile::openstack::main::observerenv
}
diff --git a/modules/role/manifests/wmcs/openstack/main/web.pp
b/modules/role/manifests/wmcs/openstack/main/web.pp
index 78ba6ff..819e869 100644
--- a/modules/role/manifests/wmcs/openstack/main/web.pp
+++ b/modules/role/manifests/wmcs/openstack/main/web.pp
@@ -1,3 +1,4 @@
class role::wmcs::openstack::main::web {
+ include profile::openstack::main::observerenv
include profile::openstack::main::cloudrepo
}
diff --git a/modules/shinken/files/shinkengen b/modules/shinken/files/shinkengen
index 51b5e38..7ea97c4 100755
--- a/modules/shinken/files/shinkengen
+++ b/modules/shinken/files/shinkengen
@@ -105,8 +105,12 @@
with open('/etc/shinkengen.yaml') as f:
config = yaml.safe_load(f)
+ with open('/etc/novaobserver.yaml') as n:
+ nova_observer = yaml.safe_load(n)
+ observer_pass = nova_observer['OS_PASSWORD']
+
for project in config['projects']:
- instances = get_instances(project, config['observer_pass'])
+ instances = get_instances(project, observer_pass)
host_configs = []
for instance in instances:
co = ConfigObject('host')
diff --git a/modules/shinken/manifests/shinkengen.pp
b/modules/shinken/manifests/shinkengen.pp
index 65aadfd..8b8c0d9 100644
--- a/modules/shinken/manifests/shinkengen.pp
+++ b/modules/shinken/manifests/shinkengen.pp
@@ -13,9 +13,6 @@
ensure => present,
}
- $novaconfig = hiera_hash('novaconfig', {})
- $observer_pass = $novaconfig['observer_password']
-
if $::openstack::version == 'liberty' or ! $::openstack::version {
fail('openstack::version must be set to Mitaka or later for python3
dependencies.')
}
diff --git a/modules/shinken/templates/shinkengen.yaml.erb
b/modules/shinken/templates/shinkengen.yaml.erb
index 7fc0fcd..8b83c94 100644
--- a/modules/shinken/templates/shinkengen.yaml.erb
+++ b/modules/shinken/templates/shinkengen.yaml.erb
@@ -1,4 +1,3 @@
projects: [ 'tools', 'deployment-prep', 'extdist', 'analytics', 'integration',
'shinken', 'cvn', 'wdq-mm', 'wmt' ]
output_path: '/etc/shinken/generated'
-observer_pass: <%= @observer_pass %>
site: <%= scope.lookupvar("::site") %>
--
To view, visit https://gerrit.wikimedia.org/r/368938
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I3b2ab05096095c4be369edeaadeffc64c7cb045f
Gerrit-PatchSet: 4
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Rush <[email protected]>
Gerrit-Reviewer: Alex Monk <[email protected]>
Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: Rush <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
