BryanDavis has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/343956 )
Change subject: wmflib: sync with upstream
......................................................................
wmflib: sync with upstream
Copy in the version of wmflib from operations/puppet.git @ 69aba47
Change-Id: I3ef41e75539befb10f13394b03d9d47cb90ef1dd
---
A puppet/modules/wmflib/.rspec
M puppet/modules/wmflib/README.md
A puppet/modules/wmflib/Rakefile
A puppet/modules/wmflib/lib/hiera/backend/httpyaml_backend.rb
M puppet/modules/wmflib/lib/hiera/backend/mwyaml_backend.rb
M puppet/modules/wmflib/lib/hiera/backend/nuyaml_backend.rb
A puppet/modules/wmflib/lib/hiera/backend/proxy_backend.rb
A puppet/modules/wmflib/lib/hiera/backend/role_backend.rb
A puppet/modules/wmflib/lib/hiera/httpcache.rb
M puppet/modules/wmflib/lib/hiera/mwcache.rb
A puppet/modules/wmflib/lib/puppet/parser/functions/conflicts.rb
A puppet/modules/wmflib/lib/puppet/parser/functions/conftool.rb
A puppet/modules/wmflib/lib/puppet/parser/functions/cron_splay.rb
A puppet/modules/wmflib/lib/puppet/parser/functions/ensure_mounted.rb
A puppet/modules/wmflib/lib/puppet/parser/functions/get_clusters.rb
A puppet/modules/wmflib/lib/puppet/parser/functions/hash_deselect_re.rb
A puppet/modules/wmflib/lib/puppet/parser/functions/hash_select_re.rb
A puppet/modules/wmflib/lib/puppet/parser/functions/htpasswd.rb
A puppet/modules/wmflib/lib/puppet/parser/functions/ini.rb
A puppet/modules/wmflib/lib/puppet/parser/functions/ipresolve.rb
M puppet/modules/wmflib/lib/puppet/parser/functions/os_version.rb
A puppet/modules/wmflib/lib/puppet/parser/functions/puppet_ssldir.rb
M puppet/modules/wmflib/lib/puppet/parser/functions/require_package.rb
A puppet/modules/wmflib/lib/puppet/parser/functions/requires_os.rb
A puppet/modules/wmflib/lib/puppet/parser/functions/role.rb
A puppet/modules/wmflib/lib/puppet/parser/functions/secret.rb
M puppet/modules/wmflib/lib/puppet/parser/functions/ssl_ciphersuite.rb
A puppet/modules/wmflib/lib/puppet/parser/functions/validate_array_re.rb
A puppet/modules/wmflib/spec/fixtures/hiera.proxy.yaml
A puppet/modules/wmflib/spec/fixtures/hiera.yaml
A puppet/modules/wmflib/spec/fixtures/hieradata/common.yaml
A puppet/modules/wmflib/spec/fixtures/hieradata/hosts/foo.yaml
A puppet/modules/wmflib/spec/fixtures/hieradata/role/common/test.yaml
A puppet/modules/wmflib/spec/fixtures/hieradata/role/common/test2.yaml
A puppet/modules/wmflib/spec/functions/conftool_spec.rb
A puppet/modules/wmflib/spec/functions/ensure_mounted_spec.rb
A puppet/modules/wmflib/spec/functions/hash_deselect_re_spec.rb
A puppet/modules/wmflib/spec/functions/hash_select_re_spec.rb
A puppet/modules/wmflib/spec/functions/ipresolve_spec.rb
A puppet/modules/wmflib/spec/functions/role_spec.rb
A puppet/modules/wmflib/spec/hiera/proxy_backend_spec.rb
A puppet/modules/wmflib/spec/hiera/role_backend_spec.rb
A puppet/modules/wmflib/spec/spec_helper.rb
43 files changed, 2,111 insertions(+), 176 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant
refs/changes/56/343956/1
diff --git a/puppet/modules/wmflib/.rspec b/puppet/modules/wmflib/.rspec
new file mode 100644
index 0000000..f449dae
--- /dev/null
+++ b/puppet/modules/wmflib/.rspec
@@ -0,0 +1,2 @@
+--format doc
+--color
diff --git a/puppet/modules/wmflib/README.md b/puppet/modules/wmflib/README.md
index 882a17a..f6ab3fe 100644
--- a/puppet/modules/wmflib/README.md
+++ b/puppet/modules/wmflib/README.md
@@ -15,6 +15,18 @@
$packages = apply_format('texlive-lang-%s', $languages)
+## conflicts
+
+`conflicts( string|resource $resource )`
+
+Throw an error if a resource is declared.
+
+### Examples
+
+ conflicts('::redis::legacy')
+ conflicts(Class['::redis-server'])
+
+
## ensure_directory
`ensure_directory( string|bool $ensure )`
@@ -66,6 +78,33 @@
}
+## ensure_mounted
+
+`ensure_mounted( string|bool $ensure )`
+
+Takes a generic `ensure` parameter value and convert it to an
+appropriate value for use with a mount declaration.
+
+If `$ensure` is `true` or `present`, the return value is `mounted`.
+Otherwise, the return value is the unmodified `$ensure` parameter.
+
+### Examples
+
+ # Sample class which mounts or unmounts '/var/lib/nginx'
+ # based on the class's generic $ensure parameter:
+ class nginx ( $ensure = present ) {
+ package { 'nginx-full':
+ ensure => $ensure,
+ }
+ mount { '/var/lib/nginx':
+ ensure => ensure_mounted($ensure),
+ device => 'tmpfs',
+ fstype => 'tmpfs',
+ options => 'defaults,noatime,uid=0,gid=0,mode=755,size=1g',
+ }
+ }
+
+
## ensure_service
`ensure_service( string|bool $ensure )`
@@ -89,6 +128,52 @@
require => Package['redis-server'],
}
}
+
+
+## hash_deselect_re
+
+`hash_deselect_re( string $regex, hash $input )`
+
+Does exactly the opposite of hash_select_re below: keys matching
+the regex are *excluded* from the new hash.
+
+
+## hash_select_re
+
+`hash_select_re( string $regex, hash $input )`
+
+This creates a new hash from the input hash, but only copies the
+keys which match the regex. In other words, it does the
+equivalent of this in Ruby pseudo-code:
+
+ return input.select { |k, _v| regex.match(k) }
+
+### Example
+
+ hash_select_re('^a', {"abc" => 1, "def" => 2, "asdf" => 3})
+
+will produce:
+
+ {"abc" => 1, "asdf" => 3}
+
+
+## ini
+
+`ini( hash $ini_settings [, hash $... ] )`
+
+Serialize a hash into the .ini-style format expected by Python's
+ConfigParser. Takes one or more hashes as arguments. If the argument
+list contains more than one hash, they are merged together. In case of
+duplicate keys, hashes to the right win.
+
+### Example
+
+ ini({'server' => {'port' => 80}})
+
+will produce:
+
+ [server]
+ port = 80
## ordered_json
@@ -127,6 +212,30 @@
file { '/etc/kibana/config.yaml':
content => ordered_yaml($options),
}
+
+
+## os_version
+
+`os_version( string $version_predicate )`
+
+Performs semantic OS version comparison.
+
+Takes one or more string arguments, each containing one or more predicate
+expressions. Each expression consts of a distribution name, followed by a
+comparison operator, followed by a release name or number. Multiple clauses
+are OR'd together. The arguments are case-insensitive.
+
+The host's OS version will be compared to to the comparison target
+using the specified operator, returning a boolean. If no operator is
+present, the equality operator is assumed.
+
+### Examples
+
+ # True if Ubuntu Trusty or newer or Debian Jessie or newer
+ os_version('ubuntu >= trusty || debian >= Jessie')
+
+ # True if exactly Debian Jessie
+ os_version('debian jessie')
## php_ini
@@ -177,23 +286,23 @@
requires_realm('labs')
-## requires_ubuntu
+## requires_os
-`requires_ubuntu( string $version_predicate )`
+`requires_os( string $version_predicate )`
-Validate that the host Ubuntu version satisfies a version
+Validate that the host OS version satisfies a version
check. Abort catalog compilation if not.
-See the documentation for ubuntu_version() for supported
+See the documentation for os_version() for supported
predicate syntax.
### Examples
- # Fail unless version is Trusty
- requires_ubuntu('trusty')
+ # Fail unless version is Trusty or Jessie
+ requires_os('ubuntu trusty || debian jessie')
# Fail unless Trusty or newer
- requires_ubuntu('> trusty')
+ requires_os('ubuntu >= trusty')
@@ -225,21 +334,18 @@
## ssl_ciphersuite
-`ssl_ciphersuite( string $servercode, string $encryption_type, int $hsts_days
)`
+`ssl_ciphersuite( string $servercode, string $encryption_type, boolean $hsts )`
Outputs the ssl configuration directives for use with either Nginx
or Apache using our selection of ciphers and SSL options.
Takes three arguments:
-- The servercode, or which browser-version combination to
- support. At the moment only 'apache-2.2', 'apache-2.4' and 'nginx'
- are supported.
+- The server to configure for: 'apache' or 'nginx'
- The compatibility mode,indicating the degree of compatibility we
want to retain with older browsers (basically, IE6, IE7 and
Android prior to 3.0)
-- An optional argument, that if non-nil will set HSTS to max-age of
- N days
+- hsts - optional boolean, true emits our standard public HSTS
Whenever called, this function will output a list of strings that
can be safely used in your configuration file as the ssl
@@ -247,7 +353,7 @@
### Examples
- ssl_ciphersuite('apache-2.4', 'compat')
+ ssl_ciphersuite('apache', 'compat', true)
ssl_ciphersuite('nginx', 'strong')
@@ -274,50 +380,22 @@
to_seconds('2 days') # 172800
-## ubuntu_version
+## validate_array_re
+`validate_array_re( array $items, string $re )`
-`ubuntu_version( string $version_predicate )`
-
-Performs semantic Ubuntu version comparison.
-
-Takes a single string argument containing a comparison operator
-followed by an optional space, followed by a comparison target,
-provided as Ubuntu version number or release name.
-
-The host's Ubuntu version will be compared to to the comparison target
-using the specified operator, returning a boolean. If no operator is
-present, the equality operator is assumed.
-
-Release names are case-insensitive. The comparison operator and
-comparison target can be provided as two separate arguments, if you
-prefer.
+Throw an error if any member of $items does not match the regular
+expression $re.
### Examples
- # True if Precise or newer
- ubuntu_version('>= precise')
- ubuntu_version('>= 12.04.4')
+ # OK -- each array item is a four-digit number.
+ validate_array_re([8123, 8124, 8125], '^\d{4}$')
- # True if older than Utopic
- ubuntu_version('< utopic')
-
- # True if newer than Precise
- ubuntu_version('> precise')
-
- # True if Trusty or older
- ubuntu_version('<= trusty')
-
- # True if exactly Trusty
- ubuntu_version('trusty')
- ubuntu_version('== trusty')
-
- # True if anything but Trusty
- ubuntu_version('!trusty')
- ubuntu_version('!= trusty')
+ # Fail -- last array item is not a four-digit number.
+ validate_array_re([8123, 8124, 812], '^\d{4}$')
## validate_ensure
-
`validate_ensure( string $ensure )`
Throw an error if the $ensure argument is not 'present' or 'absent'.
diff --git a/puppet/modules/wmflib/Rakefile b/puppet/modules/wmflib/Rakefile
new file mode 100644
index 0000000..a0c3b79
--- /dev/null
+++ b/puppet/modules/wmflib/Rakefile
@@ -0,0 +1,37 @@
+require 'rake'
+require 'fileutils'
+
+require 'rspec/core/rake_task'
+
+modulename = File.basename(File.expand_path(File.dirname(__FILE__)))
+
+symlinks = { 'spec/fixtures/modules/%s/files' % modulename =>
'../../../../files',
+ 'spec/fixtures/modules/%s/manifests' % modulename =>
'../../../../manifests',
+ 'spec/fixtures/modules/%s/templates' % modulename =>
'../../../../templates',
+ }
+
+
+task :setup do
+ FileUtils.mkdir_p('spec/fixtures/modules/%s' % modulename)
+ symlinks.each do |x|
+ if !File.exist?(x[0])
+ FileUtils.ln_s(x[1], x[0])
+ end
+ end
+end
+
+task :teardown do
+ symlinks.each { |x| FileUtils.rm(x[0], :force => true) }
+ FileUtils.rmdir('spec/fixtures/modules/%s' % modulename)
+ FileUtils.rmdir('spec/fixtures/modules')
+end
+
+RSpec::Core::RakeTask.new(:realspec) do |t|
+ t.fail_on_error = false
+ t.pattern = 'spec/*/*_spec.rb'
+end
+
+task :spec_standalone => [ :setup, :realspec, :teardown]
+
+task :default => :spec do
+end
diff --git a/puppet/modules/wmflib/lib/hiera/backend/httpyaml_backend.rb
b/puppet/modules/wmflib/lib/hiera/backend/httpyaml_backend.rb
new file mode 100644
index 0000000..fae9f51
--- /dev/null
+++ b/puppet/modules/wmflib/lib/hiera/backend/httpyaml_backend.rb
@@ -0,0 +1,44 @@
+require "hiera/httpcache"
+class Hiera
+ module Backend
+ class Httpyaml_backend
+ def initialize
+ @cache = Httpcache.new
+ end
+
+ def lookup(key, scope, order_override, resolution_type)
+ answer = nil
+ Hiera.debug("Looking up #{key}")
+
+ Backend.datasources(scope, order_override) do |source|
+ # Small hack: We don't want to search any datasource but the
+ # httpyaml/%{::labsproject} hierarchy here; so we plainly exit
+ # in any other case.
+ next unless source.start_with?('httpyaml/') && source.length >
'httpyaml/'.length
+
+ data = @cache.read(source)
+
+ next if data.nil? || data.empty?
+ next unless data.include?(key)
+
+ new_answer = Backend.parse_answer(data[key], scope)
+ case resolution_type
+ when :array
+ raise Exception, "Hiera type mismatch: expected Array and got
#{new_answer.class}" unless new_answer.kind_of?(Array) ||
new_answer.kind_of?(String)
+ answer ||= []
+ answer << new_answer
+ when :hash
+ raise Exception, "Hiera type mismatch: expected Hash and got
#{new_answer.class}" unless new_answer.kind_of? Hash
+ answer ||= {}
+ answer = Backend.merge_answer(new_answer, answer)
+ else
+ answer = new_answer
+ break
+ end
+ end
+
+ answer
+ end
+ end
+ end
+end
diff --git a/puppet/modules/wmflib/lib/hiera/backend/mwyaml_backend.rb
b/puppet/modules/wmflib/lib/hiera/backend/mwyaml_backend.rb
index 6e0c638..5f6253e 100644
--- a/puppet/modules/wmflib/lib/hiera/backend/mwyaml_backend.rb
+++ b/puppet/modules/wmflib/lib/hiera/backend/mwyaml_backend.rb
@@ -11,25 +11,30 @@
Hiera.debug("Looking up #{key}")
Backend.datasources(scope, order_override) do |source|
- # Small hack: - we don't want to search any datasource but the
- # labs/%{::instanceproject} hierarchy here; so we plainly exit
- # in any other case
- if m = /labs\/([^\/]+)$/.match(source)
- source = m[1].capitalize
- else
- next
- end
+ # Small hack: We don't want to search any datasource but the
+ # labs/%{::labsproject} hierarchy here; so we plainly exit
+ # in any other case.
+ next unless source.start_with?('labs/') && source.length >
'labs/'.length
+
+ # For hieradata/, the hierarchy is defined as
+ # "labs/%{::labsproject}/host/%{::hostname}" and
+ # "labs/%{::labsproject}/common". We map the former
+ # verbatim to "Hiera:$labsproject/host/$hostname", while the
+ # latter gets simplified to "Hiera:$labsproject". In both
+ # cases, we capitalize $labsproject.
+ source = source['labs/'.length..-1].chomp('/common').capitalize
+
data = @cache.read(source, Hash, {}) do |content|
YAML.load(content)
end
- next if data.empty?
+ next if data.nil? || data.empty?
next unless data.include?(key)
new_answer = Backend.parse_answer(data[key], scope)
case resolution_type
when :array
- raise Exception, "Hiera type mismatch: expected Array and got
#{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of?
String
+ raise Exception, "Hiera type mismatch: expected Array and got
#{new_answer.class}" unless new_answer.kind_of?(Array) ||
new_answer.kind_of?(String)
answer ||= []
answer << new_answer
when :hash
diff --git a/puppet/modules/wmflib/lib/hiera/backend/nuyaml_backend.rb
b/puppet/modules/wmflib/lib/hiera/backend/nuyaml_backend.rb
index 2f4e376..ad01bf9 100644
--- a/puppet/modules/wmflib/lib/hiera/backend/nuyaml_backend.rb
+++ b/puppet/modules/wmflib/lib/hiera/backend/nuyaml_backend.rb
@@ -7,7 +7,7 @@
#
#
# This backend allows some more flexibility over the vanilla yaml
-# backend, as path expansion in the lookup and even dynamic lookups.
+# backend, as path expansion in the lookup.
#
# == Private path
#
@@ -21,7 +21,6 @@
# :expand_path be expanded when looking the file up on disk. This
# allows both to have a more granular set of files, but also to avoid
# unnecessary cache evictions for cached data.
-#
# === Example
#
# Say your hiera.yaml has defined
@@ -41,33 +40,30 @@
#
# Unless very small, all files should be split up like this.
#
-# == Dynamic lookup
+# == Regexp matching
#
-# Sometimes we want to search for data based on variables... that are
-# hosted within hiera! Dynamic lookup allows to define hierachies that
-# will determine the full path based on results from hiera
-# itself. Tricky? Let's see with an example
+# As multiple hosts may correspond to the same rules/configs in a
+# large cluster, we allow to define a self-contained "regex.yaml" file
+# in your datadir, where each different class of servers may be
+# represented by a label and a corresponding regexp.
#
# === Example
+# Say you have a lookup for "cluster", and you have
+#"regex/%{hostname}" in your hierarchy; also, let's say that your
+# scope contains hostname = "web1001.local". So if your regex.yaml
+# file contains:
#
-# Say you have in your hiera config
-# :nuyaml:
-# :dynamic_lookup:
-# - role
-# :hierarchy:
-# - "host/%{fqdn}"
-# - role
+# databases:
+# __regex: !ruby/regex '/db.*\.local/'
+# cluster: db
#
-# What will happen will be that any key we search (say $cluster) will
-# be first searched in the specific file for that host
-# (host/hostname.yaml), if not found, it will be searched as follows:
-# - if host/hostname.yaml contains a value for role, say
-# 'refrigerator', then lookup will continue in the
-# 'role/refrigerator.yaml' file
-# - else it will looked up in the 'role/default.yaml'
+# webservices:
+# __regex: !ruby/regex '/^web.*\.local$/'
+# cluster: www
#
-# Note that for added fun you may declare one part of the hierarchy to
-# be both dynamically looked up and expanded. It works!
+# This will make it so that "cluster" will assume the value "www"
+# given the regex matches the "webservices" stanza
+#
class Hiera
module Backend
class Nuyaml_backend
@@ -76,12 +72,17 @@
require 'yaml'
@cache = cache || Filecache.new
config = Config[:nuyaml]
- @dynlookup = config[:dynlookup] || []
@expand_path = config[:expand_path] || []
end
def get_path(key, scope, source)
config_section = :nuyaml
+ # Special case: regex
+ if m = /^regex\//.match(source)
+ Hiera.debug("Regex match going on - using regex.yaml")
+ return Backend.datafile(config_section, scope, 'regex', "yaml")
+ end
+
# Special case: 'private' repository.
# We use a different datadir in this case.
# Example: private/common will search in the common source
@@ -91,23 +92,52 @@
source = m[1]
end
+ # Special case: 'secret' repository. This is practically labs only
+ # We use a different datadir in this case.
+ # Example: private/common will search in the common source
+ # within the private datadir
+ if m = /secret\/(.*)/.match(source)
+ config_section = :secret
+ source = m[1]
+ end
+
Hiera.debug("The source is: #{source}")
# If the source is in the expand_path list, perform path
# expansion. This is thought to allow large codebases to live
# with fairly small yaml files as opposed to a very large one.
# Example:
- # $apache::mpm::worker => 'worker' in common/apache/mpm.yaml
- if @expand_path.include? source
+ # $apache::mpm::worker will be in common/apache/mpm.yaml
+ paths = @expand_path.map{ |x| Backend.parse_string(x, scope) }
+ if paths.include? source
namespaces = key.gsub(/^::/,'').split('::')
- newkey = namespaces.pop
+ namespaces.pop
unless namespaces.empty?
source += "/".concat(namespaces.join('/'))
- key = newkey
end
end
- return key, Backend.datafile(config_section, scope, source, "yaml")
+ return Backend.datafile(config_section, scope, source, "yaml")
+ end
+
+ def plain_lookup(key, data, scope)
+ return nil unless data.include?(key)
+ return Backend.parse_answer(data[key], scope)
+ end
+
+ def regex_lookup(key, matchon, data, scope)
+ data.each do |label, datahash|
+ r = datahash["__regex"]
+ Hiera.debug("Scanning label #{label} for matches to '#{r}' in
'#{matchon}' ")
+ next unless r.match(matchon)
+ Hiera.debug("Label #{label} matches; searching within it")
+ next unless datahash.include?(key)
+ return Backend.parse_answer(datahash[key], scope)
+ end
+ return nil
+ rescue => detail
+ Hiera.debug(detail)
+ return nil
end
def lookup(key, scope, order_override, resolution_type)
@@ -116,23 +146,11 @@
Hiera.debug("Looking up #{key}")
Backend.datasources(scope, order_override) do |source|
- # Yes this is kind of hacky. We look it up again on hiera,
- # and build a source based on the lookup.
- if @dynlookup.include? source
- Hiera.debug("Dynamic lookup for source #{source}")
- if key == source
- next
- end
- dynsource = lookup(source, scope, order_override, :priority)
- dynsource ||= 'default'
- source += "/#{dynsource}"
- end
-
Hiera.debug("Loading info from #{source} for #{key}")
- lookup_key, yamlfile = get_path(key, scope, source)
+ yamlfile = get_path(key, scope, source)
- Hiera.debug("Searching for #{lookup_key} in #{yamlfile}")
+ Hiera.debug("Searching for #{key} in #{yamlfile}")
next if yamlfile.nil?
@@ -144,14 +162,21 @@
YAML.load(content)
end
- next if data.empty?
- next unless data.include?(lookup_key)
+ next if data.nil?
+ if m = /regex\/(.*)$/.match(source)
+ matchto = m[1]
+ new_answer = regex_lookup(key, matchto, data, scope)
+ else
+ new_answer = plain_lookup(key, data, scope)
+ end
+
+ next if new_answer.nil?
# Extra logging that we found the key. This can be outputted
# multiple times if the resolution type is array or hash but that
# should be expected as the logging will then tell the user ALL the
# places where the key is found.
- Hiera.debug("Found #{lookup_key} in #{source}")
+ Hiera.debug("Found #{key} in #{source}")
# for array resolution we just append to the array whatever
# we find, we then goes onto the next file and keep adding to
@@ -160,10 +185,9 @@
# for priority searches we break after the first found data
# item
- new_answer = Backend.parse_answer(data[lookup_key], scope)
case resolution_type
when :array
- raise Exception, "Hiera type mismatch: expected Array and got
#{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of?
String
+ raise Exception, "Hiera type mismatch: expected Array and got
#{new_answer.class}" unless new_answer.kind_of?(Array) ||
new_answer.kind_of?(String)
answer ||= []
answer << new_answer
when :hash
diff --git a/puppet/modules/wmflib/lib/hiera/backend/proxy_backend.rb
b/puppet/modules/wmflib/lib/hiera/backend/proxy_backend.rb
new file mode 100644
index 0000000..b8a77b2
--- /dev/null
+++ b/puppet/modules/wmflib/lib/hiera/backend/proxy_backend.rb
@@ -0,0 +1,91 @@
+# Proxy Hiera backend - for complex hiera queries
+#
+# Author: Giuseppe Lavagetto
+# Copyright (c) 2015 Wikimedia Foundation
+#
+# This backend allows you to plug multiple simpler backends that use
+# Backend.datasources in their lookup method to work toghether as if
+# they were part of the same hierarchy.
+#
+
+# Add Hiera::Config the ability to get overridden values
+# We use this to trick the other backends in doing one and only one
+# lookup each
+class Hiera::Config
+ class << self
+ def []=(key,value)
+ @config[key] = value
+ end
+ end
+end
+
+class Hiera
+ module Backend
+ class Proxy_backend
+ def initialize
+ Hiera.debug "Starting the proxy backend"
+ @config = Config[:proxy]
+ self.load_plugins
+ end
+
+ def load_plugins
+ @plugins ||={}
+ #Load plugins only once
+ @config[:plugins].each do |plugin|
+ Hiera.debug "Loading plugin #{plugin}"
+ begin
+ require "hiera/backend/#{plugin.downcase}_backend"
+ @plugins[plugin] ||=
+ Backend.const_get("#{plugin.capitalize}_backend").new
+ rescue
+ Hiera.warn "Failure: plugin #{plugin} failed to load"
+ end
+ end
+ end
+
+ def lookup(key, scope, order_override, resolution_type)
+ answer = nil
+ hierarchy = Config[:hierarchy].clone
+
+ Backend.datasources(scope, order_override) do |source|
+ if source.include? '@@'
+ plugin, source = source.split('@@')
+ else
+ plugin = @config[:default_plugin]
+ end
+ if not @plugins.include? plugin
+ Hiera.
+ warn "Hierarchy specifies to use plugin '#{plugin}' but can't
find it"
+ next
+ end
+ # We look up onto a foreign backend by limiting us to a
+ # single element of hierarchy.
+ Config[:hierarchy] = [source]
+ new_answer = @plugins[plugin].
+ lookup(key, scope, order_override,
+ resolution_type)
+ Config[:hierarchy] = hierarchy
+
+ next if new_answer.nil?
+
+ case resolution_type
+ when :array
+ raise Exception, "Hiera type mismatch: expected Array and got
#{new_answer.class}" unless new_answer.kind_of? Array
+ # The plugins already return an array of answers, so just
concatenate it.
+ answer ||= []
+ answer += new_answer
+ when :hash
+ raise Exception, "Hiera type mismatch: expected Hash and got
#{new_answer.class}" unless new_answer.kind_of? Hash
+ answer ||= {}
+ answer = Backend.merge_answer(new_answer,answer)
+ else
+ answer = new_answer
+ break
+ end
+ end
+
+ answer
+ end
+ end
+ end
+end
diff --git a/puppet/modules/wmflib/lib/hiera/backend/role_backend.rb
b/puppet/modules/wmflib/lib/hiera/backend/role_backend.rb
new file mode 100644
index 0000000..8de978c
--- /dev/null
+++ b/puppet/modules/wmflib/lib/hiera/backend/role_backend.rb
@@ -0,0 +1,177 @@
+# Role-based Hiera backend
+#
+# Author: Giuseppe Lavagetto <[email protected]>
+# Copyright (c) 2014 Wikimedia Foundation
+#
+#
+# This backend allows to group definitions based on the roles defined
+# at the node level using the 'role' keyword.
+# It allows searching in hierarchies that are based on the role
+# currently applied; this is very handy if you have group of hosts
+# whose configuration is dependent on their role rather than on other
+# facts like hostnames.
+#
+# == How this works
+#
+# Whenever you use the 'role' keyword in a node:
+#
+# role cache::text
+#
+# Two things happen:
+# - the class role::cache::text gets included in the current node
+# - this gets registered in a global registry
+#
+# Whenever a hiera lookup is performed and the role backend is used,
+# the key is searched in all the hierarchies defined in the
+# :role_hierarchy.
+#
+# A typical hierarchy would be:
+# - "%{::environment}"
+# - common
+#
+# The path in which the files must be is as follows:
+# role/${hierarchy}/<path>.yaml
+# where <path> corresponds to the argument passed to the 'role'
+# keyword.
+#
+# === Example
+#
+# In site.pp:
+#
+# node /pinkunicorn.wikimedia.org/ {
+# role foo::bar, fizzbuzz
+# notice(hiera('admin::groups'))
+# }
+#
+# Give the hierarchy from before, the final hiera lookup will result
+# in searching admin::groups in two distinct hierarchies:
+#
+# role/production/foo/bar.yaml
+# role/common/foo/bar.yaml
+#
+# role/production/fizzbuzz.yaml
+# role/common/fizzbuzz.yaml
+#
+# The research will be performed for *both* hierarchies as a normal
+# hiera lookup; if a plain lookup is performed, the results of the two
+# searches will be compared; if they differ, an exception will be
+# thrown, so that conflicting directives will need manual resolution.
+#
+# === Things to pay attention to
+#
+# One big caveat: if you do use multiple times the role keyword, any
+# class included by the first role keyword is declared would be
+# evaluated just after the first "role" has been called, thus
+# not having the full scope of both to search from; this could result
+# in unexpected behaviour so I advice _against_ using multiple role
+# keywords in a single node, if both roles include conflicting
+# classes.
+require 'yaml'
+class Hiera
+ module Backend
+ class Role_backend
+ def initialize(cache=nil)
+ @cache = cache || Filecache.new
+ end
+
+ def get_path(key, role, source, scope)
+ config_section = :role
+
+ # Special case: 'private' repository.
+ # We use a different datadir in this case.
+ # Example: private/common will search in the role/common source
+ # within the private datadir
+ if m = /private\/(.*)/.match(source)
+ config_section = :private
+ source = m[1]
+ end
+
+ # Variables for role::foo::bar will be searched in:
+ # role/foo/bar.yaml
+ # role/$::site/foo/bar.yaml
+ # etc, depending on your hierarchy
+ path = role.split('::').join('/')
+ src = "role/#{source}/#{path}"
+
+ # Use the datadir for the 'role' section of the config
+ return Backend.datafile(config_section, scope, src, "yaml")
+ end
+
+ def merge_answer(new_answer, answer, resolution_type)
+ case resolution_type
+ when :array
+ raise Exception, "Hiera type mismatch: expected Array and got
#{new_answer.class}" unless new_answer.kind_of?(Array) ||
new_answer.kind_of?(String)
+ answer ||= []
+ answer << new_answer
+ when :hash
+ raise Exception, "Hiera type mismatch: expected Hash and got
#{new_answer.class}" unless new_answer.kind_of? Hash
+ answer ||= {}
+ answer = Backend.merge_answer(new_answer,answer)
+ else
+ answer = new_answer
+ return true, answer
+ end
+ return false, answer
+ end
+
+ def lookup(key, scope, order_override, resolution_type)
+ topscope_var = '_roles'
+ resultset = nil
+ return nil unless scope.include?topscope_var
+ roles = scope[topscope_var]
+ return nil if roles.nil?
+ if Config.include?(:role_hierarchy)
+ hierarchy = Config[:role_hierarchy]
+ else
+ hierarchy = nil
+ end
+
+ roles.keys.each do |role|
+ Hiera.debug("Looking in hierarchy for role #{role}")
+ answer = nil
+ Backend.datasources(scope, order_override, hierarchy) do |source|
+ yamlfile = get_path(key,role,source, scope)
+ next if yamlfile.nil?
+ Hiera.debug("Searching in file #{yamlfile} for #{key}")
+ next unless File.exist?(yamlfile)
+
+ data = @cache.read(yamlfile, Hash) do |content|
+ YAML.load(content)
+ end
+
+ next if data.nil? || data.empty?
+
+ next unless data.include? key
+
+ new_answer = Backend.parse_answer(data[key], scope)
+ Hiera.debug("Found: #{key} => #{new_answer}")
+ is_done, answer = merge_answer(new_answer, answer,
+ resolution_type)
+ break if is_done
+ end
+ # skip parsing if no answer was found.
+ next if answer.nil?
+
+ # Now we got one answer for this role, we can merge it with
+ # what we got earlier.
+ case resolution_type
+ when :array
+ resultset ||= []
+ answer.each { |el| resultset.push(el) }
+ when :hash
+ resultset ||= {}
+ resultset = Backend.merge_answer(answer, resultset)
+ else
+ # We raise an exception if we have received conflicting results
+ if resultset && answer != resultset
+ raise Exception, "Conflicting value for #{key} found in role
#{role}"
+ else
+ resultset = answer
+ end
+ end
+ end
+ resultset
+ end
+ end
+ end
+end
diff --git a/puppet/modules/wmflib/lib/hiera/httpcache.rb
b/puppet/modules/wmflib/lib/hiera/httpcache.rb
new file mode 100644
index 0000000..270c449
--- /dev/null
+++ b/puppet/modules/wmflib/lib/hiera/httpcache.rb
@@ -0,0 +1,83 @@
+class Hiera
+ class Httpcache < Filecache
+ def initialize
+ super
+ require 'httpclient'
+ require 'yaml'
+ require 'json'
+ config = Config[:httpyaml]
+ @url_prefix = config[:url_prefix]
+ @http = HTTPClient.new(:agent_name => 'HieraHttpCache/0.1')
+
+ # Use the operating system's certificate store, not ruby-httpclient's
cacert.p7s which doesn't
+ # even have the root used to sign Let's Encrypt CAs (DST Root CA X3)
+ @http.ssl_config.clear_cert_store
+ @http.ssl_config.set_default_paths
+
+ @stat_ttl = config[:cache_ttl] || 60
+ if defined? @http.ssl_config.ssl_version
+ @http.ssl_config.ssl_version = 'TLSv1'
+ else
+ # Note: this seem to work in later versions of the library,
+ # but has no effect. How cute, I <3 ruby.
+ @http.ssl_config.options = OpenSSL::SSL::OP_NO_SSLv3
+ end
+ end
+
+ def read(path, expected_type=Hash, default=nil)
+ read_file(path)
+ rescue => detail
+ # When failing to read data, we raise an exception, see
https://phabricator.wikimedia.org/T78408
+ error = "Reading data from #{path} failed: #{detail.class}: #{detail}"
+ raise error
+ end
+
+ def read_file(path)
+ if stale?(path)
+ data = get_from_http(path)
+ @cache[path][:data] = data
+
+ if !@cache[path][:data].is_a?(Object)
+ raise TypeError, "Data retrieved from #{path} is #{data.class} not
Object"
+ end
+ end
+
+ @cache[path][:data]
+ end
+
+ private
+
+ def stale?(path)
+ # We don't actually have caching information lol
+ # So we just, uh, blindly cache for 60s
+ now = Time.now.to_i
+ if @cache[path].nil?
+ @cache[path] = {:data => nil, :meta => {:ts => now}}
+ return true
+ elsif (now - @cache[path][:meta][:ts]) <= @stat_ttl
+ # if we already fetched the result within the last stat_ttl seconds,
+ # we don't bother killing the mediawiki instance with a flood of
requests
+ return false
+ else
+ # This means there's a ts and it's old enough
+ @cache[path][:meta][:ts] = now
+ return true
+ end
+ end
+
+ def get_from_http(path)
+ url = path.sub('httpyaml/', @url_prefix)
+ Hiera.debug("Fetching #{url}")
+ res = @http.get(url)
+ if res.status_code != 200
+ raise IOError, "Could not correctly fetch revision for #{path}, HTTP
status code #{res.status_code}, content #{res.data}"
+ end
+ # We shamelessly throw exceptions here, and catch them upper in the chain
+ # specifically in Hiera::Mwcache.stale? and Hiera::Mwcache.read
+ # FIXME: use safe_load here somehow?
+ body = YAML.load(res.body)
+
+ body['hiera']
+ end
+ end
+end
diff --git a/puppet/modules/wmflib/lib/hiera/mwcache.rb
b/puppet/modules/wmflib/lib/hiera/mwcache.rb
index 485f167..191eedb 100644
--- a/puppet/modules/wmflib/lib/hiera/mwcache.rb
+++ b/puppet/modules/wmflib/lib/hiera/mwcache.rb
@@ -1,4 +1,7 @@
class Hiera
+ class MediawikiPageNotFoundError < Exception
+ end
+
class Mwcache < Filecache
def initialize
super
@@ -9,14 +12,32 @@
@httphost = config[:host] || 'https://wikitech.wikimedia.org'
@endpoint = config[:endpoint] || '/w/api.php'
@http = HTTPClient.new(:agent_name => 'HieraMwCache/0.1')
+
+ # Use the operating system's certificate store, not ruby-httpclient's
cacert.p7s which doesn't
+ # even have the root used to sign Let's Encrypt CAs (DST Root CA X3)
+ @http.ssl_config.clear_cert_store
+ @http.ssl_config.set_default_paths
+
@stat_ttl = config[:cache_ttl] || 60
if defined? @http.ssl_config.ssl_version
- @http.ssl_config.ssl_version = 'TLSv1_2'
+ @http.ssl_config.ssl_version = 'TLSv1'
else
# Note: this seem to work in later versions of the library,
# but has no effect. How cute, I <3 ruby.
@http.ssl_config.options = OpenSSL::SSL::OP_NO_SSLv3
end
+ end
+
+ def read(path, expected_type, default=nil, &block)
+ read_file(path, expected_type, &block)
+ rescue Hiera::MediawikiPageNotFoundError => detail
+ # Any errors other than this will cause hiera to raise an error and
puppet to fail.
+ Hiera.debug("Page #{detail} is non-existent, setting defaults
#{default}")
+ @cache[path][:data] = default
+ rescue => detail
+ # When failing to read data, we raise an exception, see
https://phabricator.wikimedia.org/T78408
+ error = "Reading data from #{path} failed: #{detail.class}: #{detail}"
+ raise error
end
def read_file(path, expected_type = Object, &block)
@@ -25,8 +46,8 @@
data = resp["*"]
@cache[path][:data] = block_given? ? yield(data) : data
- if !@cache[path][:data].is_a?(expected_type)
- raise TypeError, "Data retrieved from #{path} is #{data.class} not
#{expected_type}"
+ if !@cache[path][:data].nil? &&
!@cache[path][:data].is_a?(expected_type)
+ raise TypeError, "Data retrieved from #{path} is
#{@cache[path][:data].class}, not #{expected_type} or nil"
end
end
@@ -50,8 +71,9 @@
@cache[path][:meta] = meta
return true
end
- rescue => detail
- error = "Retrieving metadata from #{path} failed: #{detail}"
+ rescue Hiera::MediawikiPageNotFoundError => detail
+ # Any errors other than this will cause hiera to raise an error and
puppet to fail.
+ error = "Page #{detail} is non-existent"
Hiera.warn(error)
# Fill this up with very safe defaults - we cache non-existence
# for cache_ttl as well.
@@ -85,12 +107,12 @@
raise IOError, "Could not correctly fetch revision for #{path}, HTTP
status code #{res.status_code}"
end
# We shamelessly throw exceptions here, and catch them upper in the chain
- # specifically in stale? and Filecache.read
+ # specifically in Hiera::Mwcache.stale? and Hiera::Mwcache.read
body = JSON.parse(res.body)
pages = body["query"]["pages"]
# Quoting Yuvi: "MediaWiki API doesn't give a fuck about HTTP status
codes"
if pages.keys.include? "-1"
- raise IndexError, "The mediawiki page was non-existent"
+ raise Hiera::MediawikiPageNotFoundError, "Hiera:#{path}"
end
#yes, it's that convoluted.
key = pages.keys[0]
diff --git a/puppet/modules/wmflib/lib/puppet/parser/functions/conflicts.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/conflicts.rb
new file mode 100644
index 0000000..e29af6e
--- /dev/null
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/conflicts.rb
@@ -0,0 +1,18 @@
+# == Function: conflicts( string|resource $resource )
+#
+# Throw an error if a resource is declared.
+#
+# === Examples
+#
+# # Resource name
+# conflicts('::redis::legacy')
+#
+# # Resource
+# conflicts(Class['::redis-server'])
+#
+module Puppet::Parser::Functions
+ newfunction(:conflicts, :arity => 1) do |args|
+ Puppet::Parser::Functions.function(:defined)
+ fail(Puppet::ParseError, "Resource conflicts with #{args.first}.") if
function_defined(args)
+ end
+end
diff --git a/puppet/modules/wmflib/lib/puppet/parser/functions/conftool.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/conftool.rb
new file mode 100644
index 0000000..867ad1e
--- /dev/null
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/conftool.rb
@@ -0,0 +1,60 @@
+# == Function: conftool( string $tags, string $selector, string
$object_type='node')
+#
+# Fetch values from conftool. This should be used only for things that depend
on the dynamic
+# state from conftool but do not require the tight coordination.
+#
+# It will get (and json parse) values from conftool and return them, based on
the specified
+# selector.
+#
+# === Examples
+#
+# # get the current status of a node in confctl
+# $status = conftool({ name => 'cp1052.eqiad.wmnet', service => 'varnish-fe'})
# returns a list of associated services
+#
+require 'json'
+
+module Puppet::Parser::Functions
+ newfunction(:conftool, :type => :rvalue, :arity => -2) do |args|
+ case args.length
+ when 2
+ tags, type = *args
+ when 1
+ tags = args[0]
+ type = 'node'
+ end
+
+ # Raise an error if tags is empty or not an hash
+ if !tags.is_a?(Hash) || tags.empty?
+ raise Puppet::ParseError, "tags should be in hash format"
+ end
+
+ # get the data and return them parsed as json
+ begin
+ selector = tags.map { |k, v| "#{k}=#{v}" }.join ","
+ result = []
+ data = function_generate(
+ [
+ '/usr/bin/confctl',
+ '--object-type', type,
+ 'select', selector,
+ 'get'
+ ]
+ ).chomp
+
+ # No result returns the empty list
+ if data.empty?
+ return []
+ end
+
+ data.split("\n").each do |line|
+ entry = JSON.load(line)
+ tags = entry.delete 'tags'
+ obj_name = entry.keys.pop
+ result.push({'name' => obj_name, 'tags' => tags, 'value' =>
entry[obj_name]})
+ end
+ result
+ rescue
+ raise Puppet::ParseError, "Unable to read data from conftool"
+ end
+ end
+end
diff --git a/puppet/modules/wmflib/lib/puppet/parser/functions/cron_splay.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/cron_splay.rb
new file mode 100644
index 0000000..af76bf6
--- /dev/null
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/cron_splay.rb
@@ -0,0 +1,132 @@
+#
+# cron_splay.rb
+#
+
+require 'digest/md5'
+
+module Puppet::Parser::Functions
+ newfunction(:cron_splay, :type => :rvalue, :doc => <<-EOS
+Given an array of fqdn which a cron is applicable to, and a period arg which is
+one of 'hourly', 'daily', or 'weekly', this sorts the fqdn set with
+per-datacenter interleaving for DC-numbered hosts, splays them to fixed even
+intervals within the total period, and then outputs a set of crontab time
+fields for the fqdn currently being compiled-for.
+
+The idea here is to ensure each host in the set executes the cron once per time
+period, and also ensure the time between hosts is consistent (no edge cases
+much closer than the average) by splaying them as evenly as possible with
+rounding errors. For the case of hosts with NNNN numbers indicating the
+datacenter in the first digit, we also maximize the period between any two
+hosts in a given datacenter by interleaving sorted per-DC lists of hosts before
+splaying.
+
+The third and final argument is a static seed which modulates the splayed
+values in two different ways to minimize the effects of multiple cron_splay()
+with the same hostlist and period. It is used to select a determinstically
+random "offset" for the splayed time values (so that the first host doesn't
+always start at 00:00), and is also used to permute the order of the hosts
+within each DC uniquely.
+
+*Examples:*
+
+ $times = fqdn_splay($hosts, 'weekly', 'foo-static-seed')
+ cron { 'foo':
+ minute => $times['minute'],
+ hour => $times['hour'],
+ weekday => $times['weekday'],
+ }
+
+ EOS
+ ) do |arguments|
+
+ raise(Puppet::ParseError, "cron_splay(): Wrong number of arguments " +
+ "given (#{arguments.size} for 3)") if arguments.size != 3
+
+ hosts = arguments[0]
+ period = arguments[1]
+ seed = arguments[2]
+
+ unless hosts.is_a?(Array)
+ raise(Puppet::ParseError, 'cron_splay(): Argument 1 must be an array')
+ end
+
+ unless period.is_a?(String)
+ raise(Puppet::ParseError, 'cron_splay(): Argument 2 must be an string')
+ end
+
+ unless seed.is_a?(String)
+ raise(Puppet::ParseError, 'cron_splay(): Argument 3 must be an string')
+ end
+
+ case period
+ when 'hourly'
+ mins = 60
+ when 'daily'
+ mins = 1440
+ when 'weekly'
+ mins = 10080
+ else
+ raise(Puppet::ParseError, 'cron_splay(): invalid period')
+ end
+
+ # Avoid this edge case for now. At sufficiently large host counts and
+ # small period, randomization is probably better anyways.
+ if hosts.length > mins
+ raise(Puppet::ParseError, 'cron_splay(): too many hosts for period')
+ end
+
+ # split hosts into N lists based the first digit of /NNNN/, defaulting to
zero
+ sublists = [ [], [], [], [], [], [], [], [], [], [] ]
+ for h in hosts
+ match = /([1-9])[0-9]{3}/.match(h)
+ if match
+ sublists[match[1].to_i].push(h)
+ else
+ sublists[0].push(h)
+ end
+ end
+
+ # sort each sublist into a determinstic order based on seed
+ for s in sublists
+ s.sort_by! { |x| Digest::MD5.hexdigest(seed + x) }
+ end
+
+ # interleave sublists into "ordered"
+ longest = sublists.max_by(&:length)
+ sublists -= [longest]
+ ordered = longest.zip(*sublists).flatten.compact
+
+ # find the index of this host in ordered
+ this_idx = ordered.index(lookupvar('::fqdn'))
+ if this_idx.nil?
+ raise(Puppet::ParseError, 'cron_splay(): this host not in set')
+ end
+
+ # find the truncated-integer splayed value of this host
+ tval = this_idx * mins / ordered.length
+
+ # use the seed (again) to add a time offset to the splayed values,
+ # the time offset never being larger than the splayed interval
+ tval += Digest::MD5.hexdigest(seed).to_i(16) % (mins / ordered.length)
+
+ # generate the output
+ output = {}
+ output['minute'] = tval % 60
+
+ if period == 'hourly'
+ output['hour'] = '*'
+ else
+ output['hour'] = (tval / 60) % 24
+ end
+
+ if period == 'weekly'
+ output['weekday'] = tval / 1440
+ else
+ output['weekday'] = '*'
+ end
+
+ return output
+ end
+end
+
+# vim: set ts=2 sw=2 et :
diff --git
a/puppet/modules/wmflib/lib/puppet/parser/functions/ensure_mounted.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/ensure_mounted.rb
new file mode 100644
index 0000000..7a5e321
--- /dev/null
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/ensure_mounted.rb
@@ -0,0 +1,35 @@
+# == Function: ensure_mounted( string|bool $ensure )
+#
+# Takes a generic 'ensure' parameter value and convert it to an
+# appropriate value for use with a mount declaration.
+#
+# If $ensure is 'true' or 'present', the return value is 'mounted'.
+# Otherwise, the return value is the unmodified $ensure parameter.
+#
+# === Examples
+#
+# # Sample class which mounts or unmounts '/var/lib/nginx'
+# # based on the class's generic $ensure parameter:
+# class nginx ( $ensure = present ) {
+# package { 'nginx-full':
+# ensure => $ensure,
+# }
+# mount { '/var/lib/nginx':
+# ensure => ensure_mounted($ensure),
+# device => 'tmpfs',
+# fstype => 'tmpfs',
+# options => 'defaults,noatime,uid=0,gid=0,mode=755,size=1g',
+# }
+# }
+#
+module Puppet::Parser::Functions
+ newfunction(:ensure_mounted, :type => :rvalue, :arity => 1) do |args|
+
+ ensure_param = args.first
+ case ensure_param
+ when 'present', 'true', true then 'mounted'
+ when 'absent', 'false', false then ensure_param
+ else fail(ArgumentError, "ensure_directory(): invalid argument:
'#{ensure_param}'.")
+ end
+ end
+end
diff --git a/puppet/modules/wmflib/lib/puppet/parser/functions/get_clusters.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/get_clusters.rb
new file mode 100644
index 0000000..979a7b6
--- /dev/null
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/get_clusters.rb
@@ -0,0 +1,56 @@
+# == Function: get_clusters
+#
+# Given a selector hash, which can contain a cluster selector and/or
+# a site selector, this function will return a data
+# structure that contains all nodes clustered by cluster/site.
+#
+# === Parameters
+# [*selector*] An hash used to select the cluster and/or sites, if present;
+# allowed keys are 'site' and 'cluster', and should both be lists
+# of clusters and sites to select.
+#
+# === Examples
+#
+# # Return all nodes known to puppetDB
+# $nodes = get_clusters()
+#
+# # All eqiad nodes grouped by cluster/site
+# $eqiad_nodes = get_clusters({'site' => ['eqiad']})
+#
+# # All MediaWiki appserver and api nodes
+# $mw_servers = get_clusters({'cluster' => ['appserver', 'appserver_api'})
+#
+module Puppet::Parser::Functions
+ newfunction(:get_clusters, :type => :rvalue) do |args|
+ all = {}
+ # Ganglia config is the source of truth about clusters/site
+ cluster_config = function_hiera(['ganglia_clusters', {}])
+
+ # Arguments are an hash of selectors
+ selector ||= {}
+ selector = args[0] unless args.empty?
+ if selector.include? 'cluster'
+ clusters = selector['cluster']
+ else
+ clusters = cluster_config.keys
+ end
+
+ if selector.include? 'site'
+ sites = selector['site']
+ else
+ sites = false
+ end
+
+ function_query_resources([false, '@@Ganglia::Cluster', false]).each do
|node|
+ cluster = node['parameters']['cluster']
+ site = node['parameters']['site']
+ fqdn = node['title']
+ next unless clusters.include?cluster
+ next if sites && !sites.include?(site)
+ all[cluster] ||= {}
+ all[cluster][site] ||= []
+ all[cluster][site] << fqdn
+ end
+ all
+ end
+end
diff --git
a/puppet/modules/wmflib/lib/puppet/parser/functions/hash_deselect_re.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/hash_deselect_re.rb
new file mode 100644
index 0000000..c779625
--- /dev/null
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/hash_deselect_re.rb
@@ -0,0 +1,35 @@
+#
+# hash_deselect_re.rb
+#
+
+module Puppet::Parser::Functions
+ newfunction(:hash_deselect_re, :type => :rvalue, :doc => <<-EOS
+This function creates a new hash from the input hash, filtering out keys which
+match the provided regex.
+
+*Examples:*
+
+ $in = { 'abc' => 1, 'def' => 2, 'asdf' => 3 }
+ $out = hash_deselect_re('^a', $in);
+ # $out == { 'def' => 2 }
+ $out2 = hash_deselect_re('^(?!a)', $in);
+ # $out2 == { 'abc' => 1, 'asdf' => 3 }
+
+ EOS
+ ) do |arguments|
+
+ raise(Puppet::ParseError, "hash_deselect_re(): Wrong number of arguments "
+
+ "given (#{arguments.size} for 2)") if arguments.size != 2
+
+ pattern = Regexp.new(arguments[0])
+ in_hash = arguments[1]
+ unless in_hash.is_a?(Hash)
+ raise(Puppet::ParseError, 'hash_deselect_re(): Argument 2 must be a
hash')
+ end
+
+ #
https://bibwild.wordpress.com/2012/04/12/ruby-hash-select-1-8-7-and-1-9-3-simultaneously-compatible/
+ Hash[ in_hash.select { |k, _v| !pattern.match(k) } ]
+ end
+end
+
+# vim: set ts=2 sw=2 et :
diff --git
a/puppet/modules/wmflib/lib/puppet/parser/functions/hash_select_re.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/hash_select_re.rb
new file mode 100644
index 0000000..b5aa204
--- /dev/null
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/hash_select_re.rb
@@ -0,0 +1,35 @@
+#
+# hash_select_re.rb
+#
+
+module Puppet::Parser::Functions
+ newfunction(:hash_select_re, :type => :rvalue, :doc => <<-EOS
+This function creates a new hash from the input hash, filtering out keys which
+do not match the provided regex.
+
+*Examples:*
+
+ $in = { 'abc' => 1, 'def' => 2, 'asdf' => 3 }
+ $out = hash_select_re('^a', $in);
+ # $out == { 'abc' => 1, 'asdf' => 3 }
+ $out2 = hash_select_re('^(?!a)', $in);
+ # $out2 == { 'def' => 2 }
+
+ EOS
+ ) do |arguments|
+
+ raise(Puppet::ParseError, "hash_select_re(): Wrong number of arguments " +
+ "given (#{arguments.size} for 2)") if arguments.size != 2
+
+ pattern = Regexp.new(arguments[0])
+ in_hash = arguments[1]
+ unless in_hash.is_a?(Hash)
+ raise(Puppet::ParseError, 'hash_select_re(): Argument 2 must be a hash')
+ end
+
+ #
https://bibwild.wordpress.com/2012/04/12/ruby-hash-select-1-8-7-and-1-9-3-simultaneously-compatible/
+ Hash[ in_hash.select { |k, _v| pattern.match(k) } ]
+ end
+end
+
+# vim: set ts=2 sw=2 et :
diff --git a/puppet/modules/wmflib/lib/puppet/parser/functions/htpasswd.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/htpasswd.rb
new file mode 100644
index 0000000..e6d9335
--- /dev/null
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/htpasswd.rb
@@ -0,0 +1,110 @@
+# == Function htpasswd( string $password, string $salt)
+#
+# Generate a password entry for a htpasswd file using the modified md5 digest
+# method from apr.
+#
+
+require 'digest/md5'
+require 'stringio'
+
+# This class is a conversion to puppet of htauth's methods
+# See https://github.com/copiousfreetime/htauth/blob/master/LICENSE for
copying rights
+# Original code Copyright (c) 2008 Jeremy Hinegardner
+# Modifications Copyright (c) 2017 Giuseppe Lavagetto, Wikimedia Foundation,
Inc.
+class Apr1Md5
+
+ DIGEST_LENGTH = 16
+
+ def initialize(salt)
+ @salt = salt
+ end
+
+ def prefix
+ "$apr1$"
+ end
+
+ # from
https://github.com/copiousfreetime/htauth/blob/master/lib/htauth/algorithm.rb
+ # this is not the Base64 encoding, this is the to64() method from apr
+ SALT_CHARS = (%w( . / ) + ("0".."9").to_a + ('A'..'Z').to_a +
('a'..'z').to_a).freeze
+ def to_64(number, rounds)
+ r = StringIO.new
+ rounds.times do
+ r.print(SALT_CHARS[number % 64])
+ number >>= 6
+ end
+ r.string
+ end
+
+
+ # this algorithm pulled straight from apr_md5_encode() and converted to ruby
syntax
+ def encode(password)
+ primary = ::Digest::MD5.new
+ primary << password
+ primary << prefix
+ primary << @salt
+
+ md5_t = ::Digest::MD5.digest("#{password}#{@salt}#{password}")
+
+ l = password.length
+ while l > 0
+ slice_size = (l > DIGEST_LENGTH) ? DIGEST_LENGTH : l
+ primary << md5_t[0, slice_size]
+ l -= DIGEST_LENGTH
+ end
+
+ # weirdness
+ l = password.length
+ while l != 0
+ case (l & 1)
+ when 1
+ primary << 0.chr
+ when 0
+ primary << password[0, 1]
+ end
+ l >>= 1
+ end
+
+ pd = primary.digest
+
+ encoded_password = "#{prefix}#{@salt}$"
+
+ # apr_md5_encode has this comment about a 60Mhz Pentium above this loop.
+ 1000.times do |x|
+ ctx = ::Digest::MD5.new
+ ctx << (((x & 1) == 1) ? password : pd[0, DIGEST_LENGTH])
+ (ctx << @salt) unless (x % 3) == 0
+ (ctx << password) unless (x % 7) == 0
+ ctx << (((x & 1) == 0) ? password : pd[0, DIGEST_LENGTH])
+ pd = ctx.digest
+ end
+
+
+ pd = pd.bytes.to_a
+
+ l = (pd[0] << 16) | (pd[6] << 8) | pd[12]
+ encoded_password << to_64(l, 4)
+
+ l = (pd[1] << 16) | (pd[7] << 8) | pd[13]
+ encoded_password << to_64(l, 4)
+
+ l = (pd[2] << 16) | (pd[8] << 8) | pd[14]
+ encoded_password << to_64(l, 4)
+
+ l = (pd[3] << 16) | (pd[9] << 8) | pd[15]
+ encoded_password << to_64(l, 4)
+
+ l = (pd[4] << 16) | (pd[10] << 8) | pd[ 5]
+ encoded_password << to_64(l, 4)
+ encoded_password << to_64(pd[11], 2)
+
+ encoded_password
+ end
+end
+
+
+module Puppet::Parser::Functions
+ newfunction(:htpasswd, :type => :rvalue, :arity => 2) do |args|
+ generator = Apr1Md5.new args[1]
+ generator.encode args[0]
+ end
+end
diff --git a/puppet/modules/wmflib/lib/puppet/parser/functions/ini.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/ini.rb
new file mode 100644
index 0000000..19a0d28
--- /dev/null
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/ini.rb
@@ -0,0 +1,42 @@
+# == Function: ini( hash $ini_settings [, hash $... ] )
+#
+# Serialize a hash into the .ini-style format expected by Python's
+# ConfigParser. Takes one or more hashes as arguments. If the argument
+# list contains more than one hash, they are merged together. In case of
+# duplicate keys, hashes to the right win.
+#
+# === Example
+#
+# ini({'server' => {'port' => 80}})
+#
+# will produce:
+#
+# [server]
+# port = 80
+#
+def ini_flatten(map, prefix = nil)
+ map.reduce({}) do |flat, (k, v)|
+ k = [prefix, k].compact.join('.')
+ flat.merge! v.is_a?(Hash) ? ini_flatten(v, k) : Hash[k, v]
+ end
+end
+
+def ini_cast(v)
+ v.include?('.') ? Float(v) : Integer(v) rescue v
+end
+
+module Puppet::Parser::Functions
+ newfunction(:ini, :type => :rvalue, :arity => -2) do |args|
+ if args.map(&:class).uniq != [Hash]
+ fail(ArgumentError, 'ini(): hash arguments required')
+ end
+ args.reduce(&:merge).map do |section,items|
+ ini_flatten(items).map do |k, vs|
+ case vs
+ when Array then vs.map { |v| "#{k}[#{v}] = #{ini_cast(v)}" }
+ else "#{k} = #{ini_cast(vs)}"
+ end
+ end.flatten.sort.push('').unshift("[#{section}]").join("\n")
+ end.flatten.sort.push('').join("\n")
+ end
+end
diff --git a/puppet/modules/wmflib/lib/puppet/parser/functions/ipresolve.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/ipresolve.rb
new file mode 100644
index 0000000..02afde3
--- /dev/null
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/ipresolve.rb
@@ -0,0 +1,147 @@
+# == Function: ipresolve( string $name_to_resolve, bool $ipv6 = false)
+#
+# Copyright (c) 2015 Wikimedia Foundation Inc.
+#
+# Performs a name resolution (for A AND AAAA records only) and returns
+# an hash of arrays.
+#
+# Takes one or more names to resolve, and returns an array of all the
+# A or AAAA records found. The resolution is actually only done when
+# the ttl has expired. A particular nameserver can also be specified
+# so only that is used, rather than the system default.
+#
+require 'resolv'
+
+class DNSCacheEntry
+ # Data structure for storing a DNS cached result.
+ def initialize(entry, ttl)
+ @value = entry
+ @ttl = Time.now.to_i + ttl
+ end
+
+ def is_valid?(time)
+ return @ttl > time
+ end
+
+ def value
+ return @value.to_s
+ end
+end
+
+class BasicTTLCache
+ def initialize
+ @cache = {}
+ end
+
+ def write(key, value, ttl)
+ @cache[key] = DNSCacheEntry.new(value, ttl)
+ end
+
+ def delete(key)
+ @cache.delete(key) if @cache.key?(key)
+ end
+
+ def is_valid?(key)
+ # If the key exists, and its ttl has not expired, return true.
+ # Return false (and maybe clean up the stale entry) otherwise.
+ return false unless @cache.key?(key)
+ t = Time.now.to_i
+ return true if @cache[key].is_valid?t
+ return false
+ end
+
+ def read(key)
+ if is_valid?key
+ return @cache[key].value
+ end
+ return nil
+ end
+
+ def read_stale(key)
+ if @cache.key?(key)
+ return @cache[key].value
+ end
+ return nil
+ end
+end
+
+class DNSCached
+ attr_accessor :dns
+ def initialize(cache = nil, default_ttl = 300)
+ @cache = cache || BasicTTLCache.new
+ @default_ttl = default_ttl
+ end
+
+ def get_resource(name, type, nameserver)
+ if nameserver.nil?
+ dns = Resolv::DNS.open()
+ else
+ dns = Resolv::DNS.open(:nameserver => [nameserver])
+ end
+ cache_key = "#{name}_#{type}_#{nameserver}"
+ res = @cache.read(cache_key)
+ if (res.nil?)
+ begin
+ res = dns.getresource(name, type)
+ # Ruby < 1.9 returns nil as the ttl...
+ if res.ttl
+ ttl = res.ttl
+ else
+ ttl = @default_ttl
+ end
+ if type == Resolv::DNS::Resource::IN::PTR
+ retval = res.name
+ else
+ retval = res.address
+ end
+ @cache.write(cache_key, retval, ttl)
+ retval.to_s
+ rescue
+ # If resolution fails and we do have a cached stale value, use it
+ res = @cache.read_stale(cache_key)
+ if res.nil?
+ fail("DNS lookup failed for #{name} #{type}")
+ end
+ res.to_s
+ end
+ else
+ res.to_s
+ end
+ end
+end
+
+
+module Puppet::Parser::Functions
+ dns = DNSCached.new
+ newfunction(:ipresolve, :type => :rvalue, :arity => -1) do |args|
+ name = args[0]
+ if args[1].nil?
+ type = 4
+ elsif args[1].to_s.downcase == 'ptr'
+ type = 'ptr'
+ else
+ type = args[1].to_i
+ end
+ nameserver = args[2] # Ruby returns nil if there's nothing there
+ if type == 4
+ source = Resolv::DNS::Resource::IN::A
+ elsif type == 6
+ source = Resolv::DNS::Resource::IN::AAAA
+ elsif type == 'ptr'
+ source = Resolv::DNS::Resource::IN::PTR
+ # Transform the provided IP address in a PTR record
+ case name
+ when Resolv::IPv4::Regex
+ ptr = Resolv::IPv4.create(name).to_name
+ when Resolv::IPv6::Regex
+ ptr = Resolv::IPv6.create(name).to_name
+ else
+ fail("Cannot interpret #{name} as an address")
+ end
+ name = ptr
+ else
+ raise ArgumentError, 'Type must be 4, 6 or ptr'
+ end
+ return dns.get_resource(name, source, nameserver).to_s
+ end
+end
diff --git a/puppet/modules/wmflib/lib/puppet/parser/functions/os_version.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/os_version.rb
index c105f0a..badeb79 100644
--- a/puppet/modules/wmflib/lib/puppet/parser/functions/os_version.rb
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/os_version.rb
@@ -101,4 +101,4 @@
end
end
end
-end
\ No newline at end of file
+end
diff --git a/puppet/modules/wmflib/lib/puppet/parser/functions/puppet_ssldir.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/puppet_ssldir.rb
new file mode 100644
index 0000000..d0e8446
--- /dev/null
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/puppet_ssldir.rb
@@ -0,0 +1,60 @@
+# == Function puppet_ssldir( string $override = nil )
+#
+# Returns puppet's configured ssldir, using some heuristics.
+# This function is needed because we have a separate configurations
+# for the self-hosted puppetmasters ssl directory compared to the
+# standard setup. If we're ever able to simplify or remove such
+# differences, this function might become way simpler, or even
+# disappear.
+#
+# It's possible to override the heuristics and provide an override
+# parameter, which if set to 'master' will assume you are on a
+# self-hosted puppetmaster.
+#
+# == Examples
+#
+# # returns the default result based on the catalog
+# $ssldir = puppet_ssldir()
+# # Forces ssldir to be the one of a self-hosted puppetmaster
+# $ssldir = puppet_ssldir('master')
+#
+module Puppet::Parser::Functions
+ newfunction(:puppet_ssldir, :type => :rvalue) do |overrides|
+ # Check arguments
+ override = overrides[0]
+
+ fail("Only 'master', 'client' and undef " \
+ "are valid arguments of puppet_ssldir") unless
+ ['master', 'client', nil].include?override
+
+ default = '/var/lib/puppet/ssl'
+ self_master = '/var/lib/puppet/server/ssl'
+ self_client = '/var/lib/puppet/client/ssl'
+
+ # Production uses the standard layout
+ return default if lookupvar('::realm') != 'labs'
+
+ # Self-hosted puppetmasters explicit setup
+ case override
+ when 'master'
+ return self_master
+ when 'client'
+ return self_client
+ end
+
+ # Since all self-hosted puppetmasters are in .eqiad.wmflabs, while
+ # the labs masters don't
+ return default if lookupvar('::settings::certname') =~ /\.wikimedia\.org$/
+ # Non-self-hosted puppetmasters all use the default ssldir
+ puppetmaster = lookupvar('puppetmaster')
+ puppetmaster ||= function_hiera(['role::puppet::self::master', ''])
+ if puppetmaster == ''
+ # Means we aren't using any of role::puppet::self!1!
+ default
+ elsif [lookupvar('hostname'), 'localhost', '', nil].include?puppetmaster
+ self_master
+ else
+ self_client
+ end
+ end
+end
diff --git
a/puppet/modules/wmflib/lib/puppet/parser/functions/require_package.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/require_package.rb
index cf32a7c..996462f 100644
--- a/puppet/modules/wmflib/lib/puppet/parser/functions/require_package.rb
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/require_package.rb
@@ -19,14 +19,38 @@
#
module Puppet::Parser::Functions
newfunction(:require_package, :arity => -2) do |args|
- args.each do |package_name|
+ Puppet::Parser::Functions.function :create_resources
+ args.flatten.each do |package_name|
+ # Puppet class names are alphanumeric + underscore
+ # 'g++' package would yield: 'packages::g__'
class_name = 'packages::' + package_name.tr('-+', '_')
- unless compiler.topscope.find_hostclass(class_name)
+
+ # Create host class
+
+ host = compiler.topscope.find_hostclass(class_name)
+ unless host
host = Puppet::Resource::Type.new(:hostclass, class_name)
known_resource_types.add_hostclass(host)
- send Puppet::Parser::Functions.function(:create_resources),
- ['package', { package_name => { :ensure => :present } }]
end
+
+ # Create class scope
+
+ cls = Puppet::Parser::Resource.new(
+ 'class', class_name, :scope => compiler.topscope)
+ catalog.add_resource(cls) rescue nil
+ host.evaluate_code(cls) rescue nil
+
+ # Create package resource
+
+ begin
+ host_scope = compiler.topscope.class_scope(host)
+ host_scope.function_create_resources(
+ ['package', { package_name => { :ensure => :present } }])
+ rescue Puppet::Resource::Catalog::DuplicateResourceError
+ end
+
+ # Declare dependency
+
send Puppet::Parser::Functions.function(:require), [class_name]
end
end
diff --git a/puppet/modules/wmflib/lib/puppet/parser/functions/requires_os.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/requires_os.rb
new file mode 100644
index 0000000..026affe
--- /dev/null
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/requires_os.rb
@@ -0,0 +1,22 @@
+# == Function: requires_os( string $version_predicate )
+#
+# Validate that the host operating system version satisfies a version
+# check. Abort catalog compilation if not.
+#
+# See the documentation for os_version() for supported predicate syntax.
+#
+# === Examples
+#
+# # Fail unless version is exactly Debian Jessie
+# requires_os('debian jessie')
+#
+# # Fail unless Ubuntu Trusty or newer or Debian Jessie or newer
+# requires_os('ubuntu >= trusty || debian >= Jessie')
+#
+module Puppet::Parser::Functions
+ newfunction(:requires_os, :arity => 1) do |args|
+ Puppet::Parser::Functions.function(:os_version)
+ fail(ArgumentError, 'requires_os(): string argument required') unless
args.first.is_a?(String)
+ fail(Puppet::ParseError, "OS #{args.first} required.") unless
function_os_version(args)
+ end
+end
diff --git a/puppet/modules/wmflib/lib/puppet/parser/functions/role.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/role.rb
new file mode 100644
index 0000000..e03cd8a
--- /dev/null
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/role.rb
@@ -0,0 +1,85 @@
+# == Function: role ( string $role_name [, string $... ] )
+#
+# Declare the _roles variable (or add keys to it), and if the role class
+# role::$role_name is in the scope, include it. This is roughly a
+# shortcut for
+#
+# $::_roles[foo] = true
+# include role::foo
+#
+# and has the notable advantage over that the syntax is shorter. Also,
+# this function will refuse to run anywhere but at the node scope,
+# thus making any additional role added to a node explicit.
+#
+# If you have more than one role to declare, you MUST do that in one
+# single role stanza, or you would encounter unexpected behaviour. If
+# you do, an exception will be raised.
+#
+# This function is very useful with our "role" hiera backend if you
+# have global configs that are role-based
+#
+# === Example
+#
+# node /^www\d+/ {
+# role mediawiki::appserver # this will load the
role::mediawiki::appserver class
+# include ::standard #this class will use hiera lookups defined for the
role.
+# }
+#
+# node monitoring.local {
+# role icinga, ganglia::collector #GOOD
+# }
+#
+# node monitoring2.local {
+# role icinga
+# role ganglia::collector #BAD, issues a warning
+# }
+
+module Puppet::Parser::Functions
+ newfunction(:role, :arity => -1) do |args|
+ # This will add to the catalog, and to the node specifically:
+ # - A global 'role' hash with the 'role::#{arg}' key set to true;
+ # if the variable is present, append to it
+ # - Include class role::#{arg} if present
+
+ # Prevent use outside of node definitions
+ if not self.is_nodescope?
+ raise Puppet::ParseError,
+ "role can only be used in node scope, while you are in scope
#{self}"
+ end
+
+ # Now check if the variable is already set and issue a warning
+ container = '_roles'
+ rolevar = compiler.topscope.lookupvar(container)
+ if rolevar
+ raise Puppet::ParseError,
+ "Using 'role' multiple times might yield unexpected results, use
'role role1, role2' instead"
+ else
+ compiler.topscope.setvar(container, {})
+ rolevar = compiler.topscope.lookupvar(container)
+ end
+
+ # sanitize args
+ args = args.map do |x|
+ if x.start_with? '::'
+ x.gsub(/^::/, '')
+ else
+ x
+ end
+ end
+
+ # Set the variables
+ args.each do |arg|
+ rolevar[arg] = true
+ end
+
+ args.each do |arg|
+ rolename = 'role::' + arg
+ role_class = compiler.topscope.find_hostclass(rolename)
+ if role_class
+ send Puppet::Parser::Functions.function(:include), [rolename]
+ else
+ raise Puppet::ParseError, "Role class #{rolename} not found"
+ end
+ end
+ end
+end
diff --git a/puppet/modules/wmflib/lib/puppet/parser/functions/secret.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/secret.rb
new file mode 100644
index 0000000..fc66a02
--- /dev/null
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/secret.rb
@@ -0,0 +1,29 @@
+require 'pathname'
+
+module Puppet::Parser::Functions
+ newfunction(:secret, :type => :rvalue) do |args|
+ mod_name = 'secret'
+ secs_subdir = '/secrets/'
+
+ if args.length != 1 || !args.first.is_a?(String)
+ fail(ArgumentError, 'secret(): exactly one string arg')
+ end
+ in_path = args.first
+
+ if mod = Puppet::Module.find(mod_name)
+ mod_path = mod.path()
+ else
+ fail("secret(): Module #{mod_name} not found")
+ end
+
+ sec_path = mod_path + secs_subdir + in_path
+ final_path = Pathname.new(sec_path).cleanpath()
+
+ # Bail early if it's not a regular, readable file
+ if !final_path.file? || !final_path.readable?
+ fail(ArgumentError, "secret(): invalid secret #{in_path}")
+ end
+
+ return final_path.read()
+ end
+end
diff --git
a/puppet/modules/wmflib/lib/puppet/parser/functions/ssl_ciphersuite.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/ssl_ciphersuite.rb
index 26d0bf7..6143766 100644
--- a/puppet/modules/wmflib/lib/puppet/parser/functions/ssl_ciphersuite.rb
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/ssl_ciphersuite.rb
@@ -1,4 +1,4 @@
-# == Function: ssl_ciphersuite( string $servercode, string $encryption_type,
int $hsts_days )
+# == Function: ssl_ciphersuite( string $server, string $encryption_type,
boolean $hsts )
#
# Outputs the ssl configuration directives for use with either Nginx
# or Apache using our selection of ciphers and SSL options.
@@ -7,14 +7,30 @@
#
# Takes three arguments:
#
-# - The servercode, or which browser-version combination to
-# support. At the moment only 'apache-2.2', 'apache-2.4' and 'nginx'
-# are supported.
-# - The compatibility mode,indicating the degree of compatibility we
-# want to retain with older browsers (basically, IE6, IE7 and
-# Android prior to 3.0)
-# - An optional argument, that if non-nil will set HSTS to max-age of
-# N days
+# - The server to configure for: 'apache' or 'nginx'
+# - The compatibility mode, trades security vs compatibility.
+# Note that due to POODLE, SSLv3 is universally disabled and none of these
+# options are compatible with SSLv3-only clients such as IE6/XP.
+# Current options are:
+# - strong: Only TLSv1.2 with FS+AEAD ciphers. In practice this is a
+# very short list, and requires a very modern client. No
+# tradeoff is made for compatibility. Known to work with:
+# FF/Chrome, IE11, Safari 9, Java8, Android 4.4+, OpenSSL 1.0.x
+# - mid: Supports TLSv1.0 and higher, and adds several forward-secret
+# options which are not AEAD. This is compatible with many
more
+# clients than "strong". Should only be incompatible with
+# unpatched IE8/XP, ancient/un-updated Java6, and some small
+# corner cases like Nokia feature phones.
+# - compat: Supports most legacy clients, FS optional but preferred.
+# - HSTS boolean - if true, will emit our standard HSTS header for canonical
+# public domains (which is currently 1 year with preload and includeSub).
+# Default false.
+#
+# In our WMF configurations, Apache only supports DHE ciphersuites securely on
+# Debian Jessie, which is necessary for "mid" to have the compatibility level
+# stated above. When this function is used with Apache an older host (e.g.
+# Ubuntu Trusty or Precise), the "mid" and "strong" options will be downgraded
+# to "compat" with a warning.
#
# Whenever called, this function will output a list of strings that
# can be safely used in your configuration file as the ssl
@@ -22,7 +38,7 @@
#
# == Examples
#
-# ssl_ciphersuite('apache-2.4', 'compat')
+# ssl_ciphersuite('apache', 'compat', true)
# ssl_ciphersuite('nginx', 'strong')
#
# == License
@@ -46,92 +62,160 @@
require 'puppet/util/package'
module Puppet::Parser::Functions
- ciphersuites = {
- 'compat' =>
'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-ECDSA-RC4-SHA:AES128:AES256:RC4-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK:!DH',
- 'strong' =>
'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK:!DH'
+ # Basic list chunks, used to construct bigger lists
+ # General preference ordering for fullest combined list:
+ # 0) Enc: 3DES < ALL (SWEET32)
+ # 1) Kx: (EC)DHE > RSA (Forward Secrecy)
+ # 2) Mac: AEAD > ALL (AES-GCM/CHAPOLY > Others)
+ # 3) Enc: CHAPOLY > AESGCM (Old client perf, sec)
+ # 4) Kx: ECDHE > DHE (Perf, mostly)
+ # 5) Enc: AES256 > AES128 (sec)
+ # 6) Auth: ECDSA > RSA (Perf, mostly)
+ #
+ # After all of that, the fullest list of reasonably-acceptable mid/compat
+ # ciphers has been filtered further to reduce pointless clutter:
+ # *) The 'mid' list has been filtered of AES256 options on the grounds that
+ # any such client can always use AES128 instead, and it's senseless to try to
+ # set a 'more bits' security policy if not using a strong cipher in general,
+ # and clients too old for strong ciphers are more likely to be impacted by
+ # AES256 performance differentials. SHA-2 HMAC variants were filtered
+ # similarly, as all clients that would negotiate x-SHA256 also negotiate
x-SHA
+ # and there's no effective security difference between the two.
+ # *) The 'compat' list has been reduced to just the two weakest and
+ # most-popular reasonable options there. The others were mostly
statistically
+ # insignificant, and things are so bad at this level it's not worth worrying
+ # about slight cipher strength gains.
+ basic = {
+ # Forward-Secret + AEAD
+ 'strong' => [
+ '-ALL',
+ 'ECDHE-ECDSA-CHACHA20-POLY1305', # openssl-1.1.0, 1.0.2+cloudflare
+ 'ECDHE-RSA-CHACHA20-POLY1305', # openssl-1.1.0, 1.0.2+cloudflare
+ 'ECDHE-ECDSA-AES256-GCM-SHA384',
+ 'ECDHE-RSA-AES256-GCM-SHA384',
+ 'ECDHE-ECDSA-AES128-GCM-SHA256',
+ 'ECDHE-RSA-AES128-GCM-SHA256',
+ 'DHE-RSA-AES128-GCM-SHA256',
+ ],
+ # Forward-Secret, but not AEAD
+ 'mid' => [
+ 'ECDHE-ECDSA-AES128-SHA', # Various outdated IE, Safari<9, Android<4.4
+ 'ECDHE-RSA-AES128-SHA',
+ 'DHE-RSA-AES128-SHA', # Android 2.x, openssl-0.9.8, etc
+ ],
+ # not-forward-secret compat for ancient stuff
+ 'compat' => [
+ 'AES128-SHA', # Mostly evil proxies, also ancient devices
+ 'DES-CBC3-SHA', # Mostly IE7-8 on XP, also ancient devices
+ ],
}
+
+ # Final lists exposed to callers
+ ciphersuites = {
+ 'strong' => basic['strong'],
+ 'mid' => basic['strong'] + basic['mid'],
+ 'compat' => basic['strong'] + basic['mid'] + basic['compat'],
+ }
+
+ # Our standard HSTS for all public canonical domains
+ hsts_val = "max-age=31536000; includeSubDomains; preload"
+
newfunction(
:ssl_ciphersuite,
:type => :rvalue,
:doc => <<-END
Outputs the ssl configuration part of the webserver config.
Function parameters are:
- servercode - either nginx, apache-2.2 or apache-2.4
- encryption_type - either strong for PFS only, or compat for maximum
compatibility
- hsts_days - how many days should the STS header live. If not expressed, HSTS
will
- be disabled
+ server - either nginx or apache
+ encryption_type - strong, mid, or compat
+ hsts - optional boolean, true emits our standard public HSTS
Examples:
- ssl_ciphersuite('apache-2.4', 'compat') # Compatible config for apache 2.4
- ssl_ciphersuite('nginx', 'strong', '365') # PFS-only, use HSTS for 365 days
+ ssl_ciphersuite('apache', 'compat', true) # Compatible config for apache
+ ssl_ciphersuite('apache', 'mid', true) # FS-only for apache
+ ssl_ciphersuite('nginx', 'strong', true) # FS-only, AEAD-only, TLSv1.2-only
END
) do |args|
+ Puppet::Parser::Functions.function(:os_version)
+ Puppet::Parser::Functions.function(:notice)
if args.length < 2 || args.length > 3
fail(ArgumentError, 'ssl_ciphersuite() requires at least 2 arguments')
end
- servercode = args.shift
- case servercode
- when 'apache-2.4' then
- server = 'apache'
- server_version = 24
- when 'apache-2.2' then
- server = 'apache'
- server_version = 22
- when 'nginx' then
- server = 'nginx'
- server_version = nil
- else
- fail(ArgumentError, "ssl_ciphersuite(): unknown server string
'#{servercode}'")
+ server = args.shift
+ if server != 'apache' && server != 'nginx'
+ fail(ArgumentError, "ssl_ciphersuite(): unknown server string
'#{server}'")
end
ciphersuite = args.shift
- unless ciphersuites.has_key?(ciphersuite)
+ unless ciphersuites.key?(ciphersuite)
fail(ArgumentError, "ssl_ciphersuite(): unknown ciphersuite
'#{ciphersuite}'")
end
- cipherlist = ciphersuites[ciphersuite]
-
- if ciphersuite == 'strong' && server == 'apache' && server_version < 24
- fail(ArgumentError, 'ssl_ciphersuite(): apache 2.2 cannot work in strong
PFS mode')
- end
+ do_hsts = false
if args.length == 1
- hsts_days = args.shift.to_i
+ do_hsts = args.shift
+ end
+
+ # OS / Server -dependant feature flags:
+ nginx_always_ok = true
+ dhe_ok = true
+ if !function_os_version(['debian >= jessie'])
+ nginx_always_ok = false
+ if server == 'apache'
+ dhe_ok = false
+ end
+ end
+
+ if !dhe_ok && ciphersuite != 'compat'
+ function_notice([
+ 'ssl_ciphersuite(): OS needs upgrade to Jessie! Downgrading SSL
ciphersuite to "compat"'
+ ])
+ ciphersuite = 'compat'
+ end
+
+ if dhe_ok
+ cipherlist = ciphersuites[ciphersuite].join(":")
else
- hsts_days = nil
+ cipherlist = ciphersuites[ciphersuite].reject{|x| x =~
/^(DHE|EDH)-/}.join(":")
end
output = []
if server == 'apache'
- case ciphersuite
- when 'strong' then
- output.push('SSLProtocol all -SSLv2 -SSLv3 -TLSv1')
- when 'compat' then
+ if ciphersuite == 'strong'
+ output.push('SSLProtocol all -SSLv2 -SSLv3 -TLSv1 -TLSv1.1')
+ else
output.push('SSLProtocol all -SSLv2 -SSLv3')
end
output.push("SSLCipherSuite #{cipherlist}")
output.push('SSLHonorCipherOrder On')
- unless hsts_days.nil?
- hsts_seconds = hsts_days * 86400
- output.push("Header set Strict-Transport-Security
\"max-age=#{hsts_seconds}\"")
+ if dhe_ok
+ output.push('SSLOpenSSLConfCmd DHParameters "/etc/ssl/dhparam.pem"')
end
- else
- # nginx
- case ciphersuite
- when 'strong' then
- output.push('ssl_protocols TLSv1.1 TLSv1.2;')
- when 'compat' then
+ if do_hsts
+ output.push("Header always set Strict-Transport-Security
\"#{hsts_val}\"")
+ end
+ else # nginx
+ if ciphersuite == 'strong'
+ output.push('ssl_protocols TLSv1.2;')
+ else
output.push('ssl_protocols TLSv1 TLSv1.1 TLSv1.2;')
end
output.push("ssl_ciphers #{cipherlist};")
output.push('ssl_prefer_server_ciphers on;')
- unless hsts_days.nil?
- hsts_seconds = hsts_days * 86400
- output.push("add_header Strict-Transport-Security
\"max-age=#{hsts_seconds}\";")
+ if dhe_ok
+ output.push('ssl_dhparam /etc/ssl/dhparam.pem;')
+ end
+ if do_hsts
+ if nginx_always_ok
+ output.push("add_header Strict-Transport-Security \"#{hsts_val}\"
always;")
+ else
+ output.push("add_header Strict-Transport-Security
\"#{hsts_val}\";")
+ end
end
end
return output
diff --git
a/puppet/modules/wmflib/lib/puppet/parser/functions/validate_array_re.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/validate_array_re.rb
new file mode 100644
index 0000000..22d1e19
--- /dev/null
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/validate_array_re.rb
@@ -0,0 +1,23 @@
+# == Function: validate_array_re( array $items, string $re )
+#
+# Throw an error if any member of $items does not match the regular
+# expression $re.
+#
+# === Examples
+#
+# # OK -- each array item is a four-digit number.
+# validate_array_re([8123, 8124, 8125], '^\d{4}$')
+#
+# # Fail -- last array item is not a four-digit number.
+# validate_array_re([8123, 8124, 812], '^\d{4}$')
+#
+module Puppet::Parser::Functions
+ newfunction(:validate_array_re, :arity => 2) do |args|
+ items, re = args
+ re = Regexp.new(re)
+ invalid = args.first.find { |item| item.to_s !~ re }
+ unless invalid.nil?
+ fail(Puppet::ParseError, "Array element \"#{invalid}\" does not match
regular expression \"#{re.source}\".")
+ end
+ end
+end
diff --git a/puppet/modules/wmflib/spec/fixtures/hiera.proxy.yaml
b/puppet/modules/wmflib/spec/fixtures/hiera.proxy.yaml
new file mode 100644
index 0000000..1f20afb
--- /dev/null
+++ b/puppet/modules/wmflib/spec/fixtures/hiera.proxy.yaml
@@ -0,0 +1,16 @@
+:backends:
+ - proxy
+:proxy:
+ :datadir: 'spec/fixtures/hieradata'
+ :plugins:
+ - role
+ - nuyaml
+ :default_plugin: nuyaml
+:role:
+ :datadir: 'spec/fixtures/hieradata'
+:nuyaml:
+ :datadir: 'spec/fixtures/hieradata'
+:hierarchy:
+ - "hosts/%{::hostname}"
+ - "role@@common"
+ - common
diff --git a/puppet/modules/wmflib/spec/fixtures/hiera.yaml
b/puppet/modules/wmflib/spec/fixtures/hiera.yaml
new file mode 100644
index 0000000..d2dab0b
--- /dev/null
+++ b/puppet/modules/wmflib/spec/fixtures/hiera.yaml
@@ -0,0 +1,6 @@
+:backends:
+ - role
+:role:
+ :datadir: 'spec/fixtures/hieradata'
+:role_hierarchy:
+ - common
diff --git a/puppet/modules/wmflib/spec/fixtures/hieradata/common.yaml
b/puppet/modules/wmflib/spec/fixtures/hieradata/common.yaml
new file mode 100644
index 0000000..feb50a1
--- /dev/null
+++ b/puppet/modules/wmflib/spec/fixtures/hieradata/common.yaml
@@ -0,0 +1,2 @@
+cluster: misc
+mysql::innodb_threads: 15
diff --git a/puppet/modules/wmflib/spec/fixtures/hieradata/hosts/foo.yaml
b/puppet/modules/wmflib/spec/fixtures/hieradata/hosts/foo.yaml
new file mode 100644
index 0000000..477aee1
--- /dev/null
+++ b/puppet/modules/wmflib/spec/fixtures/hieradata/hosts/foo.yaml
@@ -0,0 +1,2 @@
+admin::groups:
+ - go-spurs
diff --git
a/puppet/modules/wmflib/spec/fixtures/hieradata/role/common/test.yaml
b/puppet/modules/wmflib/spec/fixtures/hieradata/role/common/test.yaml
new file mode 100644
index 0000000..878c890
--- /dev/null
+++ b/puppet/modules/wmflib/spec/fixtures/hieradata/role/common/test.yaml
@@ -0,0 +1,5 @@
+admin::groups:
+ - FooBar
+an_hash:
+ test: true
+mysql::innodb_threads: 50
diff --git
a/puppet/modules/wmflib/spec/fixtures/hieradata/role/common/test2.yaml
b/puppet/modules/wmflib/spec/fixtures/hieradata/role/common/test2.yaml
new file mode 100644
index 0000000..87bac76
--- /dev/null
+++ b/puppet/modules/wmflib/spec/fixtures/hieradata/role/common/test2.yaml
@@ -0,0 +1,6 @@
+admin::groups:
+ - FooBar1
+an_hash:
+ test2: true
+ test3:
+ another: "level"
diff --git a/puppet/modules/wmflib/spec/functions/conftool_spec.rb
b/puppet/modules/wmflib/spec/functions/conftool_spec.rb
new file mode 100644
index 0000000..888c51b
--- /dev/null
+++ b/puppet/modules/wmflib/spec/functions/conftool_spec.rb
@@ -0,0 +1,54 @@
+require 'spec_helper'
+require 'mocha/test_unit'
+require 'json'
+
+describe 'conftool' do
+
+ def gen_conftool_call(selector)
+ ['/usr/bin/conftool', '--object-type', 'node', 'select', selector, 'get']
+ end
+
+ generate = {}
+
+ before(:each) {
+ Puppet::Parser::Functions.newfunction(:generate) {
+ |args| generate.call(args)
+ }
+ generate.stubs(:call).returns('')
+ }
+
+ it "should fail if 3 args are given" do
+ should run.with_params('a', 'b', 'c').and_raise_error(Puppet::ParseError)
+ end
+
+ it "should return an empty list if no result is returned" do
+ req = {'name' => 'foo'}
+ should run.with_params(req).and_return([])
+ end
+
+ it "should return correctly the results" do
+ resultset = [
+ {"cp1052.eqiad.wmnet" => {"pooled" => "yes", "weight" => 100}, "tags"
=> "dc=eqiad,cluster=cache_text,service=varnish-be"},
+ {"cp1052.eqiad.wmnet" => {"pooled" => "yes", "weight" => 1}, "tags" =>
"dc=eqiad,cluster=cache_text,service=varnish-fe"},
+ ]
+ retval = resultset.map{ |x| JSON.dump(x) }.join "\n"
+ conftool_out = [
+ {'name' => 'cp1052.eqiad.wmnet', 'tags' => resultset[0]['tags'], 'value'
=> resultset[0]['cp1052.eqiad.wmnet']},
+ {'name' => 'cp1052.eqiad.wmnet', 'tags' => resultset[1]['tags'], 'value'
=> resultset[1]['cp1052.eqiad.wmnet']},
+ ]
+ req = { 'name' => 'cp1052.*', 'service' => 'varnish-..'}
+ genargs = gen_conftool_call('name=cp1052.*,service=varnish-..')
+ generate.stubs(:call).with(genargs).returns(retval)
+ should run.with_params(req).and_return(conftool_out)
+ end
+
+ it "should fail if conftool read fails" do
+ generate.stubs(:call).raises(Puppet::ParseError, 'something')
+ req = {'name' => 'foo'}
+ should run.with_params(req).and_raise_error(Puppet::ParseError)
+ end
+
+ it "should respond with an error if an empty tag is specified" do
+ should run.with_params({}).and_raise_error(Puppet::ParseError)
+ end
+end
diff --git a/puppet/modules/wmflib/spec/functions/ensure_mounted_spec.rb
b/puppet/modules/wmflib/spec/functions/ensure_mounted_spec.rb
new file mode 100644
index 0000000..0923893
--- /dev/null
+++ b/puppet/modules/wmflib/spec/functions/ensure_mounted_spec.rb
@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe "the ensure_mounted function" do
+ it "should exist" do
+ expect(Puppet::Parser::Functions.function("ensure_mounted")).to
eq("function_ensure_mounted")
+ end
+
+ it "should raise a ParseError if there are less than 1 arguments" do
+ expect {
+ scope.function_ensure_mounted([])
+ }.to raise_error(ArgumentError)
+ end
+
+ it "should raise a ParseError if there are more than 1 arguments" do
+ expect {
+ scope.function_ensure_mounted(['a', 'b'])
+ }.to raise_error(ArgumentError)
+ end
+
+ it "should return 'mounted' for param 'present'" do
+ expect(scope.function_ensure_mounted(['present'])).to eq('mounted')
+ end
+
+ it "should return 'mounted' for param 'true'" do
+ expect(scope.function_ensure_mounted([true])).to eq('mounted')
+ end
+
+ it "should return 'absent' for param 'absent'" do
+ expect(scope.function_ensure_mounted(['absent'])).to eq('absent')
+ end
+
+ it "should return 'false' for param 'false'" do
+ expect(scope.function_ensure_mounted([false])).to eq(false)
+ end
+
+end
diff --git a/puppet/modules/wmflib/spec/functions/hash_deselect_re_spec.rb
b/puppet/modules/wmflib/spec/functions/hash_deselect_re_spec.rb
new file mode 100644
index 0000000..805310c
--- /dev/null
+++ b/puppet/modules/wmflib/spec/functions/hash_deselect_re_spec.rb
@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe "the hash_deselect_re function" do
+ it "should exist" do
+ expect(Puppet::Parser::Functions.function("hash_deselect_re")).to
eq("function_hash_deselect_re")
+ end
+
+ it "should raise a ParseError if there are less than 2 arguments" do
+ expect {
+ scope.function_hash_deselect_re(['a'])
+ }.to raise_error(Puppet::ParseError)
+ end
+
+ it "should raise a ParseError if there are more than 2 arguments" do
+ expect {
+ scope.function_hash_deselect_re(['a', 'b', 'c'])
+ }.to raise_error(Puppet::ParseError)
+ end
+
+ it "should select the right keys (simple)" do
+ expect(
+ scope.function_hash_deselect_re(['^a', {'abc' => 1, 'def' => 2, 'asdf'
=> 3}])
+ ).to eq({'def' => 2})
+ end
+
+ it "should select the right keys (neg lookahead)" do
+ expect(
+ scope.function_hash_deselect_re(['^(?!a)', {'abc' => 1, 'def' => 2,
'asdf' => 3}])
+ ).to eq({'abc' => 1, 'asdf' => 3})
+ end
+end
diff --git a/puppet/modules/wmflib/spec/functions/hash_select_re_spec.rb
b/puppet/modules/wmflib/spec/functions/hash_select_re_spec.rb
new file mode 100644
index 0000000..cc86663
--- /dev/null
+++ b/puppet/modules/wmflib/spec/functions/hash_select_re_spec.rb
@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe "the hash_select_re function" do
+ it "should exist" do
+ expect(Puppet::Parser::Functions.function("hash_select_re")).to
eq("function_hash_select_re")
+ end
+
+ it "should raise a ParseError if there are less than 2 arguments" do
+ expect {
+ scope.function_hash_select_re(['a'])
+ }.to raise_error(Puppet::ParseError)
+ end
+
+ it "should raise a ParseError if there are more than 2 arguments" do
+ expect {
+ scope.function_hash_select_re(['a', 'b', 'c'])
+ }.to raise_error(Puppet::ParseError)
+ end
+
+ it "should select the right keys (simple)" do
+ expect(
+ scope.function_hash_select_re(['^a', {'abc' => 1, 'def' => 2, 'asdf' =>
3}])
+ ).to eq({'abc' => 1, 'asdf' => 3})
+ end
+
+ it "should select the right keys (neg lookahead)" do
+ expect(
+ scope.function_hash_select_re(['^(?!a)', {'abc' => 1, 'def' => 2, 'asdf'
=> 3}])
+ ).to eq({'def' => 2})
+ end
+end
diff --git a/puppet/modules/wmflib/spec/functions/ipresolve_spec.rb
b/puppet/modules/wmflib/spec/functions/ipresolve_spec.rb
new file mode 100644
index 0000000..070dc01
--- /dev/null
+++ b/puppet/modules/wmflib/spec/functions/ipresolve_spec.rb
@@ -0,0 +1,24 @@
+require 'spec_helper'
+describe 'ipresolve' do
+
+ it "should resolve ipv4 addresses by default" do
+ should
run.with_params('install1002.wikimedia.org').and_return('208.80.154.86')
+ end
+ it "should resolve ipv4 addresses when explicitly asked to" do
+ should run.with_params('install1002.wikimedia.org',
'4').and_return('208.80.154.86')
+ end
+
+ it "should resolve ipv6 addresses" do
+ should run.with_params('install1002.wikimedia.org',
'6').and_return('2620::861:1:208:80:154:86')
+ end
+
+ it "should be able to perform a reverse DNS lookup" do
+ should run.with_params('2620::861:1:208:80:154:86',
'ptr').and_return('install1002.wikimedia.org')
+ should run.with_params('208.80.154.86',
'ptr').and_return('install1002.wikimedia.org')
+ end
+
+ it "fails when resolving an inexistent name" do
+ # This will test if your ISP does DNS hijacking too
+ should
run.with_params('host.does.not.exists').and_raise_error(RuntimeError)
+ end
+end
diff --git a/puppet/modules/wmflib/spec/functions/role_spec.rb
b/puppet/modules/wmflib/spec/functions/role_spec.rb
new file mode 100644
index 0000000..fd9a3b5
--- /dev/null
+++ b/puppet/modules/wmflib/spec/functions/role_spec.rb
@@ -0,0 +1,49 @@
+require 'spec_helper'
+describe 'role' do
+
+ before :each do
+ @compiler = Puppet::Parser::Compiler.new(Puppet::Node.new("foo"))
+ @scope = Puppet::Parser::Scope.new(@compiler)
+ @scope.source = Puppet::Resource::Type.new(:node, :foo)
+ @scope.stub(:is_nodescope?).and_return(true)
+ @topscope = @scope.compiler.topscope
+ @scope.parent = @topscope
+ roleclasses = ['role::test', 'role::test2']
+ roleclasses.each do |roleclass|
+ unless @compiler.topscope.find_hostclass(roleclass)
+ host_cls = Puppet::Resource::Type.new(:hostclass, roleclass)
+ @scope.known_resource_types.add_hostclass(host_cls)
+ end
+ end
+ end
+
+ it "should be called with one parameter" do
+ should run.and_raise_error(ArgumentError)
+ end
+ it "throws error if called outside of the node scope" do
+ should run.with_params('cache::text').and_raise_error(Puppet::ParseError)
+ end
+
+ it "throws error if called on a non-existing role" do
+ expect { @scope.function_role(['foo::bar']) }.to
raise_error(Puppet::ParseError)
+ end
+
+ it "includes the role class" do
+ expect { @scope.function_role(['test']) }.to_not raise_error()
+ end
+
+ it "raises an error when called more than once in a scope" do
+ @scope.function_role(['test2'])
+ expect { @scope.function_role(['test']) }.to
raise_error(Puppet::ParseError)
+ end
+
+ it "adds the keys to the top-scope variable" do
+ @scope.function_role(['test', 'test2'])
+ expect(@topscope.lookupvar('_roles')).to eq({'test' => true, 'test2' =>
true})
+ end
+
+ it "includes the role classes" do
+ @scope.function_role(['test'])
+ expect(@scope.find_hostclass('role::test')).to
be_an_instance_of(Puppet::Resource::Type)
+ end
+end
diff --git a/puppet/modules/wmflib/spec/hiera/proxy_backend_spec.rb
b/puppet/modules/wmflib/spec/hiera/proxy_backend_spec.rb
new file mode 100644
index 0000000..9b0574e
--- /dev/null
+++ b/puppet/modules/wmflib/spec/hiera/proxy_backend_spec.rb
@@ -0,0 +1,52 @@
+require 'spec_helper'
+require 'hiera'
+require 'hiera/backend/proxy_backend'
+
+describe 'proxy_backend' do
+ before :each do
+ # Build a node with two roles applied
+ @hiera = Hiera.new({:config => 'spec/fixtures/hiera.proxy.yaml'})
+ Hiera::Config.load('spec/fixtures/hiera.proxy.yaml')
+ @backend = Hiera::Backend::Proxy_backend.new()
+ @compiler = Puppet::Parser::Compiler.new(Puppet::Node.new("foo"))
+ @scope = Puppet::Parser::Scope.new(@compiler)
+ @scope.source = Puppet::Resource::Type.new(:node, :foo)
+ @scope.stub(:is_nodescope?).and_return(true)
+ @topscope = @scope.compiler.topscope
+ @topscope.setvar('hostname', 'foo')
+ @scope.parent = @topscope
+ roleclasses = ['role::test', 'role::test2']
+ roleclasses.each do |roleclass|
+ unless @compiler.topscope.find_hostclass(roleclass)
+ host_cls = Puppet::Resource::Type.new(:hostclass, roleclass)
+ @scope.known_resource_types.add_hostclass(host_cls)
+ end
+ end
+ end
+
+ it "lookup returns the default when no role is defined" do
+ expect(
+ @backend.lookup('mysql::innodb_threads',@topscope, nil, nil)
+ ).to eq(15)
+ end
+
+ it "lookup returns the role-specific value if a role is defined" do
+ @scope.function_role(['test'])
+ expect(
+ @backend.lookup('mysql::innodb_threads',@topscope, nil, nil)
+ ).to eq(50)
+ end
+
+ it "return the host-overridden value for a role-defined variable" do
+ @scope.function_role(['test'])
+ expect(
+ @backend.lookup('admin::groups',@topscope, nil, nil)
+ ).to eq(['go-spurs'])
+ end
+
+ it "merges values when using an array lookup" do
+ @scope.function_role(['test'])
+ expect(@backend.lookup('admin::groups', @topscope, nil, :array)).to
eq([['go-spurs'],['FooBar']])
+ end
+
+end
diff --git a/puppet/modules/wmflib/spec/hiera/role_backend_spec.rb
b/puppet/modules/wmflib/spec/hiera/role_backend_spec.rb
new file mode 100644
index 0000000..19c436a
--- /dev/null
+++ b/puppet/modules/wmflib/spec/hiera/role_backend_spec.rb
@@ -0,0 +1,53 @@
+require 'spec_helper'
+require 'hiera/backend/role_backend'
+require 'hiera'
+
+describe 'role_backend' do
+ before :each do
+
+ @hiera = Hiera.new({:config => 'spec/fixtures/hiera.yaml'})
+ Hiera::Config.load('spec/fixtures/hiera.yaml')
+ @backend = Hiera::Backend::Role_backend.new()
+ @compiler = Puppet::Parser::Compiler.new(Puppet::Node.new("foo"))
+ @scope = Puppet::Parser::Scope.new(@compiler)
+ @scope.source = Puppet::Resource::Type.new(:node, :foo)
+ @scope.stub(:is_nodescope?).and_return(true)
+ @topscope = @scope.compiler.topscope
+ @scope.parent = @topscope
+ roleclasses = ['role::test', 'role::test2']
+ roleclasses.each do |roleclass|
+ unless @compiler.topscope.find_hostclass(roleclass)
+ host_cls = Puppet::Resource::Type.new(:hostclass, roleclass)
+ @scope.known_resource_types.add_hostclass(host_cls)
+ end
+ end
+ end
+
+ it "get_path returns the correct path" do
+ expect(@backend.get_path('pippo', 'test', 'common', @scope)).to
eq("spec/fixtures/hieradata/role/common/test.yaml")
+ end
+
+ it "lookup returns nil when no role is defined" do
+ expect(@backend.lookup('admin::groups', @topscope, nil, nil)).to eq(nil)
+ end
+
+ it "lookup returns a value when a role is defined" do
+ @scope.function_role(['test'])
+ expect(@backend.lookup('admin::groups', @topscope, nil, nil)).to
eq(['FooBar'])
+ end
+
+ it "lookup raises an error if conflicting values are given in different
roles" do
+ @scope.function_role(['test', 'test2'])
+ expect {@backend.lookup('admin::groups', @topscope, nil, nil)}.to
raise_error(Exception, "Conflicting value for admin::groups found in role
test2")
+ end
+
+ it "merges values when using an array lookup" do
+ @scope.function_role(['test', 'test2'])
+ expect(@backend.lookup('admin::groups', @topscope, nil, :array)).to
eq([['FooBar'],['FooBar1']])
+ end
+
+ it "merges values when using hash lookup" do
+ @scope.function_role(['test', 'test2'])
+ expect(@backend.lookup('an_hash', @topscope, nil, :hash)).to
eq({"test2"=>true, "test3"=>{"another"=>"level"}, "test"=>true})
+ end
+end
diff --git a/puppet/modules/wmflib/spec/spec_helper.rb
b/puppet/modules/wmflib/spec/spec_helper.rb
new file mode 100644
index 0000000..d3923f8
--- /dev/null
+++ b/puppet/modules/wmflib/spec/spec_helper.rb
@@ -0,0 +1,8 @@
+require 'rspec-puppet'
+
+fixture_path = File.expand_path(File.join(__FILE__, '..', 'fixtures'))
+
+RSpec.configure do |c|
+ c.module_path = File.join(fixture_path, 'modules')
+ c.manifest_dir = File.join(fixture_path, 'manifests')
+end
--
To view, visit https://gerrit.wikimedia.org/r/343956
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I3ef41e75539befb10f13394b03d9d47cb90ef1dd
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: master
Gerrit-Owner: BryanDavis <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
