Yuvipanda has uploaded a new change for review. https://gerrit.wikimedia.org/r/310619
Change subject: labs: Introduce httpyaml backend for labtest ...................................................................... labs: Introduce httpyaml backend for labtest Bug: T133412 Change-Id: I3d4ebd5d72afbe5b1f71a84ffe0f9b16aa11f579 --- M modules/puppetmaster/files/labtest.hiera.yaml A modules/wmflib/lib/hiera/backend/httpyaml_backend.rb A modules/wmflib/lib/hiera/httpcache.rb 3 files changed, 123 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/19/310619/1 diff --git a/modules/puppetmaster/files/labtest.hiera.yaml b/modules/puppetmaster/files/labtest.hiera.yaml index 61e8998..448f6bb 100644 --- a/modules/puppetmaster/files/labtest.hiera.yaml +++ b/modules/puppetmaster/files/labtest.hiera.yaml @@ -10,6 +10,8 @@ :datadir: /etc/puppet/private/hieradata :secret: :datadir: /etc/puppet/secret/hieradata +:httpyaml: + :url_prefix: http://localhost:8100/v1/ :hierarchy: - "labs/%{::labsproject}/host/%{::hostname}" - "labs/%{::labsproject}/common" @@ -21,3 +23,4 @@ - common - "secret/common" - "private/common" + - "httpyaml/%{::labsproject}/node/%{::fqdn}" diff --git a/modules/wmflib/lib/hiera/backend/httpyaml_backend.rb b/modules/wmflib/lib/hiera/backend/httpyaml_backend.rb new file mode 100644 index 0000000..32aab64 --- /dev/null +++ b/modules/wmflib/lib/hiera/backend/httpyaml_backend.rb @@ -0,0 +1,44 @@ +require "hiera/httpcache" +class Hiera + module Backend + class Mwyaml_backend + def initialize(cache=nil) + @cache = cache || Httpcache.new + end + + def lookup(key, scope, order_override, resolution_type) + answer = nil + Hiera.debug("Looking up #{key}") + + Backend.datasources(scope, order_override) do |source| + # Small hack: We don't want to search any datasource but the + # httpyaml/%{::labsproject} hierarchy here; so we plainly exit + # in any other case. + next unless source.start_with?('httpyaml/') && source.length > 'httpyaml/'.length + + data = @cache.read(source, Hash, {}) + + next if data.nil? || data.empty? + next unless data.include?(key) + + new_answer = Backend.parse_answer(data[key], scope) + case resolution_type + when :array + raise Exception, "Hiera type mismatch: expected Array and got #{new_answer.class}" unless new_answer.kind_of?(Array) || new_answer.kind_of?(String) + answer ||= [] + answer << new_answer + when :hash + raise Exception, "Hiera type mismatch: expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash + answer ||= {} + answer = Backend.merge_answer(new_answer,answer) + else + answer = new_answer + break + end + end + + return answer + end + end + end +end diff --git a/modules/wmflib/lib/hiera/httpcache.rb b/modules/wmflib/lib/hiera/httpcache.rb new file mode 100644 index 0000000..ceca3b7 --- /dev/null +++ b/modules/wmflib/lib/hiera/httpcache.rb @@ -0,0 +1,76 @@ +class Hiera + class Httpcache < Filecache + def initialize + super + require 'httpclient' + require 'yaml' + require 'json' + config = Config[:httpyaml] + @url_prefix = config[:url_prefix] + @http = HTTPClient.new(:agent_name => 'HieraHttpCache/0.1') + @stat_ttl = config[:cache_ttl] || 60 + if defined? @http.ssl_config.ssl_version + @http.ssl_config.ssl_version = 'TLSv1' + else + # Note: this seem to work in later versions of the library, + # but has no effect. How cute, I <3 ruby. + @http.ssl_config.options = OpenSSL::SSL::OP_NO_SSLv3 + end + end + + def read(path, expected_type, default=nil, &block) + read_file(path, expected_type, &block) + rescue => detail + # When failing to read data, we raise an exception, see https://phabricator.wikimedia.org/T78408 + error = "Reading data from #{path} failed: #{detail.class}: #{detail}" + raise error + end + + def read_file(path, expected_type = Object, &block) + if stale?(path) + data = get_from_http(path) + @cache[path][:data] = block_given? ? yield(data) : data + + if !@cache[path][:data].is_a?(expected_type) + raise TypeError, "Data retrieved from #{path} is #{data.class} not #{expected_type}" + end + end + + @cache[path][:data] + end + + private + + def stale?(path) + # We don't actually have caching information lol + # So we just, uh, blindly cache for 60s + now = Time.now.to_i + if @cache[path].nil? + @cache[path] = {:data => nil, :meta => {:ts => now}} + return True + elsif (now - @cache[path][:meta][:ts]) <= @stat_ttl + # if we already fetched the result within the last stat_ttl seconds, + # we don't bother killing the mediawiki instance with a flood of requests + return False + else + # This means there's a ts and it's old enough + @cache[path][:meta][:ts] => now + return True + end + end + + def get_from_http(path) + url = path.sub('httpyaml/', @url_prefix) + Hiera.debug("Fetching #{url}") + res = @http.get(url) + if res.status_code != 200 + raise IOError, "Could not correctly fetch revision for #{path}, HTTP status code #{res.status_code}, content #{res.data}" + end + # We shamelessly throw exceptions here, and catch them upper in the chain + # specifically in Hiera::Mwcache.stale? and Hiera::Mwcache.read + body = YAML.parse(res.body) + + return body['hiera'] + end + end +end -- To view, visit https://gerrit.wikimedia.org/r/310619 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I3d4ebd5d72afbe5b1f71a84ffe0f9b16aa11f579 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Yuvipanda <yuvipa...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
