Giuseppe Lavagetto has uploaded a new change for review. https://gerrit.wikimedia.org/r/168984
Change subject: hiera: mediawiki-based backend for labs (wip) ...................................................................... hiera: mediawiki-based backend for labs (wip) Change-Id: I3710fdb23828693701025aa3879dd4a5813d4088 Signed-off-by: Giuseppe Lavagetto <[email protected]> --- A modules/wmflib/lib/hiera/backend/mwyaml_backend.rb A modules/wmflib/lib/hiera/mwcache.rb 2 files changed, 116 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/84/168984/1 diff --git a/modules/wmflib/lib/hiera/backend/mwyaml_backend.rb b/modules/wmflib/lib/hiera/backend/mwyaml_backend.rb new file mode 100644 index 0000000..9cf06a6 --- /dev/null +++ b/modules/wmflib/lib/hiera/backend/mwyaml_backend.rb @@ -0,0 +1,41 @@ +class Hiera + module Backend + class Mwyaml_backend + def initialize(cache=nil) + @httphost = config[:host] || nil + @cache = cache || MWcache.new + end + + def lookup(key, scope, order_override, resolution_type) + answer = nil + Hiera.debug("Looking up #{key}") + + Backend.datasources(scope, order_override) do |source| + data = @cache.read(source, Hash) do |content| + YAML.load(content) + end + + next if data.empty? + next unless data.include?(key) + + new_answer = Backend.parse_answer(data[key], scope) + case resolution_type + when :array + raise Exception, "Hiera type mismatch: expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String + answer ||= [] + answer << new_answer + when :hash + raise Exception, "Hiera type mismatch: expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash + answer ||= {} + answer = Backend.merge_answer(new_answer,answer) + else + answer = new_answer + break + end + end + + return answer + end + end + end +end diff --git a/modules/wmflib/lib/hiera/mwcache.rb b/modules/wmflib/lib/hiera/mwcache.rb new file mode 100644 index 0000000..1296ff7 --- /dev/null +++ b/modules/wmflib/lib/hiera/mwcache.rb @@ -0,0 +1,75 @@ +class Hiera + class Mwcache < Hiera::Filecache + def initialize + super + require 'httpclient' + require 'yaml' + require 'json' + config = Config[:mwyaml] + @httphost = config[:host] || 'wikitech.wikimedia.org' + @endpoint = config[:endpoint] || '/w/api.php' + @http = HTTPClient.new(:agent_name => 'HieraMwCache/0.1') + @stat_ttl = config[:cache_ttl] || 60 + #TODO: httpclient on precise may not support 1.2 + @http.ssl_config.ssl_version = 'TLSv1_2' + end + + def read_file(path, expected_type = Object, &block) + if stale?(path) + resp = @http.get("https://#{httphost}#{endpoint}?titles=#{path}&action=query&format=json";) + if resp.status_code == '200' + data = JSON.parse(resp.body) + @cache[path][:data] = block_given? ? yield(data) : data + end + if !@cache[path][:data].is_a?(expected_type) + raise TypeError, "Data retrieved from #{path} is #{data.class} not #{expected_type}" + end + end + + @cache[path][:data] + end + + private + + def stale?(path) + # Performs a request for the revision only + meta = path_metadata(path) + + if @cache[path][:meta].nil? + @cache[path][:meta] = meta + return true + end + if @cache[path][:meta][:revision] == meta[:revision] + @cache[path][:meta][:ts] = meta[:ts] + return false + else + @cache[path][:meta] = meta + return true + end + rescue => detail + error = "Retreiving metadata from ${path} failed: #{detail}" + Hiera.debug(error) + # Fill this up with very safe defaults + @cache[path][:meta] = {:ts => 0, :revision => 0} + return true + end + + def path_metadata(path) + now = Time.now.to_i + if @cache[path].nil? + @cache[path] = {:data => nil, :meta => nil} + elsif (now - @cache[path][:meta][:ts]) <= @stat_ttl + # if we already fetched the result within the last stat_ttl seconds, + # we don't bother killing the mediawiki instance with a flood of requests + return @cache[path][:meta] + end + # TODO: add some locking mechanism for requests? Maybe overkill, maybe not. + res = @http.get("#{httphost}#{endpoint}?labsproject=#{path}&gimme_revid_only=1") + if res.status_code == 200 + return {:ts => now, :revision => res.http_body} + else + raise IOError, "Could not correctly fetch revision for #{path}, HTTP status code #{res.status_code}" + end + end + end +end -- To view, visit https://gerrit.wikimedia.org/r/168984 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I3710fdb23828693701025aa3879dd4a5813d4088 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Giuseppe Lavagetto <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
