Adds a new resource type to puppet for web requests and implements a provider of that type using the ruby curl interface provided by the 'curb' rubygem
This is the second revision of the patch I had previous sent out, including put and delete methods, better session security and handling, and many more tests Signed-off-by: Mo Morsi <[email protected]> --- Local-branch: feature/master/7474 lib/puppet/external/curl.rb | 77 ++++++++++++++++++++++ lib/puppet/provider/web/curl.rb | 100 ++++++++++++++++++++++++++++ lib/puppet/type/web.rb | 110 +++++++++++++++++++++++++++++++ spec/unit/provider/web/curl_spec.rb | 122 +++++++++++++++++++++++++++++++++++ spec/unit/type/web_spec.rb | 90 ++++++++++++++++++++++++++ 5 files changed, 499 insertions(+), 0 deletions(-) create mode 100644 lib/puppet/external/curl.rb create mode 100644 lib/puppet/provider/web/curl.rb create mode 100644 lib/puppet/type/web.rb create mode 100644 spec/unit/provider/web/curl_spec.rb create mode 100644 spec/unit/type/web_spec.rb diff --git a/lib/puppet/external/curl.rb b/lib/puppet/external/curl.rb new file mode 100644 index 0000000..ecffb5e --- /dev/null +++ b/lib/puppet/external/curl.rb @@ -0,0 +1,77 @@ +# Provides an interface to curl using the curb gem for puppet +require 'curb' + +# uses nokogiri to verify responses w/ xpath +require 'nokogiri' + +class Curl::Easy + + # Format request parameters for the specified request method + def self.format_params(method, params) + if([:get, :delete].include?(method)) + return params.collect { |k,v| "#{k}=#{v}" }.join("&") unless params.nil? + return "" + end + # post, put: + cparams = [] + params.each_pair { |k,v| cparams << Curl::PostField.content(k,v) } unless params.nil? + return cparams + end + + # Format a url for the specified request method, base uri, and parameters + def self.format_url(method, uri, params) + if([:get, :delete].include?(method)) + url = uri + url += ";" + format_params(method, params) + return url + end + # post, put: + return uri + end + + # Invoke a new curl request and return result + def self.web_request(method, uri, request_params, params = {}) + raise Puppet::Error, "Must specify http method (#{method}) and uri (#{uri})" if method.nil? || uri.nil? + + curl = self.new + + if params.has_key?(:cookie) && !params[:cookie].nil? + curl.enable_cookies = true + curl.cookiefile = params[:cookie] + curl.cookiejar = params[:cookie] + end + + curl.follow_location = (params.has_key?(:follow) && params[:follow]) + + case(method) + when 'get' + curl.url = format_url(method, uri, request_params) + curl.http_get + return curl + + when 'post' + curl.url = format_url(method, uri, request_params) + curl.http_post(format_params(method, request_params)) + return curl + + when 'put' + curl.url = format_url(method, uri, request_params) + curl.http_put(format_params(method, request_params)) + return curl + + when 'delete' + curl.url = format_url(method, uri, request_params) + curl.http_delete + return curl + end + end + + def valid_status_code?(valid_values=[]) + valid_values.include?(response_code.to_s) + end + + def valid_xpath?(xpath="/") + !Nokogiri::HTML(body_str.to_s).xpath(xpath.to_s).empty? + end + +end diff --git a/lib/puppet/provider/web/curl.rb b/lib/puppet/provider/web/curl.rb new file mode 100644 index 0000000..5f37742 --- /dev/null +++ b/lib/puppet/provider/web/curl.rb @@ -0,0 +1,100 @@ +require 'fileutils' +require 'puppet/external/curl' + +# Puppet provider definition +Puppet::Type.type(:web).provide :curl do + desc "Use curl to access web resources" + + def get + @uri + end + + def post + @uri + end + + def delete + @uri + end + + def put + @uri + end + + def get=(uri) + @uri = uri + process_params('get', @resource, uri) + end + + def post=(uri) + @uri = uri + process_params('post', @resource, uri) + end + + def delete=(uri) + @uri = uri + process_params('delete', @resource, uri) + end + + def put=(uri) + @uri = uri + process_params('put', @resource, uri) + end + + private + + # Helper to process/parse web parameters + def process_params(request_method, params, uri) + begin + cookies = nil + if params[:store_cookies_at] + FileUtils.touch(params[:store_cookies_at]) if !File.exist?(params[:store_cookies_at]) + cookies = params[:store_cookies_at] + elsif params[:use_cookies_at] + cookies = params[:use_cookies_at] + end + + # Actually run the request and verify the result + result = Curl::Easy::web_request(request_method, uri, params[:parameters], + :cookie => cookies, + :follow => params[:follow]) + verify_result(result, + :returns => params[:returns], + :does_not_return => params[:does_not_return], + :contains => params[:contains], + :does_not_contain => params[:does_not_contain] ) + result.close + + rescue Exception => e + raise Puppet::Error, "An exception was raised when invoking web request: #{e}" + + ensure + FileUtils.rm_f(cookies) if params[:remove_cookies] + end + end + + # Helper to verify the response + def verify_result(result, verify = {}) + if !verify[:returns].nil? && + !result.valid_status_code?(verify[:returns]) + raise Puppet::Error, "Invalid HTTP Return Code: #{result.response_code}, + was expecting one of #{verify[:returns].join(", ")}" + end + + if !verify[:does_not_return].nil? && + result.valid_status_code?(verify[:does_not_return]) + raise Puppet::Error, "Invalid HTTP Return Code: #{result.response_code}, + was not expecting one of #{verify[:does_not_return].join(", ")}" + end + + if !verify[:contains].nil? && + !result.valid_xpath?(verify[:contains]) + raise Puppet::Error, "Expecting #{verify[:contains]} in the result" + end + + if !verify[:does_not_contain].nil? && + result.valid_xpath?(verify[:does_not_contain]) + raise Puppet::Error, "Not expecting #{verify[:does_not_contain]} in the result" + end + end +end diff --git a/lib/puppet/type/web.rb b/lib/puppet/type/web.rb new file mode 100644 index 0000000..c59fece --- /dev/null +++ b/lib/puppet/type/web.rb @@ -0,0 +1,110 @@ +require 'uri' + +# A puppet resource type used to access resources on the World Wide Web +Puppet::Type.newtype(:web) do + @doc = "Issue a request to a resource on the world wide web" + + private + + # Validates uris passed in + def self.validate_uri(url) + begin + uri = URI.parse(url) + raise ArgumentError, "Specified uri #{url} is not valid" if ![URI::HTTP, URI::HTTPS].include?(uri.class) + rescue URI::InvalidURIError + raise ArgumentError, "Specified uri #{url} is not valid" + end + end + + # Validates http statuses passed in + def self.validate_http_status(status) + status = [status] unless status.is_a?(Array) + status.each { |stat| + stat = stat.to_s + unless ['100', '101', '102', '122', + '200', '201', '202', '203', '204', '205', '206', '207', '226', + '300', '301', '302', '303', '304', '305', '306', '307', + '400', '401', '402', '403', '404', '405', '406', '407', '408', '409', + '410', '411', '412', '413', '414', '415', '416', '417', '418', + '422', '423', '424', '425', '426', '444', '449', '450', '499', + '500', '501', '502', '503', '504', '505', '506', '507', '508', ' 509', '510' + ].include?(stat) + raise ArgumentError, "Invalid http status code #{stat} specified" + end + } + end + + # Convert singular params into arrays of strings + def self.munge_array_params(value) + value = [value] unless value.is_a?(Array) + value = value.collect { |val| val.to_s } + value + end + + newparam :name + + newproperty(:get) do + desc "Issue get request to the specified uri" + validate do |value| Puppet::Type::Web.validate_uri(value) end + end + + newproperty(:post) do + desc "Issue post request to the specified uri" + validate do |value| Puppet::Type::Web.validate_uri(value) end + end + + newproperty(:delete) do + desc "Issue delete request to the specified uri" + validate do |value| Puppet::Type::Web.validate_uri(value) end + end + + newproperty(:put) do + desc "Issue put request to the specified uri" + validate do |value| Puppet::Type::Web.validate_uri(value) end + end + + newparam(:parameters) do + desc "Hash of parameters to include in the web request" + end + + newparam(:follow) do + desc "Boolean indicating if redirects should be followed" + newvalues(:true, :false) + end + + newparam(:store_cookies_at) do + desc "String indicating where session cookies should be stored" + end + + newparam(:use_cookies_at) do + desc "String indicating where session cookies should be read from" + end + + newparam(:remove_cookies) do + desc "Boolean indicating if cookies should be removed after using them" + newvalues(:true, :false) + end + + newparam(:returns) do + desc "Expected http return codes of the request" + defaultto ["200"] + validate do |value| Puppet::Type::Web.validate_http_status(value) end + munge do |value| Puppet::Type::Web.munge_array_params(value) end + end + + newparam(:does_not_return) do + desc "Unexecpected http return codes of the request" + validate do |value| Puppet::Type::Web.validate_http_status(value) end + munge do |value| Puppet::Type::Web.munge_array_params(value) end + end + + newparam(:contains) do + desc "XPath to verify as part of the result" + munge do |value| Puppet::Type::Web.munge_array_params(value) end + end + + newparam(:does_not_contain) do + desc "XPath to verify as not being part of the result" + munge do |value| Puppet::Type::Web.munge_array_params(value) end + end +end diff --git a/spec/unit/provider/web/curl_spec.rb b/spec/unit/provider/web/curl_spec.rb new file mode 100644 index 0000000..36f1ddf --- /dev/null +++ b/spec/unit/provider/web/curl_spec.rb @@ -0,0 +1,122 @@ +#!/usr/bin/env rspec +require 'spec_helper' + +provider_class = Puppet::Type.type(:web).provider(:curl) + +describe provider_class do + before :each do + @resource = Puppet::Resource.new(:web, 'foo') + @resource.stubs(:[]).returns(nil) + @provider = provider_class.new(@resource) + end + + def http_request(http_method, url) + @provider.method("#{http_method}=".to_sym).call url + end + + ['get', 'post', 'put', 'delete'].each do |http_method| + describe "##{http_method}" do + it "should issue #{http_method} request to uri" do + proc { + http_request http_method, "http://www.puppetlabs.com"; + }.should_not raise_error(Puppet::Error) + end + + it "should accept parameters for #{http_method} request to uri" do + proc { + @resource.stubs(:[]).with(:parameters).returns({:q => 'puppet' }) + @resource.stubs(:[]).with(:follow).returns(true) + http_request http_method, "http://www.google.com"; + }.should_not raise_error(Puppet::Error) + end + + it "should verify default success return http status code for #{http_method} request to uri" do + proc { + http_request http_method, "http://foobar"; + }.should raise_error(Puppet::Error) + + proc { + @resource.stubs(:[]).with(:follow).returns(true) + http_request http_method, "http://google.com"; + }.should_not raise_error(Puppet::Error) + end + + it "should verify return http status code for #{http_method} request to uri" do + proc { + expected_return_code = '500' + @resource.stubs(:[]).with(:follow).returns(false) + @resource.stubs(:[]).with(:returns).returns([expected_return_code]) + http_request http_method, "http://google.com"; + }.should raise_error(Puppet::Error) + + proc { + expected_return_code = http_method == "get" ? '301' : '405' + @resource.stubs(:[]).with(:follow).returns(false) + @resource.stubs(:[]).with(:returns).returns([expected_return_code]) + http_request http_method, "http://google.com"; + }.should_not raise_error(Puppet::Error) + end + + it "should verify does_not_return http status code for #{http_method} request to uri" do + proc { + return_code_not_expected = http_method == "get" ? '301' : '405' + @resource.stubs(:[]).with(:does_not_return).returns(return_code_not_expected) + http_request http_method, "http://google.com"; + }.should raise_error(Puppet::Error) + + proc { + return_code_not_expected = "500" + @resource.stubs(:[]).with(:follow).returns(true) + @resource.stubs(:[]).with(:does_not_return).returns(return_code_not_expected) + http_request http_method, "http://google.com"; + }.should_not raise_error(Puppet::Error) + end + + it "should verify result contains specified xpath for #{http_method} request to uri" do + proc { + @resource.stubs(:[]).with(:contains).returns('/html/body') + @resource.stubs(:[]).with(:follow).returns(true) + http_request http_method, "http://www.puppetlabs.com"; + }.should_not raise_error(Puppet::Error) + + proc { + @resource.stubs(:[]).with(:contains).returns('/html/head/body') + @resource.stubs(:[]).with(:follow).returns(true) + http_request http_method, "http://www.puppetlabs.com"; + }.should raise_error(Puppet::Error) + end + + it "should verify result does_not_contain specified xpath for #{http_method} request to uri" do + proc { + @resource.stubs(:[]).with(:does_not_contain).returns('/html/head/body') + @resource.stubs(:[]).with(:follow).returns(true) + http_request http_method, "http://www.puppetlabs.com"; + }.should_not raise_error(Puppet::Error) + + proc { + @resource.stubs(:[]).with(:does_not_contain).returns('/html/body') + @resource.stubs(:[]).with(:follow).returns(true) + http_request http_method, "http://www.puppetlabs.com"; + }.should raise_error(Puppet::Error) + end + + it "should verify cookies are stored for a #{http_method} request to uri" do + @resource.stubs(:[]).with(:store_cookies_at).returns("/tmp/#{http_method}.cookie") + http_request http_method, "http://amazon.com"; + File.exist?("/tmp/#{http_method}.cookie").should be_true + FileUtils.rm_f "/tmp/#{http_method}.cookie" + end + + #it "should verify cookies are used when for a #{http_method} request to uri" do + # how ? + #end + + it "should verify cookies are removed for a #{http_method} request to uri" do + @resource.stubs(:[]).with(:store_cookies_at).returns("/tmp/#{http_method}.cookie") + @resource.stubs(:[]).with(:remove_cookies).returns(true) + http_request http_method, "http://amazon.com"; + File.exist?("/tmp/#{http_method}.cookie").should be_false + end + end + end +end diff --git a/spec/unit/type/web_spec.rb b/spec/unit/type/web_spec.rb new file mode 100644 index 0000000..e170681 --- /dev/null +++ b/spec/unit/type/web_spec.rb @@ -0,0 +1,90 @@ +#!/usr/bin/env rspec +require 'spec_helper' + +host = Puppet::Type.type(:web) + +describe Puppet::Type.type(:web) do + before do + @class = host + end + + it "should have :name be its namevar" do + @class.key_attributes.should == [:name] + end + + describe "when validating attributes" do + [:parameters, :follow, :store_cookies_at, :use_cookies_at, :remove_cookies, + :returns, :does_not_return, :contains, :does_not_contain].each do |param| + it "should have a #{param} parameter" do + @class.attrtype(param).should == :param + end + end + + [:get, :post, :put, :delete ].each do |property| + it "should have a #{property} property" do + @class.attrtype(property).should == :property + end + end + end + + describe "when validating values" do + it "should validate uris" do + proc { Puppet::Type::Web.validate_uri("http://google.com";) }.should_not raise_error + proc { Puppet::Type::Web.validate_uri("foobar123") }.should raise_error(ArgumentError) end + + it "should validate http_status" do + proc { Puppet::Type::Web.validate_http_status("200") }.should_not raise_error + proc { Puppet::Type::Web.validate_http_status(["200", "400"]) }.should_not raise_error + proc { Puppet::Type::Web.validate_http_status("909") }.should raise_error(ArgumentError) + proc { Puppet::Type::Web.validate_http_status(["200", "909"]) }.should raise_error(ArgumentError) + end + + it "should munge array parameters" do + Puppet::Type::Web.munge_array_params(200).should == ["200"] + Puppet::Type::Web.munge_array_params([200]).should == ["200"] + Puppet::Type::Web.munge_array_params(["200"]).should == ["200"] + end + + [:get, :post, :put, :delete ].each do |property| + it "should require a valid uri for #{property} requests" do + proc { @class.new(:name => "#{property}_uri", property.to_sym => "") }.should raise_error(Puppet::Error,"Parameter #{property} failed: Specified uri is not valid") + end + + it "should require a valid returns value for #{property} requests if specified" do + proc { @class.new(:name => "#{property}_uri", property.to_sym => "http://www.puppetlabs.com";, :returns => "invalid" ) }.should raise_error(Puppet::Error,"Parameter returns failed: Invalid http status code invalid specified") + end + + it "should munge the returns value for #{property} requests if specified" do + type = @class.new(:name => "#{property}_uri", property.to_sym => "http://www.puppetlabs.com";, :returns => 200 ) + type.parameters[:returns].value.should == ['200'] + end + + it "should require a valid does_not_return value for #{property} requests if specified" do + proc { @class.new(:name => "#{property}_uri", property.to_sym => "http://www.puppetlabs.com";, :does_not_return => "invalid" ) }.should raise_error(Puppet::Error,"Parameter does_not_return failed: Invalid http status code invalid specified") + end + + it "should munge the does_not_return value for #{property} requests if specified" do + type = @class.new(:name => "#{property}_uri", property.to_sym => "http://www.puppetlabs.com";, :does_not_return => 200 ) + type.parameters[:does_not_return].value.should == ['200'] + end + + it "should require a valid follow value for #{property} requests" do + proc { @class.new(:name => "#{property}_uri", property.to_sym => "http://www.puppetlabs.com";, :follow => "invalid" ) }.should raise_error(Puppet::Error,'Parameter follow failed: Invalid value "invalid". Valid values are true, false. ') + end + + it "should munge the contains value for #{property} requests if specified" do + type = @class.new(:name => "#{property}_uri", property.to_sym => "http://www.puppetlabs.com";, :contains => '/foobar' ) + type.parameters[:contains].value.should == ['/foobar'] + end + + it "should munge the does_not_contains value for #{property} requests if specified" do + type = @class.new(:name => "#{property}_uri", property.to_sym => "http://www.puppetlabs.com";, :does_not_contain => '/foobar' ) + type.parameters[:does_not_contain].value.should == ['/foobar'] + end + + it "should require a valid remove_cookies value for #{property} requests" do + proc { @class.new(:name => "#{property}_uri", property.to_sym => "http://www.puppetlabs.com";, :remove_cookies => 5 ) }.should raise_error(Puppet::Error,'Parameter remove_cookies failed: Invalid value 5. Valid values are true, false. ') + end + end + end +end -- 1.7.5.4 -- You received this message because you are subscribed to the Google Groups "Puppet Developers" group. To post to this group, send email to [email protected]. To unsubscribe from this group, send email to [email protected]. For more options, visit this group at http://groups.google.com/group/puppet-dev?hl=en.
