This specific format can unformat/format json in a streaming way. To activate it: --preferred_serialization_format=yajl
Apparently pson was serializing ruby objects by calling to_s. Yajl puts its public properties in a hash, since ResourceReference don't have a specific to_pson_data_hash we force them to serialize as string, thus enabling the same result as with pson. Signed-off-by: Brice Figureau <[email protected]> --- lib/puppet/feature/yajl.rb | 24 ++++++++ lib/puppet/network/formats.rb | 108 +++++++++++++++++++++++++++++++++++ lib/puppet/resource.rb | 2 +- spec/integration/network/formats.rb | 82 ++++++++++++++++++++++++++- spec/unit/network/formats.rb | 106 +++++++++++++++++++++++++++++++++- 5 files changed, 317 insertions(+), 5 deletions(-) create mode 100644 lib/puppet/feature/yajl.rb diff --git a/lib/puppet/feature/yajl.rb b/lib/puppet/feature/yajl.rb new file mode 100644 index 0000000..b4d4954 --- /dev/null +++ b/lib/puppet/feature/yajl.rb @@ -0,0 +1,24 @@ +require 'puppet/util/feature' + +# We want this to load if possible, but it's not automatically +# required. +Puppet.features.rubygems? +Puppet.features.add(:yajl) do + found = false + begin + require 'rubygems' + require 'yajl' + + #Yajl::Encoder.enable_json_gem_compatability + + class ::Object + def to_pson(*args, &block) + "\"#{to_s}\"" + end + end + + found = true + rescue LoadError => detail + end + found +end diff --git a/lib/puppet/network/formats.rb b/lib/puppet/network/formats.rb index a98dcbc..8b9b68d 100644 --- a/lib/puppet/network/formats.rb +++ b/lib/puppet/network/formats.rb @@ -186,3 +186,111 @@ Puppet::Network::FormatHandler.create(:pson, :mime => "text/pson", :weight => 10 klass.from_pson(data) end end + +Puppet::Network::FormatHandler.create(:yajl, :mime => "text/yajl", :weight => 20) do + confine :true => Puppet.features.yajl? + + class Puppet::ParsingComplete + attr_accessor :result + + def initialize + @result = [] + end + + def complete(object) + @result << object + end + end + + module ::PSON + alias :old_parse :parse + def parse(string) + Yajl::Parser.parse(string) + end + end + + def parse(content) + if content.respond_to?(:stream?) + unless content.stream? + Yajl::Parser.parse(content.content) + else + complete = Puppet::ParsingComplete.new + parser = Yajl::Parser.new + parser.on_parse_complete = complete.method(:complete) + content.stream do |r| + parser << r + end + result = complete.result + return result.shift if result.size == 1 + result + end + else + Yajl::Parser.parse(content) + end + end + + def intern(klass, content) + data_to_instance(klass, parse(content)) + end + + def intern_multiple(klass, content) + parse(content).collect do |data| + data_to_instance(klass, data) + end + end + + def render(instance) + Yajl::Encoder.encode(instance_to_data(instance)) + end + + def render_multiple(instances) + out = "" + encoder = Yajl::Encoder.new + instances.collect do |i| + out << encoder.encode(instance_to_data(i)) + end + out + end + + # supported only for brave souls installing yajl + def supported?(klass) + Puppet.features.yajl? + end + + def support_stream? + # of course we do + true + end + + # If they pass class information, we want to ignore it. By default, + # we'll include class information but we won't rely on it - we don't + # want class names to be required because we then can't change our + # internal class names, which is bad. + def data_to_instance(klass, data) + if data.is_a?(Hash) and d = data['data'] + data = d + end + if data.is_a?(klass) + return data + end + klass.from_pson(data) + end + + # recursively call to_pson_data_hash on objects + # supporting it + def instance_to_data(instance) + instance = instance.to_pson_data_hash if instance.respond_to?(:to_pson_data_hash) + case instance + when Hash + instance = instance.inject({}) do |h, (k,v)| + h[k] = instance_to_data(v) + h + end + when Array + instance.collect! do |i| + instance_to_data(i) + end + end + instance + end +end diff --git a/lib/puppet/resource.rb b/lib/puppet/resource.rb index 91dd547..2bf99c0 100644 --- a/lib/puppet/resource.rb +++ b/lib/puppet/resource.rb @@ -53,7 +53,7 @@ class Puppet::Resource # Don't duplicate the title as the namevar next hash if param == namevar and value == title - hash[param] = value + hash[param] = value.is_a?(Puppet::Resource::Reference) ? value.to_s : value hash end diff --git a/spec/integration/network/formats.rb b/spec/integration/network/formats.rb index 35e7977..6eef347 100755 --- a/spec/integration/network/formats.rb +++ b/spec/integration/network/formats.rb @@ -18,11 +18,15 @@ class PsonIntTest @string = string end - def to_pson(*args) + def to_pson_data_hash { 'type' => self.class.name, 'data' => [...@string] - }.to_pson(*args) + } + end + + def to_pson(*args) + to_pson_data_hash.to_pson(*args) end def self.canonical_order(s) @@ -108,3 +112,77 @@ describe Puppet::Network::FormatHandler.format(:pson) do end end end + +describe Puppet::Network::FormatHandler.format(:yajl) do + describe "when yajl is absent" do + confine "'yajl' library is present" => (! Puppet.features.yajl?) + + before do + @yajl = Puppet::Network::FormatHandler.format(:yajl) + end + + it "should not be suitable" do + @yajl.should_not be_suitable + end + + it "should not be supported" do + @yajl.should_not be_supported + end + end + + describe "when yajl is available" do + confine "Missing 'yajl' library" => Puppet.features.yajl? + + before do + @yajl = Puppet::Network::FormatHandler.format(:yajl) + end + + it "should be able to render an instance to json" do + instance = PsonIntTest.new("foo") + PsonIntTest.canonical_order(@yajl.render(instance)).should == PsonIntTest.canonical_order('{"type":"PsonIntTest","data":["foo"]}' ) + end + + it "should be able to render arrays to json" do + @yajl.render([1,2]).should == '[1,2]' + end + + it "should be able to render arrays containing hashes to json" do + @yajl.render([{"one"=>1},{"two"=>2}]).should == '[{"one":1},{"two":2}]' + end + + it "should be able to render multiple instances to json" do + Puppet.features.add(:yajl, :libs => %w{yajl}) + + one = PsonIntTest.new("one") + two = PsonIntTest.new("two") + + PsonIntTest.canonical_order(@yajl.render([one,two])).should == PsonIntTest.canonical_order('[{"type":"PsonIntTest","data":["one"]},{"type":"PsonIntTest","data":["two"]}]') + end + + it "should be able to intern a stream" do + content = stub 'stream', :stream? => true + content.expects(:stream).multiple_yields('{"type":"PsonIntTest",', '"data":["foo"]}') + @yajl.intern(PsonIntTest, content).should == PsonIntTest.new("foo") + end + + it "should be able to intern json into an instance" do + @yajl.intern(PsonIntTest, '{"type":"PsonIntTest","data":["foo"]}').should == PsonIntTest.new("foo") + end + + it "should be able to intern json with no class information into an instance" do + @yajl.intern(PsonIntTest, '["foo"]').should == PsonIntTest.new("foo") + end + + it "should be able to intern multiple instances from json" do + @yajl.intern_multiple(PsonIntTest, '[{"type": "PsonIntTest", "data": ["one"]},{"type": "PsonIntTest", "data": ["two"]}]').should == [ + PsonIntTest.new("one"), PsonIntTest.new("two") + ] + end + + it "should be able to intern multiple instances from json with no class information" do + @yajl.intern_multiple(PsonIntTest, '[["one"],["two"]]').should == [ + PsonIntTest.new("one"), PsonIntTest.new("two") + ] + end + end +end diff --git a/spec/unit/network/formats.rb b/spec/unit/network/formats.rb index a241306..208c6f4 100755 --- a/spec/unit/network/formats.rb +++ b/spec/unit/network/formats.rb @@ -18,11 +18,15 @@ class PsonTest @string = string end - def to_pson(*args) + def to_pson_data_hash { 'type' => self.class.name, 'data' => @string - }.to_pson(*args) + } + end + + def to_pson(*args) + to_pson_data_hash.to_pson(*args) end end @@ -362,4 +366,102 @@ describe "Puppet Network Format" do end end end + + + it "should include a yajl format" do + Puppet::Network::FormatHandler.format(:yajl).should_not be_nil + end + + describe "yajl" do + confine "Missing 'yajl' library" => Puppet.features.yajl? + + before do + @yajl = Puppet::Network::FormatHandler.format(:yajl) + end + + it "should have its mime type set to text/yajl" do + Puppet::Network::FormatHandler.format(:yajl).mime.should == "text/yajl" + end + + it "should require the :render_method" do + Puppet::Network::FormatHandler.format(:yajl).required_methods.should be_include(:render_method) + end + + it "should require the :intern_method" do + Puppet::Network::FormatHandler.format(:yajl).required_methods.should be_include(:intern_method) + end + + it "should have a weight of 20" do + @yajl.weight.should == 20 + end + + describe "when supported" do + it "should render by calling 'to_pson_data_hash' on the instance" do + instance = PsonTest.new("foo") + instance.expects(:to_pson_data_hash).returns "foo" + @yajl.render(instance).should == "\"foo\"" + end + + it "should render multiple instances by calling 'to_pson_data_hash' on each element array" do + instance = mock "instance" + instances = [instance] + + instance.expects(:to_pson_data_hash).returns "foo" + + @yajl.render_multiple(instances).should == "\"foo\"" + end + + it "should intern by calling 'Yajl::Parser.parse' on the text and then using from_pson to convert the data into an instance" do + content = stub 'foo', :stream? => false, :content => "foo" + Yajl::Parser.expects(:parse).with("foo").returns("type" => "PsonTest", "data" => "foo") + PsonTest.expects(:from_pson).with("foo").returns "parsed_yajl" + @yajl.intern(PsonTest, content).should == "parsed_yajl" + end + + it "should intern by calling 'Yajl::Parser.parse' in stream mode if content is streamable" do + content = stub 'foo', :stream? => true + content.expects(:stream).yields "foo" + parser = stub_everything 'parser' + Yajl::Parser.expects(:new).returns(parser) + parser.expects(:<<).with("foo").returns("type" => "PsonTest", "data" => "foo") + @yajl.intern(PsonTest, content) + end + + it "should return parsed objects in stream mode" do + content = stub 'foo', :stream? => true + content.stubs(:stream).yields "foo" + parsing_complete = stub 'parsing_complete' + parsing_complete.expects(:method) + Puppet::ParsingComplete.expects(:new).returns(parsing_complete) + parser = stub_everything 'parser' + Yajl::Parser.expects(:new).returns(parser) + parser.expects(:<<).with("foo").returns("type" => "PsonTest", "data" => "foo") + parsing_complete.expects(:result).returns([]) + @yajl.intern(PsonTest, content) + end + + it "should not render twice if 'Yajl::Parser.parse' creates the appropriate instance" do + text = stub 'foo', :stream? => false, :content => "foo" + instance = PsonTest.new("foo") + Yajl::Parser.expects(:parse).with("foo").returns(instance) + PsonTest.expects(:from_pson).never + @yajl.intern(PsonTest, text).should equal(instance) + end + + it "should intern by calling 'Yajl::Parser.parse' on the text and then using from_pson to convert the actual into an instance if the yajl has no class/data separation" do + text = stub 'foo', :stream? => false, :content => "foo" + Yajl::Parser.expects(:parse).with("foo").returns("foo") + PsonTest.expects(:from_pson).with("foo").returns "parsed_yajl" + @yajl.intern(PsonTest, text).should == "parsed_yajl" + end + + it "should intern multiples by parsing the text and using 'class.intern' on each resulting data structure" do + text = stub 'foo', :stream? => false, :content => "foo" + Yajl::Parser.expects(:parse).with("foo").returns ["bar", "baz"] + PsonTest.expects(:from_pson).with("bar").returns "BAR" + PsonTest.expects(:from_pson).with("baz").returns "BAZ" + @yajl.intern_multiple(PsonTest, text).should == %w{BAR BAZ} + end + end + end end -- 1.6.6.1 -- You received this message because you are subscribed to the Google Groups "Puppet Developers" group. To post to this group, send email to [email protected]. To unsubscribe from this group, send email to [email protected]. For more options, visit this group at http://groups.google.com/group/puppet-dev?hl=en.
