Is this going to be contributed upstream to the json developers? -- Luke Kanies | +1-615-594-8199
On Oct 13, 2010, at 14:58, Markus Roberts <[email protected]> wrote: > The PSON library needlessly assumed that the data to be transmitted was well- > formed unicode. This made Latin-1 users (and anyone who needed to serialize > arbitrary binary data) sad. This patch goes some of the way to resolving the > issues, by passing through non-unicode data rather than just failing, adds > tests, and cleans up a pernicious assumption about escape characters in ruby > regular expressions not marked "n" (no-encoding). > > Signed-off-by: Markus Roberts <[email protected]> > --- > lib/puppet/external/pson/pure/generator.rb | 17 +++--------- > spec/unit/util/json_spec.rb | 21 --------------- > spec/unit/util/pson_spec.rb | 38 ++++++++++++++++++++++++++++ > 3 files changed, 43 insertions(+), 33 deletions(-) > delete mode 100755 spec/unit/util/json_spec.rb > create mode 100755 spec/unit/util/pson_spec.rb > > diff --git a/lib/puppet/external/pson/pure/generator.rb > b/lib/puppet/external/pson/pure/generator.rb > index ef8b36d..4180be5 100644 > --- a/lib/puppet/external/pson/pure/generator.rb > +++ b/lib/puppet/external/pson/pure/generator.rb > @@ -63,22 +63,15 @@ module PSON > end > else > def utf8_to_pson(string) # :nodoc: > - string = string.gsub(/["\\\x0-\x1f]/) { MAP[$MATCH] } > - string.gsub!(/( > - (?: > + string. > + gsub(/["\\\x0-\x1f]/n) { MAP[$MATCH] }. > + gsub(/((?: > [\xc2-\xdf][\x80-\xbf] | > [\xe0-\xef][\x80-\xbf]{2} | > [\xf0-\xf4][\x80-\xbf]{3} > - )+ | > - [\x80-\xc1\xf5-\xff] # invalid > - )/nx) { |c| > - c.size == 1 and raise GeneratorError, "invalid utf8 byte: '#{c}'" > - s = PSON::UTF8toUTF16.iconv(c).unpack('H*')[0] > - s.gsub!(/.{4}/n, '\\\\u\&') > + )+)/nx) { |c| > + PSON::UTF8toUTF16.iconv(c).unpack('H*')[0].gsub(/.{4}/n, '\\\\u\&') > } > - string > - rescue Iconv::Failure => e > - raise GeneratorError, "Caught #{e.class}: #{e}" > end > end > module_function :utf8_to_pson > diff --git a/spec/unit/util/json_spec.rb b/spec/unit/util/json_spec.rb > deleted file mode 100755 > index 4f6cea9..0000000 > --- a/spec/unit/util/json_spec.rb > +++ /dev/null > @@ -1,21 +0,0 @@ > -#!/usr/bin/env ruby > - > -Dir.chdir(File.dirname(__FILE__)) { (s = lambda { |f| File.exist?(f) ? > require(f) : Dir.chdir("..") { s.call(f) } }).call("spec/spec_helper.rb") } > - > -require 'puppet/util/pson' > - > -class PsonUtil > - include Puppet::Util::Pson > -end > - > -describe Puppet::Util::Pson do > - it "should fail if no data is provided" do > - lambda { PsonUtil.new.pson_create("type" => "foo") }.should > raise_error(ArgumentError) > - end > - > - it "should call 'from_pson' with the provided data" do > - pson = PsonUtil.new > - pson.expects(:from_pson).with("mydata") > - pson.pson_create("type" => "foo", "data" => "mydata") > - end > -end > diff --git a/spec/unit/util/pson_spec.rb b/spec/unit/util/pson_spec.rb > new file mode 100755 > index 0000000..d02d285 > --- /dev/null > +++ b/spec/unit/util/pson_spec.rb > @@ -0,0 +1,38 @@ > +#!/usr/bin/env ruby > + > +Dir.chdir(File.dirname(__FILE__)) { (s = lambda { |f| File.exist?(f) ? > require(f) : Dir.chdir("..") { s.call(f) } }).call("spec/spec_helper.rb") } > + > +require 'puppet/util/pson' > + > +class PsonUtil > + include Puppet::Util::Pson > +end > + > +describe Puppet::Util::Pson do > + it "should fail if no data is provided" do > + lambda { PsonUtil.new.pson_create("type" => "foo") }.should > raise_error(ArgumentError) > + end > + > + it "should call 'from_pson' with the provided data" do > + pson = PsonUtil.new > + pson.expects(:from_pson).with("mydata") > + pson.pson_create("type" => "foo", "data" => "mydata") > + end > + > + > + { > + 'foo' => '"foo"', > + 1 => '1', > + "\x80" => "\"\x80\"", > + [] => '[]' > + }.each { |str,pson| > + it "should be able to encode #{str.inspect}" do > + str.to_pson.should == pson > + end > + } > + > + it "should be able to handle arbitrary binary data" do > + bin_string = (1..20000).collect { |i| ((17*i+13*i*i) % 255).chr }.join > + PSON.parse(%Q{{ "type": "foo", "data": #{bin_string.to_pson} > }})["data"].should == bin_string > + end > +end > -- > 1.7.0.4 > > -- > You received this message because you are subscribed to the Google Groups > "Puppet Developers" group. > To post to this group, send email to [email protected]. > To unsubscribe from this group, send email to > [email protected]. > For more options, visit this group at > http://groups.google.com/group/puppet-dev?hl=en. > -- You received this message because you are subscribed to the Google Groups "Puppet Developers" group. To post to this group, send email to [email protected]. To unsubscribe from this group, send email to [email protected]. For more options, visit this group at http://groups.google.com/group/puppet-dev?hl=en.
