Fix for #4832 -- Making PSON handle arbitrary binary data

The PSON library needlessly assumed that the data to be transmitted was well-
formed unicode.  This made Latin-1 users (and anyone who needed to serialize
arbitrary binary data) sad.  This patch goes some of the way to resolving the
issues, by passing through non-unicode data rather than just failing, adds
tests, and cleans up a pernicious assumption about escape characters in ruby
regular expressions not marked "n" (no-encoding).

Author: MarkusQ

lib/puppet/external/pson/pure/generator.rb

@@ -63,22 +63,15 @@ def utf8_to_pson(string) # :nodoc:
     end
   else
     def utf8_to_pson(string) # :nodoc:
-      string = string.gsub(/["\\\x0-\x1f]/) { MAP[$MATCH] }
-      string.gsub!(/(
-        (?:
+      string.
+        gsub(/["\\\x0-\x1f]/n) { MAP[$MATCH] }.
+        gsub(/((?:
           [\xc2-\xdf][\x80-\xbf]    |
           [\xe0-\xef][\x80-\xbf]{2} |
           [\xf0-\xf4][\x80-\xbf]{3}
-            )+ |
-            [\x80-\xc1\xf5-\xff]       # invalid
-              )/nx) { |c|
-        c.size == 1 and raise GeneratorError, "invalid utf8 byte: '#{c}'"
-        s = PSON::UTF8toUTF16.iconv(c).unpack('H*')[0]
-        s.gsub!(/.{4}/n, '\\\\u\&')
+            )+)/nx) { |c|
+        PSON::UTF8toUTF16.iconv(c).unpack('H*')[0].gsub(/.{4}/n, '\\\\u\&')
       }
-      string
-    rescue Iconv::Failure => e
-      raise GeneratorError, "Caught #{e.class}: #{e}"
     end
   end
   module_function :utf8_to_pson

spec/unit/util/json_spec.rb spec/unit/util/pson_spec.rb

@@ -18,4 +18,21 @@ class PsonUtil
     pson.expects(:from_pson).with("mydata")
     pson.pson_create("type" => "foo", "data" => "mydata")
   end
+
+
+  { 
+    'foo' => '"foo"',
+    1 => '1',
+    "\x80" => "\"\x80\"",
+    [] => '[]'
+  }.each { |str,pson|
+    it "should be able to encode #{str.inspect}" do
+      str.to_pson.should == pson
+    end
+  }
+
+  it "should be able to handle arbitrary binary data" do
+    bin_string = (1..20000).collect { |i| ((17*i+13*i*i) % 255).chr }.join
+    PSON.parse(%Q{{ "type": "foo", "data": #{bin_string.to_pson} }})["data"].should == bin_string
+  end
 end