Re: [protobuf] BinaryToJsonString mishandling strings containing UTF8 multibyte characters
Ron, could you post and example and some explanation on how to (de)serialize proto3 to JSON using LIBPROTOBUF_EXPORT util::Status BinaryToJsonString( TypeResolver* resolver, const string& type_url, const string& binary_input, string* json_output, const JsonOptions& options); How to create TypeResolver and what is type_url? I am asking because you seem to be the only one with expertise on the subject. Thank you, Zach. On Thursday, November 26, 2015 at 12:51:07 AM UTC-8, Ron Ben-Yosef wrote: > > > On Wednesday, November 25, 2015 at 8:56:51 PM UTC+2, Feng Xiao wrote: >> >> Thanks for the explanation. Could you help file a bug for this on >> protobuf github site? If you know of an solution to this, you are also >> welcomed to send us a pull request. >> > > Sure, no problem. > > https://github.com/google/protobuf/issues/1010 > -- You received this message because you are subscribed to the Google Groups "Protocol Buffers" group. To unsubscribe from this group and stop receiving emails from it, send an email to protobuf+unsubscr...@googlegroups.com. To post to this group, send email to protobuf@googlegroups.com. Visit this group at https://groups.google.com/group/protobuf. For more options, visit https://groups.google.com/d/optout.
Re: [protobuf] Example of BinaryToJsonString in C++
Thank you, I am working on it. Zach. On Tuesday, April 5, 2016 at 11:41:48 AM UTC-7, Feng Xiao wrote: > > See examples here: > > https://github.com/google/protobuf/blob/master/src/google/protobuf/util/json_util_test.cc#L78 > > On Mon, Apr 4, 2016 at 8:45 PM, Zachary Deretsky <zder...@gmail.com > > wrote: > >> Are there examples of (de)serializing to JSON in C++? >> What is TypeResolver and how to create it? >> Thanks, Zach. >> >> -- >> You received this message because you are subscribed to the Google Groups >> "Protocol Buffers" group. >> To unsubscribe from this group and stop receiving emails from it, send an >> email to protobuf+u...@googlegroups.com . >> To post to this group, send email to prot...@googlegroups.com >> . >> Visit this group at https://groups.google.com/group/protobuf. >> For more options, visit https://groups.google.com/d/optout. >> > > -- You received this message because you are subscribed to the Google Groups "Protocol Buffers" group. To unsubscribe from this group and stop receiving emails from it, send an email to protobuf+unsubscr...@googlegroups.com. To post to this group, send email to protobuf@googlegroups.com. Visit this group at https://groups.google.com/group/protobuf. For more options, visit https://groups.google.com/d/optout.
[protobuf] BinaryToJsonString generates invalid json for map
{"intMap":["4":500,"0":100,"1":200,"2":300,"3":400]}
The above json is declared invalid by several json validators.
ZZ It is generated by the jtest.proto:
syntax = "proto3";
package jtest;
message Jmap {
map int_map = 2;
}
ZZZ make_json.cpp :
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include "netopt/proto/json2string.h"
#include "jtest.grpc.pb.h"
using grpc::Channel;
using grpc::ClientContext;
using grpc::ClientReader;
using grpc::ClientReaderWriter;
using grpc::ClientWriter;
using grpc::Status;
using jtest::Jmap;
using ::google::protobuf::Map;
using ::google::protobuf::int32;
typedef ::google::protobuf::Map< ::google::protobuf::int32,
::google::protobuf::int32 > IntMap;
typedef ::google::protobuf::MapPair< ::google::protobuf::int32,
::google::protobuf::int32 > IntPair;
#define MAP_COUNT 5
Jmap MakeJmap() {
Jmap m;
IntMap* map_p =m.mutable_int_map();
for(int i = 0; i < MAP_COUNT; i++) {
IntPair pair1(i, (i + 1)*100 );
map_p->insert(pair1);
}
return m;
}
int main(int argc, char** argv) {
std::string m_file_name = "jmap.json";
std::string a_file_name = "array.json";
Jmap j = MakeJmap();
google::protobuf::util::Json2String j2s;
google::protobuf::util::JsonOptions jo;
// jo.add_whitespace = true;
jo.always_print_primitive_fields = true;
std::ofstream m_doc(m_file_name);
if(!m_doc.is_open()) {
GM_COUT << "Could not open " << m_file_name << ", EXITING!!!" <<
std::endl;
return 1;
}
std::string m_line = j2s.ToJson(j, jo);
if(m_line.empty()) {
GM_COUT << "Could not print to string, EXITING!!!" << std::endl;
return 1;
}
m_doc << m_line;
m_doc.close();
return 0;
}
where json2string.h is:
#include
#include
#include
#include
#include
#include
#include "common/gen_headers/macros.h"
#include "common/gen_headers/typedefs.h"
namespace google {
namespace protobuf {
namespace util {
static const char kTypeUrlPrefix[] = "type.googleapis.com";
static string GetTypeUrl(const Descriptor* message) {
return string(kTypeUrlPrefix) + "/" + message->full_name();
}
// As functions defined in json_util.h are just thin wrappers around
the
// JSON conversion code in //net/proto2/util/converter, in this test
we
// only cover some very basic cases to make sure the wrappers have
forwarded
// parameters to the underlying implementation correctly. More
detailed
// tests are contained in the //net/proto2/util/converter directory.
class Json2String {
public:
Json2String() {
resolver_.reset(NewTypeResolverForDescriptorPool(kTypeUrlPrefix,
DescriptorPool::generated_pool()));
}
string ToJson(const Message& message, const JsonOptions& options) {
string result;
Status status =
BinaryToJsonString(resolver_.get(),
GetTypeUrl(message.GetDescriptor()),
message.SerializeAsString(), , options);
if(!status.ok()) {
GM_COUT << status.ToString() << std::endl;
}
return result;
}
bool FromJson(const string& json, Message* message) {
string binary;
util::Status status =
JsonToBinaryString(resolver_.get(),
GetTypeUrl(message->GetDescriptor()), json, );
if(!status.ok()) {
GM_COUT << status.ToString() << std::endl;
return false;
}
return message->ParseFromString(binary);
}
google::protobuf::scoped_ptr resolver_;
};
} // namespace util
} // namespace protobuf
} // namespace google
--
You received this message because you are subscribed to the Google Groups
"Protocol Buffers" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to protobuf+unsubscr...@googlegroups.com.
To post to this group, send email to protobuf@googlegroups.com.
Visit this group at https://groups.google.com/group/protobuf.
For more options, visit https://groups.google.com/d/optout.
[protobuf] Re: BinaryToJsonString mishandling strings containing UTF8 multibyte characters
Thanks Ron, now I can generate json, but is is invalid for maps. I just posted an example with my code. Regards, Zach. On Tuesday, November 24, 2015 at 11:42:22 AM UTC-8, Ron Ben-Yosef wrote: > > Hi, > > When using *BinaryToJsonString *or *BinaryToJsonStream*, I seem to > encounter a problem whenever there's a message containing a string > containing multibyte characters. > After some debugging, it seems the place where things start to go wrong is > in *ReadCodePoint* (in json_escaping.cc) when the first byte of the > multibyte character is being read from the string (as char) and assigned > into a variable of type uint32. This casting directly from a signed 1-byte > value to an unsigned 4-byte value seems to produce values that are > different than intended and different than expected a little later on by > some *if-else* statements trying to look at that value to determine the > correct length of the multibyte character. From there things go wrong and > the string isn't serialized and just gets dropped... > > For now as a temporary solution I added a cast of the value returned by > StringPiece's *operator[ ]* to uint8 before the assignment into uint32, > but any advice or a more permanent solution will be appreciated. > > Thanks, > Ron > -- You received this message because you are subscribed to the Google Groups "Protocol Buffers" group. To unsubscribe from this group and stop receiving emails from it, send an email to protobuf+unsubscr...@googlegroups.com. To post to this group, send email to protobuf@googlegroups.com. Visit this group at https://groups.google.com/group/protobuf. For more options, visit https://groups.google.com/d/optout.
[protobuf] Does proto3 allow recursion?
Is recursion handled, for example:
message Node {
string value = 1;
Node parent = 2;
message Child {
string key = 1;
Node child = 2;
}
repeated Child children = 3;
}
or
message Node {
string value = 1;
Node parent = 2;
map children = 3;
}
--
You received this message because you are subscribed to the Google Groups
"Protocol Buffers" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to protobuf+unsubscr...@googlegroups.com.
To post to this group, send email to protobuf@googlegroups.com.
Visit this group at https://groups.google.com/group/protobuf.
For more options, visit https://groups.google.com/d/optout.
[protobuf] Example of BinaryToJsonString in C++
Are there examples of (de)serializing to JSON in C++? What is TypeResolver and how to create it? Thanks, Zach. -- You received this message because you are subscribed to the Google Groups "Protocol Buffers" group. To unsubscribe from this group and stop receiving emails from it, send an email to protobuf+unsubscr...@googlegroups.com. To post to this group, send email to protobuf@googlegroups.com. Visit this group at https://groups.google.com/group/protobuf. For more options, visit https://groups.google.com/d/optout.
[protobuf] Re: Does proto3 allow recursion?
Thank you. I am slowly struggling with my first grpc project, cmake being
part of the struggle.
On Friday, April 1, 2016 at 4:33:59 PM UTC-7, Josh Haberman wrote:
>
> The easiest way to find out is just to try. :) Both of these examples
> compile fine with proto3.
>
> On Monday, March 21, 2016 at 10:43:23 AM UTC-7, Zachary Deretsky wrote:
>>
>> Is recursion handled, for example:
>>
>> message Node {
>> string value = 1;
>> Node parent = 2;
>> message Child {
>> string key = 1;
>> Node child = 2;
>> }
>> repeated Child children = 3;
>> }
>>
>>
>> or
>>
>> message Node {
>> string value = 1;
>> Node parent = 2;
>> map<string, Node> children = 3;
>> }
>>
>>
>>
>>
>>
--
You received this message because you are subscribed to the Google Groups
"Protocol Buffers" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to protobuf+unsubscr...@googlegroups.com.
To post to this group, send email to protobuf@googlegroups.com.
Visit this group at https://groups.google.com/group/protobuf.
For more options, visit https://groups.google.com/d/optout.
Re: [protobuf] BinaryToJsonString generates invalid json for map
I created a github issue.
I am using proto3. I downloaded and installed grpc with protobuf following
instructions here: https://github.com/grpc/grpc/blob/release-0_13/INSTALL.md
I did this at the end of March.
Regards, Zach.
On Wednesday, April 13, 2016 at 11:10:53 AM UTC-7, Feng Xiao wrote:
>
>
>
> On Tue, Apr 12, 2016 at 5:18 PM, Zachary Deretsky <zder...@gmail.com
> > wrote:
>
>> {"intMap":["4":500,"0":100,"1":200,"2":300,"3":400]}
>>
> Which version of protobuf are you using? Clearly it should be using "{}"
> rather than "[]" for map fields. Could you help create a github issue for
> this bug?
> https://github.com/golang/protobuf/issues
>
>
>>
>> The above json is declared invalid by several json validators.
>>
>> ZZ It is generated by the jtest.proto:
>>
>> syntax = "proto3";
>> package jtest;
>>
>> message Jmap {
>> map<int32, int32> int_map = 2;
>> }
>>
>> ZZZ make_json.cpp :
>>
>> #include
>> #include
>> #include
>> #include
>> #include
>> #include
>> #include
>> #include
>>
>> #include
>> #include
>> #include <grpc++/channel.h>
>> #include <grpc++/client_context.h>
>> #include <grpc++/create_channel.h>
>> #include <grpc++/security/credentials.h>
>> #include "netopt/proto/json2string.h"
>> #include "jtest.grpc.pb.h"
>>
>> using grpc::Channel;
>> using grpc::ClientContext;
>> using grpc::ClientReader;
>> using grpc::ClientReaderWriter;
>> using grpc::ClientWriter;
>> using grpc::Status;
>>
>> using jtest::Jmap;
>>
>> using ::google::protobuf::Map;
>> using ::google::protobuf::int32;
>> typedef ::google::protobuf::Map< ::google::protobuf::int32,
>> ::google::protobuf::int32 > IntMap;
>> typedef ::google::protobuf::MapPair< ::google::protobuf::int32,
>> ::google::protobuf::int32 > IntPair;
>>
>> #define MAP_COUNT 5
>>
>>
>> Jmap MakeJmap() {
>> Jmap m;
>> IntMap* map_p =m.mutable_int_map();
>> for(int i = 0; i < MAP_COUNT; i++) {
>> IntPair pair1(i, (i + 1)*100 );
>> map_p->insert(pair1);
>> }
>> return m;
>> }
>>
>> int main(int argc, char** argv) {
>> std::string m_file_name = "jmap.json";
>> std::string a_file_name = "array.json";
>> Jmap j = MakeJmap();
>>
>> google::protobuf::util::Json2String j2s;
>> google::protobuf::util::JsonOptions jo;
>> // jo.add_whitespace = true;
>> jo.always_print_primitive_fields = true;
>>
>> std::ofstream m_doc(m_file_name);
>> if(!m_doc.is_open()) {
>> GM_COUT << "Could not open " << m_file_name << ", EXITING!!!" <<
>> std::endl;
>> return 1;
>> }
>> std::string m_line = j2s.ToJson(j, jo);
>> if(m_line.empty()) {
>> GM_COUT << "Could not print to string, EXITING!!!" << std::endl;
>> return 1;
>> }
>> m_doc << m_line;
>> m_doc.close();
>>
>> return 0;
>> }
>>
>> where json2string.h is:
>>
>> #include
>>
>> #include
>> #include
>>
>> #include
>> #include
>> #include
>> #include "common/gen_headers/macros.h"
>> #include "common/gen_headers/typedefs.h"
>>
>> namespace google {
>> namespace protobuf {
>> namespace util {
>> static const char kTypeUrlPrefix[] = "type.googleapis.com";
>>
>> static string GetTypeUrl(const Descriptor* message) {
>> return string(kTypeUrlPrefix) + "/" + message->full_name();
>> }
>>
>> // As functions defined in json_util.h are just thin wrappers
>> around the
>> // JSON conversion code in //net/proto2/util/converter, in this
>> test we
>> // only cover some very basic cases to make sure the wrappers have
>> forwarded
>> // parameters to the underlying implementation correctly. More
>> detailed
>> // tests are contained in the //net/proto2/util/converter directory.
>> class Json2String {
>> public:
>> Json2String() {
>> resolver_.reset(NewTypeResolverForDescriptorPool(kTypeUrlPrefix,
>> DescriptorPool::generated_pool()));
>> }
>>
>
