This is an automated email from the ASF dual-hosted git repository. zychen pushed a commit to branch http_improve in repository https://gitbox.apache.org/repos/asf/incubator-brpc.git
commit 8a4ffd01b1e8323d42a36199722c8c63736ca091 Author: Zhangyi Chen <[email protected]> AuthorDate: Mon Nov 23 19:46:07 2020 +0800 Compatibility improvement of protobuf json format and spring http spec --- src/brpc/policy/http_rpc_protocol.cpp | 7 ++- src/brpc/policy/http_rpc_protocol.h | 1 + src/json2pb/json_to_pb.cpp | 105 ++++++++++++++++++++++++++++--- src/json2pb/pb_to_json.cpp | 2 +- test/brpc_http_rpc_protocol_unittest.cpp | 37 +++++++++++ test/brpc_protobuf_json_unittest.cpp | 19 ++++-- 6 files changed, 157 insertions(+), 14 deletions(-) diff --git a/src/brpc/policy/http_rpc_protocol.cpp b/src/brpc/policy/http_rpc_protocol.cpp index 99579fe..c175f4f 100644 --- a/src/brpc/policy/http_rpc_protocol.cpp +++ b/src/brpc/policy/http_rpc_protocol.cpp @@ -21,6 +21,7 @@ #include <json2pb/pb_to_json.h> // ProtoMessageToJson #include <json2pb/json_to_pb.h> // JsonToProtoMessage +#include "brpc/policy/http_rpc_protocol.h" #include "butil/unique_ptr.h" // std::unique_ptr #include "butil/string_splitter.h" // StringMultiSplitter #include "butil/string_printf.h" @@ -110,6 +111,7 @@ CommonStrings::CommonStrings() , CONTENT_TYPE_TEXT("text/plain") , CONTENT_TYPE_JSON("application/json") , CONTENT_TYPE_PROTO("application/proto") + , CONTENT_TYPE_SPRING_PROTO("application/x-protobuf") , ERROR_CODE("x-bd-error-code") , AUTHORIZATION("authorization") , ACCEPT_ENCODING("accept-encoding") @@ -189,6 +191,9 @@ HttpContentType ParseContentType(butil::StringPiece ct, bool* is_grpc_ct) { } else if (ct.starts_with("proto")) { type = HTTP_CONTENT_PROTO; ct.remove_prefix(5); + } else if (ct.starts_with("x-protobuf")) { + type = HTTP_CONTENT_PROTO; + ct.remove_prefix(10); } else { return HTTP_CONTENT_OTHERS; } @@ -511,7 +516,7 @@ void SerializeHttpRequest(butil::IOBuf* /*not used*/, opt.bytes_to_base64 = cntl->has_pb_bytes_to_base64(); opt.jsonify_empty_array = cntl->has_pb_jsonify_empty_array(); opt.always_print_primitive_fields = cntl->has_always_print_primitive_fields(); - + opt.enum_option = (FLAGS_pb_enum_as_number ? json2pb::OUTPUT_ENUM_BY_NUMBER : json2pb::OUTPUT_ENUM_BY_NAME); diff --git a/src/brpc/policy/http_rpc_protocol.h b/src/brpc/policy/http_rpc_protocol.h index b6f986a..d43ff7e 100644 --- a/src/brpc/policy/http_rpc_protocol.h +++ b/src/brpc/policy/http_rpc_protocol.h @@ -37,6 +37,7 @@ struct CommonStrings { std::string CONTENT_TYPE_TEXT; std::string CONTENT_TYPE_JSON; std::string CONTENT_TYPE_PROTO; + std::string CONTENT_TYPE_SPRING_PROTO; std::string ERROR_CODE; std::string AUTHORIZATION; std::string ACCEPT_ENCODING; diff --git a/src/json2pb/json_to_pb.cpp b/src/json2pb/json_to_pb.cpp index 41da40d..f4face7 100644 --- a/src/json2pb/json_to_pb.cpp +++ b/src/json2pb/json_to_pb.cpp @@ -24,6 +24,7 @@ #include <typeinfo> #include <limits> #include <google/protobuf/descriptor.h> +#include "butil/strings/string_number_conversions.h" #include "json_to_pb.h" #include "zero_copy_stream_reader.h" // ZeroCopyStreamReader #include "encode_decode.h" @@ -207,10 +208,63 @@ inline bool convert_enum_type(const BUTIL_RAPIDJSON_NAMESPACE::Value&item, bool return true; } +inline bool convert_int64_type(const BUTIL_RAPIDJSON_NAMESPACE::Value& item, bool repeated, + google::protobuf::Message* message, + const google::protobuf::FieldDescriptor* field, + const google::protobuf::Reflection* reflection, + std::string* err) { + + int64_t num; + if (item.IsInt64()) { + if (repeated) { + reflection->AddInt64(message, field, item.GetInt64()); + } else { + reflection->SetInt64(message, field, item.GetInt64()); + } + } else if (item.IsString() && + butil::StringToInt64({item.GetString(), item.GetStringLength()}, + &num)) { + if (repeated) { + reflection->AddInt64(message, field, num); + } else { + reflection->SetInt64(message, field, num); + } + } else { + return value_invalid(field, "INT64", item, err); + } + return true; +} + +inline bool convert_uint64_type(const BUTIL_RAPIDJSON_NAMESPACE::Value& item, + bool repeated, + google::protobuf::Message* message, + const google::protobuf::FieldDescriptor* field, + const google::protobuf::Reflection* reflection, + std::string* err) { + uint64_t num; + if (item.IsUint64()) { + if (repeated) { + reflection->AddUInt64(message, field, item.GetUint64()); + } else { + reflection->SetUInt64(message, field, item.GetUint64()); + } + } else if (item.IsString() && + butil::StringToUint64({item.GetString(), item.GetStringLength()}, + &num)) { + if (repeated) { + reflection->AddUInt64(message, field, num); + } else { + reflection->SetUInt64(message, field, num); + } + } else { + return value_invalid(field, "UINT64", item, err); + } + return true; +} + bool JsonValueToProtoMessage(const BUTIL_RAPIDJSON_NAMESPACE::Value& json_value, google::protobuf::Message* message, - const Json2PbOptions& options, - std::string* err); + const Json2PbOptions& options, std::string* err); //Json value to protobuf convert rules for type: //Json value type Protobuf type convert rules @@ -219,9 +273,10 @@ bool JsonValueToProtoMessage(const BUTIL_RAPIDJSON_NAMESPACE::Value& json_value, //int64 int uint int64 uint64 valid convert is available //uint64 int uint int64 uint64 valid convert is available //int uint int64 uint64 float double available -//"NaN" "Infinity" "-Infinity" float double only "NaN" "Infinity" "-Infinity" is available +//"NaN" "Infinity" "-Infinity" float double only "NaN" "Infinity" "-Infinity" is available //int enum valid enum number value is available -//string enum valid enum name value is available +//string enum valid enum name value is available +//string int64 uint64 valid convert is available //other mismatch type convertion will be regarded as error. #define J2PCHECKTYPE(value, cpptype, jsontype) ({ \ MatchType match_type = TYPE_MATCH; \ @@ -234,6 +289,7 @@ bool JsonValueToProtoMessage(const BUTIL_RAPIDJSON_NAMESPACE::Value& json_value, match_type; \ }) + static bool JsonValueToProtoField(const BUTIL_RAPIDJSON_NAMESPACE::Value& value, const google::protobuf::FieldDescriptor* field, google::protobuf::Message* message, @@ -271,15 +327,48 @@ static bool JsonValueToProtoField(const BUTIL_RAPIDJSON_NAMESPACE::Value& value, reflection->Set##method(message, field, value.Get##jsontype()); \ } \ break; \ - } + } \ + CASE_FIELD_TYPE(INT32, Int32, Int); CASE_FIELD_TYPE(UINT32, UInt32, Uint); CASE_FIELD_TYPE(BOOL, Bool, Bool); - CASE_FIELD_TYPE(INT64, Int64, Int64); - CASE_FIELD_TYPE(UINT64, UInt64, Uint64); #undef CASE_FIELD_TYPE - case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: + case google::protobuf::FieldDescriptor::CPPTYPE_INT64: + if (field->is_repeated()) { + const BUTIL_RAPIDJSON_NAMESPACE::SizeType size = value.Size(); + for (BUTIL_RAPIDJSON_NAMESPACE::SizeType index = 0; index < size; + ++index) { + const BUTIL_RAPIDJSON_NAMESPACE::Value& item = value[index]; + if (!convert_int64_type(item, true, message, field, reflection, + err)) { + return false; + } + } + } else if (!convert_int64_type(value, false, message, field, reflection, + err)) { + return false; + } + break; + + case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: + if (field->is_repeated()) { + const BUTIL_RAPIDJSON_NAMESPACE::SizeType size = value.Size(); + for (BUTIL_RAPIDJSON_NAMESPACE::SizeType index = 0; index < size; + ++index) { + const BUTIL_RAPIDJSON_NAMESPACE::Value& item = value[index]; + if (!convert_uint64_type(item, true, message, field, reflection, + err)) { + return false; + } + } + } else if (!convert_uint64_type(value, false, message, field, reflection, + err)) { + return false; + } + break; + + case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: if (field->is_repeated()) { const BUTIL_RAPIDJSON_NAMESPACE::SizeType size = value.Size(); for (BUTIL_RAPIDJSON_NAMESPACE::SizeType index = 0; index < size; ++index) { diff --git a/src/json2pb/pb_to_json.cpp b/src/json2pb/pb_to_json.cpp index 704f963..55578c3 100644 --- a/src/json2pb/pb_to_json.cpp +++ b/src/json2pb/pb_to_json.cpp @@ -51,7 +51,7 @@ public: bool Convert(const google::protobuf::Message& message, Handler& handler); const std::string& ErrorText() const { return _error; } - + private: template <typename Handler> bool _PbFieldToJson(const google::protobuf::Message& message, diff --git a/test/brpc_http_rpc_protocol_unittest.cpp b/test/brpc_http_rpc_protocol_unittest.cpp index 15ef84b..3452332 100644 --- a/test/brpc_http_rpc_protocol_unittest.cpp +++ b/test/brpc_http_rpc_protocol_unittest.cpp @@ -1443,4 +1443,41 @@ TEST_F(HttpTest, http2_handle_goaway_streams) { brpc::Join(ids[i]); } } + +TEST_F(HttpTest, spring_protobuf_content_type) { + const int port = 8923; + brpc::Server server; + EXPECT_EQ(0, server.AddService(&_svc, brpc::SERVER_DOESNT_OWN_SERVICE)); + EXPECT_EQ(0, server.Start(port, nullptr)); + + brpc::Channel channel; + brpc::ChannelOptions options; + options.protocol = "http"; + ASSERT_EQ(0, channel.Init(butil::EndPoint(butil::my_ip(), port), &options)); + + brpc::Controller cntl; + test::EchoRequest req; + test::EchoResponse res; + req.set_message(EXP_REQUEST); + cntl.http_request().set_method(brpc::HTTP_METHOD_POST); + cntl.http_request().uri() = "/EchoService/Echo"; + cntl.http_request().set_content_type("application/x-protobuf"); + cntl.request_attachment().append(req.SerializeAsString()); + channel.CallMethod(nullptr, &cntl, nullptr, nullptr, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ("application/x-protobuf", cntl.http_response().content_type()); + ASSERT_TRUE(res.ParseFromString(cntl.response_attachment().to_string())); + ASSERT_EQ(EXP_RESPONSE, res.message()); + + brpc::Controller cntl2; + test::EchoService_Stub stub(&channel); + req.set_message(EXP_REQUEST); + res.Clear(); + cntl2.http_request().set_content_type("application/x-protobuf"); + stub.Echo(&cntl2, &req, &res, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(EXP_RESPONSE, res.message()); + ASSERT_EQ("application/x-protobuf", cntl.http_response().content_type()); +} + } //namespace diff --git a/test/brpc_protobuf_json_unittest.cpp b/test/brpc_protobuf_json_unittest.cpp index d7b4d1c..50b8e72 100644 --- a/test/brpc_protobuf_json_unittest.cpp +++ b/test/brpc_protobuf_json_unittest.cpp @@ -1308,7 +1308,7 @@ TEST_F(ProtobufJsonTest, pb_to_json_encode_decode_perf_case) { } TEST_F(ProtobufJsonTest, pb_to_json_complex_perf_case) { - + std::ifstream in("jsonout", std::ios::in); std::ostringstream tmp; tmp << in.rdbuf(); @@ -1317,8 +1317,8 @@ TEST_F(ProtobufJsonTest, pb_to_json_complex_perf_case) { printf("----------test pb to json performance------------\n\n"); - std::string error; - + std::string error; + butil::Timer timer; bool res; float avg_time1 = 0; @@ -1331,7 +1331,7 @@ TEST_F(ProtobufJsonTest, pb_to_json_complex_perf_case) { res = JsonToProtoMessage(info3, &data, option, &error); timer.stop(); avg_time1 += timer.u_elapsed(); - ASSERT_TRUE(res); + ASSERT_TRUE(res) << error; ProfilerStart("pb_to_json_complex_perf.prof"); for (int i = 0; i < times; i++) { std::string error1; @@ -1460,4 +1460,15 @@ TEST_F(ProtobufJsonTest, extension_case) { ASSERT_EQ("{\"hobby\":\"coding\",\"name\":\"hello\",\"id\":9,\"datadouble\":2.2,\"datafloat\":1.0}", output); } +TEST_F(ProtobufJsonTest, string_to_int64) { + auto json = R"({"name":"hello", "id":9, "data": "123456", "datadouble":2.2, "datafloat":1.0})"; + Person person; + std::string err; + ASSERT_TRUE(json2pb::JsonToProtoMessage(json, &person, &err)) << err; + ASSERT_EQ(person.data(), 123456); + json = R"({"name":"hello", "id":9, "data": "1234567", "datadouble":2.2, "datafloat":1.0})"; + ASSERT_TRUE(json2pb::JsonToProtoMessage(json, &person)); + ASSERT_EQ(person.data(), 1234567); +} + } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
