This is an automated email from the ASF dual-hosted git repository. bmahler pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git
commit b275a032f217794f20dae15d86f188e55d43ce59 Author: Benjamin Mahler <[email protected]> AuthorDate: Fri Nov 8 16:13:40 2019 -0800 Support jsonifying v0 protobuf to v1 protobuf. This allows us to jsonify a v0 protobuf directly to a v1 protobuf efficiently, with no need to `evolve()` the message (which is rather expensive). The way this works is by converting all "slave" and "SLAVE" strings in fields and enum values, respectively, to "agent" and "AGENT". Our current v0 to v1 conversion for the v1 operator API simply serializes the v0 message and de-serializes into a v1 message, which means all field tags and message structures are the same, except for field names. The only difference with field names is the use of "agent" in place of "slave". Review: https://reviews.apache.org/r/71748 --- src/common/http.cpp | 290 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/common/http.hpp | 13 +++ 2 files changed, 303 insertions(+) diff --git a/src/common/http.cpp b/src/common/http.cpp index b79074f..c5b2a91 100644 --- a/src/common/http.cpp +++ b/src/common/http.cpp @@ -21,6 +21,12 @@ #include <utility> #include <vector> +#include <google/protobuf/descriptor.h> +#include <google/protobuf/descriptor.pb.h> +#include <google/protobuf/message.h> +#include <google/protobuf/reflection.h> +#include <google/protobuf/repeated_field.h> + #include <mesos/attributes.hpp> #include <mesos/http.hpp> #include <mesos/resources.hpp> @@ -133,6 +139,290 @@ bool streamingMediaType(ContentType contentType) } +string lowerSlaveToAgent(string s) +{ + size_t index = 0; + + while (true) { + index = s.find("slave", index); + if (index == std::string::npos) break; + s.replace(index, 5, "agent"); + index += 5; + } + + return s; +} + + +string upperSlaveToAgent(string s) +{ + size_t index = 0; + + while (true) { + index = s.find("SLAVE", index); + if (index == std::string::npos) break; + s.replace(index, 5, "AGENT"); + index += 5; + } + + return s; +} + + +// This is a copy of the version in stout: +// +// https://github.com/apache/mesos/blob/1.9.0/3rdparty/ +// stout/include/stout/protobuf.hpp#L816 +// +// Except that we remap "slave" to "agent" in field names and +// enum values. +// +// Therefore, this is pretty brittle! If we change the one in +// stout we'll likely not realize this one should get a similar +// change. Note however that this can be removed if we were to +// keep up-to-date v1 state related objects in the master per +// MESOS-10040. +// +// Hopefully keeping a copy of the stout code doesn't prove +// too big of an issue since the stout version is pretty +// stable. Tickets to change the stout code (MESOS-3449, +// MESOS-6568, and MESOS-8727 have been updated with a note +// to change this copy as well). +void json(JSON::ObjectWriter* writer, const asV1Protobuf& protobuf) +{ + using google::protobuf::FieldDescriptor; + + const google::protobuf::Message& message = protobuf; + + const google::protobuf::Descriptor* descriptor = message.GetDescriptor(); + const google::protobuf::Reflection* reflection = message.GetReflection(); + + // We first look through all the possible fields to determine both the set + // fields __and__ the optional fields with a default that are not set. + // `Reflection::ListFields()` alone will only include set fields and + // is therefore insufficient. + int fieldCount = descriptor->field_count(); + std::vector<const FieldDescriptor*> fields; + fields.reserve(fieldCount); + for (int i = 0; i < fieldCount; ++i) { + const FieldDescriptor* field = descriptor->field(i); + if (field->is_repeated()) { // Repeated or Map. + if (reflection->FieldSize(message, field) > 0) { + // Has repeated field with members, output as JSON. + fields.push_back(field); + } + } else if ( + reflection->HasField(message, field) || + (field->has_default_value() && !field->options().deprecated())) { + // Field is set or has default, output as JSON. + fields.push_back(field); + } + } + + foreach (const FieldDescriptor* field, fields) { + if (field->is_repeated() && !field->is_map()) { + writer->field( + lowerSlaveToAgent(field->name()), + [&field, &reflection, &message](JSON::ArrayWriter* writer) { + int fieldSize = reflection->FieldSize(message, field); + for (int i = 0; i < fieldSize; ++i) { + switch (field->cpp_type()) { + case FieldDescriptor::CPPTYPE_BOOL: + writer->element( + reflection->GetRepeatedBool(message, field, i)); + break; + case FieldDescriptor::CPPTYPE_INT32: + writer->element( + reflection->GetRepeatedInt32(message, field, i)); + break; + case FieldDescriptor::CPPTYPE_INT64: + writer->element( + reflection->GetRepeatedInt64(message, field, i)); + break; + case FieldDescriptor::CPPTYPE_UINT32: + writer->element( + reflection->GetRepeatedUInt32(message, field, i)); + break; + case FieldDescriptor::CPPTYPE_UINT64: + writer->element( + reflection->GetRepeatedUInt64(message, field, i)); + break; + case FieldDescriptor::CPPTYPE_FLOAT: + writer->element( + reflection->GetRepeatedFloat(message, field, i)); + break; + case FieldDescriptor::CPPTYPE_DOUBLE: + writer->element( + reflection->GetRepeatedDouble(message, field, i)); + break; + case FieldDescriptor::CPPTYPE_MESSAGE: + writer->element( + asV1Protobuf( + reflection->GetRepeatedMessage(message, field, i))); + break; + case FieldDescriptor::CPPTYPE_ENUM: + writer->element( + upperSlaveToAgent( + reflection->GetRepeatedEnum(message, field, i) + ->name())); + break; + case FieldDescriptor::CPPTYPE_STRING: + const std::string& s = reflection->GetRepeatedStringReference( + message, field, i, nullptr); + if (field->type() == FieldDescriptor::TYPE_BYTES) { + writer->element(base64::encode(s)); + } else { + writer->element(s); + } + break; + } + } + }); + } else { // field->is_map() || !field->is_repeated() + auto writeField = [&writer]( + const std::string& fieldName, + const google::protobuf::Reflection* reflection, + const google::protobuf::Message& message, + const FieldDescriptor* field) { + switch (field->cpp_type()) { + case FieldDescriptor::CPPTYPE_BOOL: + writer->field(fieldName, reflection->GetBool(message, field)); + break; + case FieldDescriptor::CPPTYPE_INT32: + writer->field(fieldName, reflection->GetInt32(message, field)); + break; + case FieldDescriptor::CPPTYPE_INT64: + writer->field(fieldName, reflection->GetInt64(message, field)); + break; + case FieldDescriptor::CPPTYPE_UINT32: + writer->field(fieldName, reflection->GetUInt32(message, field)); + break; + case FieldDescriptor::CPPTYPE_UINT64: + writer->field(fieldName, reflection->GetUInt64(message, field)); + break; + case FieldDescriptor::CPPTYPE_FLOAT: + writer->field(fieldName, reflection->GetFloat(message, field)); + break; + case FieldDescriptor::CPPTYPE_DOUBLE: + writer->field(fieldName, reflection->GetDouble(message, field)); + break; + case FieldDescriptor::CPPTYPE_MESSAGE: + writer->field( + fieldName, + asV1Protobuf(reflection->GetMessage(message, field))); + break; + case FieldDescriptor::CPPTYPE_ENUM: + writer->field( + fieldName, + upperSlaveToAgent(reflection->GetEnum(message, field)->name())); + break; + case FieldDescriptor::CPPTYPE_STRING: + const std::string& s = + reflection->GetStringReference(message, field, nullptr); + if (field->type() == FieldDescriptor::TYPE_BYTES) { + writer->field(fieldName, base64::encode(s)); + } else { + writer->field(fieldName, s); + } + break; + } + }; + + if (!field->is_repeated()) { // Singular field. + writeField( + lowerSlaveToAgent(field->name()), reflection, message, field); + } else { // Map field. + CHECK(field->is_map()); + writer->field( + lowerSlaveToAgent(field->name()), + [&field, &reflection, &message, &writeField]( + JSON::ObjectWriter* writer) { + foreach ( + const auto& mapEntry, + reflection->GetRepeatedFieldRef<google::protobuf::Message>( + message, field)) { + const google::protobuf::Descriptor* mapEntryDescriptor = + mapEntry.GetDescriptor(); + const google::protobuf::Reflection* mapEntryReflection = + mapEntry.GetReflection(); + + // Map entry must contain exactly two fields: `key` and `value`. + CHECK_EQ(mapEntryDescriptor->field_count(), 2); + + const FieldDescriptor* keyField = mapEntryDescriptor->field(0); + const FieldDescriptor* valueField = + mapEntryDescriptor->field(1); + + switch (keyField->cpp_type()) { + case FieldDescriptor::CPPTYPE_BOOL: + writeField( + jsonify( + mapEntryReflection->GetBool(mapEntry, keyField)), + mapEntryReflection, + mapEntry, + valueField); + break; + case FieldDescriptor::CPPTYPE_INT32: + writeField( + jsonify( + mapEntryReflection->GetInt32(mapEntry, keyField)), + mapEntryReflection, + mapEntry, + valueField); + break; + case FieldDescriptor::CPPTYPE_INT64: + writeField( + jsonify( + mapEntryReflection->GetInt64(mapEntry, keyField)), + mapEntryReflection, + mapEntry, + valueField); + break; + case FieldDescriptor::CPPTYPE_UINT32: + writeField( + jsonify( + mapEntryReflection->GetUInt32(mapEntry, keyField)), + mapEntryReflection, + mapEntry, + valueField); + break; + case FieldDescriptor::CPPTYPE_UINT64: + writeField( + jsonify( + mapEntryReflection->GetUInt64(mapEntry, keyField)), + mapEntryReflection, + mapEntry, + valueField); + break; + case FieldDescriptor::CPPTYPE_STRING: + if (keyField->type() == FieldDescriptor::TYPE_BYTES) { + LOG(FATAL) + << "Unexpected key field type in protobuf Map: " + << keyField->type_name(); + } + + writeField( + mapEntryReflection->GetStringReference( + mapEntry, keyField, nullptr), + mapEntryReflection, + mapEntry, + valueField); + break; + case FieldDescriptor::CPPTYPE_FLOAT: + case FieldDescriptor::CPPTYPE_DOUBLE: + case FieldDescriptor::CPPTYPE_MESSAGE: + case FieldDescriptor::CPPTYPE_ENUM: + LOG(FATAL) << "Unexpected key field type in protobuf Map: " + << keyField->cpp_type_name(); + } + } + }); + } + } + } +} + + // TODO(bmahler): Kill these in favor of automatic Proto->JSON // Conversion (when it becomes available). diff --git a/src/common/http.hpp b/src/common/http.hpp index 062586c..b9ab561 100644 --- a/src/common/http.hpp +++ b/src/common/http.hpp @@ -181,6 +181,19 @@ struct StreamingHttpConnection }; +// The representation of generic v0 protobuf => v1 protobuf as JSON, +// e.g., `jsonify(asV1Protobuf(message))`. +// +// Specifically, this acts the same as JSON::Protobuf, except that +// it remaps "slave" to "agent" in field names and enum values. +struct asV1Protobuf : Representation<google::protobuf::Message> +{ + using Representation<google::protobuf::Message>::Representation; +}; + +void json(JSON::ObjectWriter* writer, const asV1Protobuf& protobuf); + + JSON::Object model(const Resources& resources); JSON::Object model(const hashmap<std::string, Resources>& roleResources); JSON::Object model(const Attributes& attributes);
