pitrou commented on code in PR #50874:
URL: https://github.com/apache/arrow/pull/50874#discussion_r3845782203
##########
cpp/src/arrow/extension/tensor_internal.cc:
##########
@@ -32,16 +32,120 @@ namespace arrow::internal {
namespace {
-// Names indexed by rapidjson::Type enum value:
-// kNullType=0, kFalseType=1, kTrueType=2, kObjectType=3,
-// kArrayType=4, kStringType=5, kNumberType=6.
-constexpr const char* kJsonTypeNames[] = {"Null", "False", "True", "Object",
- "Array", "String", "Number"};
+const char* JsonTypeName(simdjson::dom::element_type type) {
+ switch (type) {
+ case simdjson::dom::element_type::ARRAY:
+ return "array";
+ case simdjson::dom::element_type::OBJECT:
+ return "object";
+ case simdjson::dom::element_type::INT64:
+ case simdjson::dom::element_type::UINT64:
+ case simdjson::dom::element_type::DOUBLE:
+ case simdjson::dom::element_type::BIGINT:
+ return "number";
+ case simdjson::dom::element_type::STRING:
+ return "string";
+ case simdjson::dom::element_type::BOOL:
+ return "boolean";
+ case simdjson::dom::element_type::NULL_VALUE:
+ return "null";
+ }
+ return "unknown";
+}
+
+Result<simdjson::dom::array> GetJsonArray(simdjson::dom::element value,
+ std::string_view name) {
+ if (!value.is_array()) {
+ return Status::Invalid(name, " must be an array, got ",
JsonTypeName(value.type()));
+ }
+ return ResolveSimdjsonResult(value.get_array(), "Failed to get JSON array");
+}
+
+Result<int64_t> GetJsonInt(simdjson::dom::element value, std::string_view name,
+ std::string_view expected) {
+ if (!value.is_int64()) {
+ return Status::Invalid(name, " must contain ", expected, ", got ",
+ JsonTypeName(value.type()));
+ }
+ return ResolveSimdjsonResult(value.get_int64(), "Failed to get JSON
integer");
+}
} // namespace
-const char* JsonTypeName(const ::arrow::rapidjson::Value& v) {
- return kJsonTypeNames[v.GetType()];
+Result<simdjson::dom::object> ParseJsonObject(simdjson::dom::parser& parser,
+ const std::string& json) {
+ return ResolveSimdjsonResult(parser.parse(json).get_object(),
+ "Invalid serialized JSON data");
+}
+
+Result<std::optional<simdjson::dom::element>> GetOptionalJsonField(
+ const simdjson::dom::object& object, std::string_view key) {
+ auto field = object.at_key(key);
Review Comment:
Note that:
> This function has linear-time complexity: the keys are checked one by one.
according to
https://simdjson.github.io/simdjson/classsimdjson_1_1dom_1_1object.html#a4fb3e04e0ea591132da90341bf9ed819
This is not necessary a problem if we except only a couple keys in the
object, but worth adding a comment IMHO.
##########
cpp/src/arrow/extension/fixed_shape_tensor.cc:
##########
@@ -116,75 +115,58 @@ Result<std::shared_ptr<DataType>>
FixedShapeTensorType::Deserialize(
return Status::Invalid("Expected FixedSizeList storage type, got ",
storage_type->ToString());
}
+
auto fsl_type =
internal::checked_pointer_cast<FixedSizeListType>(storage_type);
auto value_type = fsl_type->value_type();
- rj::Document document;
- if (document.Parse(serialized_data.data(),
serialized_data.length()).HasParseError() ||
- !document.IsObject() || !document.HasMember("shape") ||
- !document["shape"].IsArray()) {
- return Status::Invalid("Invalid serialized JSON data: ", serialized_data);
- }
- std::vector<int64_t> shape;
- for (const auto& x : document["shape"].GetArray()) {
- if (!x.IsInt64()) {
- return Status::Invalid("shape must contain integers, got ",
- internal::JsonTypeName(x));
- }
- shape.emplace_back(x.GetInt64());
- }
+ simdjson::dom::parser parser;
+ ARROW_ASSIGN_OR_RAISE(auto object, internal::ParseJsonObject(parser,
serialized_data));
+
+ ARROW_ASSIGN_OR_RAISE(auto shape_value,
+ internal::ResolveSimdjsonResult(object.at_key("shape"),
+ "Invalid serialized
JSON data"));
+ ARROW_ASSIGN_OR_RAISE(auto shape, internal::GetJsonIntArray(shape_value,
"shape"));
+ ARROW_ASSIGN_OR_RAISE(auto permutation_value,
+ internal::GetOptionalJsonField(object, "permutation"));
std::vector<int64_t> permutation;
- if (document.HasMember("permutation")) {
- const auto& json_permutation = document["permutation"];
- if (!json_permutation.IsArray()) {
- return Status::Invalid("permutation must be an array, got ",
- internal::JsonTypeName(json_permutation));
- }
- for (const auto& x : json_permutation.GetArray()) {
- if (!x.IsInt64()) {
- return Status::Invalid("permutation must contain integers, got ",
- internal::JsonTypeName(x));
- }
- permutation.emplace_back(x.GetInt64());
- }
+ if (permutation_value.has_value()) {
+ ARROW_ASSIGN_OR_RAISE(permutation,
+ internal::GetJsonIntArray(*permutation_value,
"permutation"));
+
if (shape.size() != permutation.size()) {
return Status::Invalid("Invalid permutation");
}
RETURN_NOT_OK(internal::IsPermutationValid(permutation));
}
+
+ ARROW_ASSIGN_OR_RAISE(auto dim_names_value,
+ internal::GetOptionalJsonField(object, "dim_names"));
+
std::vector<std::string> dim_names;
- if (document.HasMember("dim_names")) {
- const auto& json_dim_names = document["dim_names"];
- if (!json_dim_names.IsArray()) {
- return Status::Invalid("dim_names must be an array, got ",
- internal::JsonTypeName(json_dim_names));
- }
- for (const auto& x : json_dim_names.GetArray()) {
- if (!x.IsString()) {
- return Status::Invalid("dim_names must contain strings, got ",
- internal::JsonTypeName(x));
- }
- dim_names.emplace_back(x.GetString());
- }
+ if (dim_names_value.has_value()) {
+ ARROW_ASSIGN_OR_RAISE(dim_names,
+ internal::GetJsonStringArray(*dim_names_value,
"dim_names"));
+
if (shape.size() != dim_names.size()) {
return Status::Invalid("Invalid dim_names");
}
}
- // Validate product of shape dimensions matches storage type list_size.
- // This check is intentionally after field parsing so that metadata-level
errors
- // (type mismatches, size mismatches) are reported first.
Review Comment:
Please don't remove comments when they are not outdated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
