kou commented on code in PR #50874:
URL: https://github.com/apache/arrow/pull/50874#discussion_r3827463807
##########
cpp/src/arrow/extension/fixed_shape_tensor.cc:
##########
@@ -116,60 +115,189 @@ Result<std::shared_ptr<DataType>>
FixedShapeTensorType::Deserialize(
return Status::Invalid("Expected FixedSizeList storage type, got ",
storage_type->ToString());
}
+
auto fsl_type =
internal::checked_pointer_cast<FixedSizeListType>(storage_type);
auto value_type = fsl_type->value_type();
- rj::Document document;
- if (document.Parse(serialized_data.data(),
serialized_data.length()).HasParseError() ||
- !document.IsObject() || !document.HasMember("shape") ||
- !document["shape"].IsArray()) {
- return Status::Invalid("Invalid serialized JSON data: ", serialized_data);
- }
- std::vector<int64_t> shape;
- for (const auto& x : document["shape"].GetArray()) {
- if (!x.IsInt64()) {
- return Status::Invalid("shape must contain integers, got ",
- internal::JsonTypeName(x));
- }
- shape.emplace_back(x.GetInt64());
- }
+ simdjson::padded_string padded_json(serialized_data);
+ simdjson::ondemand::parser parser;
+
+ ARROW_ASSIGN_OR_RAISE(auto document,
+
internal::ResolveSimdjsonResult(parser.iterate(padded_json),
+ "Invalid serialized
JSON data"));
+ ARROW_ASSIGN_OR_RAISE(auto object,
+ internal::ResolveSimdjsonResult(document.get_object(),
+ "Invalid serialized
JSON data"));
+
+ std::vector<int64_t> shape;
std::vector<int64_t> permutation;
- if (document.HasMember("permutation")) {
- const auto& json_permutation = document["permutation"];
- if (!json_permutation.IsArray()) {
- return Status::Invalid("permutation must be an array, got ",
- internal::JsonTypeName(json_permutation));
- }
- for (const auto& x : json_permutation.GetArray()) {
- if (!x.IsInt64()) {
- return Status::Invalid("permutation must contain integers, got ",
- internal::JsonTypeName(x));
+ std::vector<std::string> dim_names;
+
+ bool has_shape = false;
+
+ for (auto field_result : object) {
+ ARROW_ASSIGN_OR_RAISE(auto field, internal::ResolveSimdjsonResult(
+ field_result, "Failed to iterate
JSON object"));
+
+ ARROW_ASSIGN_OR_RAISE(
+ auto key, internal::ResolveSimdjsonResult(field.unescaped_key(),
+ "Failed to get JSON object
key"));
+
+ auto value = field.value();
+
+ if (key == "shape") {
+ has_shape = true;
+
+ ARROW_ASSIGN_OR_RAISE(auto type, internal::ResolveSimdjsonResult(
+ value.type(), "Invalid serialized
JSON data"));
+
+ if (type == simdjson::ondemand::json_type::unknown) {
+ return Status::Invalid("Invalid serialized JSON data: ",
serialized_data);
+ }
+
+ if (type != simdjson::ondemand::json_type::array) {
+ return Status::Invalid("shape must be an array, got ",
+ internal::JsonTypeName(type));
+ }
+
+ ARROW_ASSIGN_OR_RAISE(auto array,
+ internal::ResolveSimdjsonResult(value.get_array(),
+ "Failed to get
shape array"));
+
+ for (auto element_result : array) {
+ ARROW_ASSIGN_OR_RAISE(auto element,
+ internal::ResolveSimdjsonResult(
+ element_result, "Failed to iterate shape
array"));
+
+ ARROW_ASSIGN_OR_RAISE(
+ auto element_type,
+ internal::ResolveSimdjsonResult(
+ element.type(), "Failed to determine shape element JSON
type"));
+
+ if (element_type != simdjson::ondemand::json_type::number) {
+ return Status::Invalid("shape must contain integers, got ",
+ internal::JsonTypeName(element_type));
+ }
+
+ ARROW_ASSIGN_OR_RAISE(
+ auto number_type,
+ internal::ResolveSimdjsonResult(element.get_number_type(),
+ "Failed to determine shape number
type"));
+
+ if (number_type != simdjson::ondemand::number_type::signed_integer) {
+ return Status::Invalid("shape must contain integers, got number");
+ }
+
+ ARROW_ASSIGN_OR_RAISE(
+ auto number, internal::ResolveSimdjsonResult(element.get_int64(),
+ "Failed to get shape
integer"));
+
+ shape.emplace_back(number);
+ }
+
+ } else if (key == "permutation") {
+ ARROW_ASSIGN_OR_RAISE(auto type, internal::ResolveSimdjsonResult(
+ value.type(), "Invalid serialized
JSON data"));
+
+ if (type == simdjson::ondemand::json_type::unknown) {
+ return Status::Invalid("Invalid serialized JSON data: ",
serialized_data);
+ }
+
+ if (type != simdjson::ondemand::json_type::array) {
+ return Status::Invalid("permutation must be an array, got ",
+ internal::JsonTypeName(type));
+ }
+
+ ARROW_ASSIGN_OR_RAISE(
+ auto array, internal::ResolveSimdjsonResult(value.get_array(),
+ "Failed to get
permutation array"));
+
+ for (auto element_result : array) {
+ ARROW_ASSIGN_OR_RAISE(auto element,
+ internal::ResolveSimdjsonResult(
+ element_result, "Failed to iterate
permutation array"));
+
+ ARROW_ASSIGN_OR_RAISE(
+ auto element_type,
+ internal::ResolveSimdjsonResult(
+ element.type(), "Failed to determine permutation element JSON
type"));
+
+ if (element_type != simdjson::ondemand::json_type::number) {
+ return Status::Invalid("permutation must contain integers, got ",
+ internal::JsonTypeName(element_type));
+ }
+
+ ARROW_ASSIGN_OR_RAISE(auto number_type,
+ internal::ResolveSimdjsonResult(
+ element.get_number_type(),
+ "Failed to determine permutation number
type"));
+
+ if (number_type != simdjson::ondemand::number_type::signed_integer) {
+ return Status::Invalid("permutation must contain integers, got
number");
+ }
+
+ ARROW_ASSIGN_OR_RAISE(
+ auto number, internal::ResolveSimdjsonResult(
+ element.get_int64(), "Failed to get permutation
integer"));
+
+ permutation.emplace_back(number);
+ }
+
+ } else if (key == "dim_names") {
+ ARROW_ASSIGN_OR_RAISE(auto type, internal::ResolveSimdjsonResult(
+ value.type(), "Invalid serialized
JSON data"));
+
+ if (type == simdjson::ondemand::json_type::unknown) {
+ return Status::Invalid("Invalid serialized JSON data: ",
serialized_data);
+ }
Review Comment:
I think that `shape must be an array, got unknown` is also a good error
message for invalid JSON.
We may want to use `shape must be an array: {"shape":(3,4)}` instead for
easy to debug.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
