Reranko05 commented on code in PR #51396:
URL: https://github.com/apache/arrow/pull/51396#discussion_r4381165773
##########
cpp/examples/arrow/simdjson_row_converter.cc:
##########
@@ -75,101 +88,180 @@ arrow::Result<std::optional<simdjson::dom::element>>
GetField(
return value;
}
-// Append a JSON value to an Arrow builder according to the expected Arrow
type.
-// This example only handles the types used by the example schema; extend this
-// switch when adapting it to schemas with additional Arrow types.
-arrow::Status AppendJsonValue(simdjson::dom::element value,
- const std::shared_ptr<arrow::DataType>& type,
- arrow::ArrayBuilder* builder);
-
-arrow::Status AppendJsonStruct(simdjson::dom::element value,
- const arrow::StructType& type,
- arrow::StructBuilder* builder) {
- ARROW_ASSIGN_OR_RAISE(auto object,
- Unwrap(value.get_object(), "Failed to get JSON
object"));
-
- for (int i = 0; i < type.num_fields(); ++i) {
- const auto& field = type.field(i);
-
- ARROW_ASSIGN_OR_RAISE(auto child, GetField(object, field->name()));
-
- if (!child.has_value()) {
- ARROW_RETURN_NOT_OK(builder->child_builder(i)->AppendNull());
- } else {
- ARROW_RETURN_NOT_OK(
- AppendJsonValue(*child, field->type(),
builder->child_builder(i).get()));
+// A JSON value, or std::nullopt if it is missing.
+using JsonValue = std::optional<simdjson::dom::element>;
+
+bool IsNull(const JsonValue& value) { return !value.has_value() ||
value->is_null(); }
+
+// Collect the value of a field from each of the given JSON objects.
+// Missing fields, and fields of null objects, are returned as std::nullopt.
+arrow::Result<std::vector<JsonValue>> GetChildValues(
+ const std::vector<JsonValue>& objects, std::string_view name) {
+ std::vector<JsonValue> children;
+ children.reserve(objects.size());
+ for (const JsonValue& value : objects) {
+ if (IsNull(value)) {
+ children.push_back(std::nullopt);
+ continue;
}
+ ARROW_ASSIGN_OR_RAISE(auto object,
+ Unwrap(value->get_object(), "Failed to get JSON
object"));
+ ARROW_ASSIGN_OR_RAISE(auto child, GetField(object, name));
+ children.push_back(child);
}
-
- return builder->Append();
+ return children;
}
-arrow::Status AppendJsonList(simdjson::dom::element value, const
arrow::ListType& type,
- arrow::ListBuilder* builder) {
- ARROW_ASSIGN_OR_RAISE(auto array,
- Unwrap(value.get_array(), "Failed to get JSON array"));
-
- ARROW_RETURN_NOT_OK(builder->Append());
-
- for (simdjson::dom::element element : array) {
- ARROW_RETURN_NOT_OK(
- AppendJsonValue(element, type.value_field()->type(),
builder->value_builder()));
+/// \brief Visitor that appends the JSON values of one column to an Arrow
builder.
+///
+/// Implements Visit() methods for each Arrow type; each one appends one value
per row.
+class JsonValueConverter {
+ public:
+ /// \brief Append `values` (one per row) to `builder`, according to `type`.
+ arrow::Status Convert(const arrow::DataType& type, arrow::ArrayBuilder*
builder,
+ const std::vector<JsonValue>& values) {
+ builder_ = builder;
+ values_ = &values;
+ return arrow::VisitTypeInline(type, this);
}
- return arrow::Status::OK();
-}
+ // Default implementation
+ arrow::Status Visit(const arrow::DataType& type) {
+ return arrow::Status::NotImplemented(
+ "Cannot convert json value to Arrow array of type ", type.ToString());
+ }
-arrow::Status AppendJsonValue(simdjson::dom::element value,
- const std::shared_ptr<arrow::DataType>& type,
- arrow::ArrayBuilder* builder) {
- if (value.is_null()) {
- return builder->AppendNull();
+ arrow::Status Visit(const arrow::Int64Type& type) {
+ auto* builder = static_cast<arrow::Int64Builder*>(builder_);
+ for (const JsonValue& value : *values_) {
+ if (IsNull(value)) {
+ ARROW_RETURN_NOT_OK(builder->AppendNull());
+ } else if (value->is_uint64()) {
+ ARROW_ASSIGN_OR_RAISE(auto number,
+ Unwrap(value->get_uint64(), "Failed to get JSON
uint64"));
+ if (number >
static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) {
+ return arrow::Status::Invalid("JSON uint64 value does not fit in
int64");
+ }
+ ARROW_RETURN_NOT_OK(builder->Append(static_cast<int64_t>(number)));
+ } else {
+ ARROW_ASSIGN_OR_RAISE(auto number,
+ Unwrap(value->get_int64(), "Failed to get JSON
int64"));
+ ARROW_RETURN_NOT_OK(builder->Append(number));
+ }
+ }
+ return arrow::Status::OK();
}
- switch (type->id()) {
- case arrow::Type::INT64: {
- if (value.is_uint64()) {
+ arrow::Status Visit(const arrow::DoubleType& type) {
+ auto* builder = static_cast<arrow::DoubleBuilder*>(builder_);
+ for (const JsonValue& value : *values_) {
+ if (IsNull(value)) {
+ ARROW_RETURN_NOT_OK(builder->AppendNull());
+ } else {
ARROW_ASSIGN_OR_RAISE(auto number,
- Unwrap(value.get_uint64(), "Failed to get JSON
uint64"));
- return static_cast<arrow::Int64Builder*>(builder)->Append(
- static_cast<int64_t>(number));
+ Unwrap(value->get_double(), "Failed to get JSON
double"));
+ ARROW_RETURN_NOT_OK(builder->Append(number));
}
- ARROW_ASSIGN_OR_RAISE(auto number,
- Unwrap(value.get_int64(), "Failed to get JSON
int64"));
- return static_cast<arrow::Int64Builder*>(builder)->Append(number);
}
+ return arrow::Status::OK();
+ }
- case arrow::Type::DOUBLE: {
- ARROW_ASSIGN_OR_RAISE(auto number,
- Unwrap(value.get_double(), "Failed to get JSON
double"));
- return static_cast<arrow::DoubleBuilder*>(builder)->Append(number);
+ arrow::Status Visit(const arrow::StringType& type) {
+ auto* builder = static_cast<arrow::StringBuilder*>(builder_);
+ for (const JsonValue& value : *values_) {
+ if (IsNull(value)) {
+ ARROW_RETURN_NOT_OK(builder->AppendNull());
+ } else {
+ ARROW_ASSIGN_OR_RAISE(auto string,
+ Unwrap(value->get_string(), "Failed to get JSON
string"));
+ ARROW_RETURN_NOT_OK(builder->Append(string));
+ }
}
+ return arrow::Status::OK();
+ }
- case arrow::Type::STRING: {
- ARROW_ASSIGN_OR_RAISE(auto string,
- Unwrap(value.get_string(), "Failed to get JSON
string"));
- return static_cast<arrow::StringBuilder*>(builder)->Append(string);
+ arrow::Status Visit(const arrow::BooleanType& type) {
+ auto* builder = static_cast<arrow::BooleanBuilder*>(builder_);
+ for (const JsonValue& value : *values_) {
+ if (IsNull(value)) {
+ ARROW_RETURN_NOT_OK(builder->AppendNull());
+ } else {
+ ARROW_ASSIGN_OR_RAISE(auto boolean,
+ Unwrap(value->get_bool(), "Failed to get JSON
boolean"));
+ ARROW_RETURN_NOT_OK(builder->Append(boolean));
+ }
}
+ return arrow::Status::OK();
+ }
- case arrow::Type::BOOL: {
- ARROW_ASSIGN_OR_RAISE(auto boolean,
- Unwrap(value.get_bool(), "Failed to get JSON
boolean"));
- return static_cast<arrow::BooleanBuilder*>(builder)->Append(boolean);
+ arrow::Status Visit(const arrow::StructType& type) {
+ auto* builder = static_cast<arrow::StructBuilder*>(builder_);
+
+ // Convert each child column recursively.
+ for (int i = 0; i < type.num_fields(); ++i) {
+ std::shared_ptr<arrow::Field> child_field = type.field(i);
+ ARROW_ASSIGN_OR_RAISE(auto child_values,
+ GetChildValues(*values_, child_field->name()));
+ JsonValueConverter child_converter;
+ ARROW_RETURN_NOT_OK(child_converter.Convert(
+ *child_field->type(), builder->child_builder(i).get(),
child_values));
}
Review Comment:
Good point! Addressed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
