Reranko05 commented on code in PR #51396: URL: https://github.com/apache/arrow/pull/51396#discussion_r4381159332
########## cpp/examples/arrow/CMakeLists.txt: ########## @@ -17,9 +17,9 @@ add_arrow_example(row_wise_conversion_example) -if(ARROW_WITH_RAPIDJSON) - add_arrow_example(rapidjson_row_converter EXTRA_LINK_LIBS RapidJSON) - add_arrow_example(from_json_string_example EXTRA_LINK_LIBS RapidJSON) +if(ARROW_JSON) + add_arrow_example(simdjson_row_converter EXTRA_LINK_LIBS arrow::simdjson) + add_arrow_example(from_json_string_example EXTRA_LINK_LIBS arrow::simdjson) Review Comment: Addressed. ########## cpp/examples/arrow/simdjson_row_converter.cc: ########## @@ -0,0 +1,589 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <arrow/api.h> +#include <arrow/result.h> +#include <arrow/table_builder.h> +#include <arrow/type_traits.h> +#include <arrow/util/iterator.h> +#include <arrow/visit_array_inline.h> +#include <arrow/visit_type_inline.h> + +#include <simdjson.h> + +#include <cmath> +#include <cstdint> +#include <cstdio> +#include <cstdlib> +#include <iostream> +#include <limits> +#include <locale> +#include <memory> +#include <optional> +#include <sstream> +#include <string> +#include <string_view> +#include <type_traits> +#include <utility> +#include <vector> + +// Transforming dynamic row data into Arrow data +// When building connectors to other data systems, it's common to receive data in +// row-based structures. While the row_wise_conversion_example.cc shows how to +// handle this conversion for fixed schemas, this example demonstrates how to +// convert between row-based JSON data and Arrow data for a useful subset of +// nested Arrow schemas. +// +// As an example, this conversion is between JSON strings and Arrow tables. +// +// We use the following helpers and patterns here: +// * simdjson's DOM API for parsing JSON rows +// * arrow::RecordBatchBuilder for constructing Arrow arrays from row data +// * arrow::TableBatchReader and Arrow iterators for converting Arrow tables back +// into row-based JSON data +// * arrow::VisitTypeInline and arrow::VisitArrayInline for implementing a visitor +// pattern with Arrow to handle different types and array types +// * arrow::enable_if_primitive_ctype to create a template method that handles +// conversion for Arrow types that have corresponding C types (bool, integer, +// float). + +// Convert a simdjson result into an arrow::Result, so that errors are reported +// as arrow::Status rather than as exceptions. +template <typename T> +arrow::Result<T> Unwrap(simdjson::simdjson_result<T> result, const char* what) { + T value{}; + simdjson::error_code error = std::move(result).get(value); + if (error) { + return arrow::Status::Invalid(what, ": ", simdjson::error_message(error)); + } + return value; +} + +// Look up a field of a JSON object. Returns std::nullopt if the field is missing. +arrow::Result<std::optional<simdjson::dom::element>> GetField( + simdjson::dom::object object, std::string_view name) { + simdjson::dom::element value; + simdjson::error_code error = object.at_key(name).get(value); + if (error == simdjson::NO_SUCH_FIELD) { + return std::nullopt; + } + if (error) { + return arrow::Status::Invalid("Failed to get JSON field '", name, + "': ", simdjson::error_message(error)); + } + return value; +} + +// A JSON value, or std::nullopt if it is missing. +using JsonValue = std::optional<simdjson::dom::element>; + +bool IsNull(const JsonValue& value) { return !value.has_value() || value->is_null(); } + +// Collect the value of a field from each of the given JSON objects. +// Missing fields, and fields of null objects, are returned as std::nullopt. +arrow::Result<std::vector<JsonValue>> GetChildValues( + const std::vector<JsonValue>& objects, std::string_view name) { + std::vector<JsonValue> children; + children.reserve(objects.size()); + for (const JsonValue& value : objects) { + if (IsNull(value)) { + children.push_back(std::nullopt); + continue; + } + ARROW_ASSIGN_OR_RAISE(auto object, + Unwrap(value->get_object(), "Failed to get JSON object")); + ARROW_ASSIGN_OR_RAISE(auto child, GetField(object, name)); + children.push_back(child); + } + return children; +} + +/// \brief Visitor that appends the JSON values of one column to an Arrow builder. +/// +/// Implements Visit() methods for each Arrow type; each one appends one value per row. +class JsonValueConverter { + public: + /// \brief Append `values` (one per row) to `builder`, according to `type`. + arrow::Status Convert(const arrow::DataType& type, arrow::ArrayBuilder* builder, + const std::vector<JsonValue>& values) { + builder_ = builder; + values_ = &values; + return arrow::VisitTypeInline(type, this); + } + + // Default implementation + arrow::Status Visit(const arrow::DataType& type) { + return arrow::Status::NotImplemented( + "Cannot convert json value to Arrow array of type ", type.ToString()); + } + + arrow::Status Visit(const arrow::Int64Type& type) { + auto* builder = static_cast<arrow::Int64Builder*>(builder_); + for (const JsonValue& value : *values_) { + if (IsNull(value)) { + ARROW_RETURN_NOT_OK(builder->AppendNull()); + } else if (value->is_uint64()) { + ARROW_ASSIGN_OR_RAISE(auto number, + Unwrap(value->get_uint64(), "Failed to get JSON uint64")); + if (number > static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) { + return arrow::Status::Invalid("JSON uint64 value does not fit in int64"); + } + ARROW_RETURN_NOT_OK(builder->Append(static_cast<int64_t>(number))); + } else { + ARROW_ASSIGN_OR_RAISE(auto number, + Unwrap(value->get_int64(), "Failed to get JSON int64")); + ARROW_RETURN_NOT_OK(builder->Append(number)); + } + } + return arrow::Status::OK(); + } + + arrow::Status Visit(const arrow::DoubleType& type) { + auto* builder = static_cast<arrow::DoubleBuilder*>(builder_); + for (const JsonValue& value : *values_) { + if (IsNull(value)) { + ARROW_RETURN_NOT_OK(builder->AppendNull()); + } else { + ARROW_ASSIGN_OR_RAISE(auto number, + Unwrap(value->get_double(), "Failed to get JSON double")); + ARROW_RETURN_NOT_OK(builder->Append(number)); + } + } + return arrow::Status::OK(); + } + + arrow::Status Visit(const arrow::StringType& type) { + auto* builder = static_cast<arrow::StringBuilder*>(builder_); + for (const JsonValue& value : *values_) { + if (IsNull(value)) { + ARROW_RETURN_NOT_OK(builder->AppendNull()); + } else { + ARROW_ASSIGN_OR_RAISE(auto string, + Unwrap(value->get_string(), "Failed to get JSON string")); + ARROW_RETURN_NOT_OK(builder->Append(string)); + } + } + return arrow::Status::OK(); + } + + arrow::Status Visit(const arrow::BooleanType& type) { + auto* builder = static_cast<arrow::BooleanBuilder*>(builder_); + for (const JsonValue& value : *values_) { + if (IsNull(value)) { + ARROW_RETURN_NOT_OK(builder->AppendNull()); + } else { + ARROW_ASSIGN_OR_RAISE(auto boolean, + Unwrap(value->get_bool(), "Failed to get JSON boolean")); + ARROW_RETURN_NOT_OK(builder->Append(boolean)); + } + } + return arrow::Status::OK(); + } + + arrow::Status Visit(const arrow::StructType& type) { + auto* builder = static_cast<arrow::StructBuilder*>(builder_); + + // Convert each child column recursively. + for (int i = 0; i < type.num_fields(); ++i) { + std::shared_ptr<arrow::Field> child_field = type.field(i); + ARROW_ASSIGN_OR_RAISE(auto child_values, + GetChildValues(*values_, child_field->name())); + JsonValueConverter child_converter; + ARROW_RETURN_NOT_OK(child_converter.Convert( + *child_field->type(), builder->child_builder(i).get(), child_values)); + } + + // Make null bitmap + for (const JsonValue& value : *values_) { + ARROW_RETURN_NOT_OK(builder->Append(!IsNull(value))); + } + return arrow::Status::OK(); + } + + arrow::Status Visit(const arrow::ListType& type) { + auto* builder = static_cast<arrow::ListBuilder*>(builder_); + + // Values and offsets need to be interleaved in ListBuilder, so first collect the + // elements of all the lists and convert them into a temporary array. + std::vector<JsonValue> child_values; + std::vector<int64_t> sizes; + for (const JsonValue& value : *values_) { + int64_t size = 0; + if (!IsNull(value)) { + ARROW_ASSIGN_OR_RAISE(auto array, + Unwrap(value->get_array(), "Failed to get JSON array")); + for (simdjson::dom::element element : array) { + child_values.push_back(element); + ++size; + } + } + sizes.push_back(size); + } + + std::unique_ptr<arrow::ArrayBuilder> tmp_value_builder; + ARROW_ASSIGN_OR_RAISE(tmp_value_builder, + arrow::MakeBuilder(builder->value_builder()->type())); + JsonValueConverter child_converter; + ARROW_RETURN_NOT_OK(child_converter.Convert(*type.value_type(), + tmp_value_builder.get(), child_values)); + + std::shared_ptr<arrow::Array> values_array; + ARROW_RETURN_NOT_OK(tmp_value_builder->Finish(&values_array)); + arrow::ArraySpan values_span(*values_array->data()); + + arrow::ArrayBuilder* value_builder = builder->value_builder(); + int64_t offset = 0; + for (size_t i = 0; i < values_->size(); ++i) { + ARROW_RETURN_NOT_OK(builder->Append(!IsNull((*values_)[i]))); + if (sizes[i] > 0) { + ARROW_RETURN_NOT_OK( + value_builder->AppendArraySlice(values_span, offset, sizes[i])); + offset += sizes[i]; + } + } + return arrow::Status::OK(); + } + + private: + arrow::ArrayBuilder* builder_ = nullptr; + const std::vector<JsonValue>* values_ = nullptr; +}; // JsonValueConverter + +arrow::Result<std::shared_ptr<arrow::RecordBatch>> ConvertToRecordBatch( + const std::vector<std::string>& rows, std::shared_ptr<arrow::Schema> schema) { + // RecordBatchBuilder will create array builders for us for each field in our + // schema. By passing the number of output rows (`rows.size()`) we can + // pre-allocate the correct size of arrays, except of course in the case of + // string, byte, and list arrays, which have dynamic lengths. + std::unique_ptr<arrow::RecordBatchBuilder> batch_builder; + ARROW_ASSIGN_OR_RAISE( + batch_builder, + arrow::RecordBatchBuilder::Make(schema, arrow::default_memory_pool(), rows.size())); + + // The converter below visits all rows once per column, but a DOM element is only + // valid until its parser is used again. So each row gets its own parser. + std::vector<simdjson::dom::parser> parsers(rows.size()); + std::vector<JsonValue> row_values; + row_values.reserve(rows.size()); + for (size_t i = 0; i < rows.size(); ++i) { + ARROW_ASSIGN_OR_RAISE(auto row, + Unwrap(parsers[i].parse(rows[i]), "Failed to parse JSON row")); + row_values.push_back(row); + } Review Comment: Addressed. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
