This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 4d1e84b641 GH-50724: [C++] Add `JsonWriter::WriteValue` for simdjson
values (#50725)
4d1e84b641 is described below
commit 4d1e84b6418387d987d0dbd67ae0ef02d2c41040
Author: Aaditya Srinivasan <[email protected]>
AuthorDate: Mon Aug 3 03:16:58 2026 +0530
GH-50724: [C++] Add `JsonWriter::WriteValue` for simdjson values (#50725)
### Rationale for this change
This PR continues the simdjson migration by adding support for serializing
`simdjson::ondemand::value` directly with `JsonWriter`.
This provides a reusable API for future migration work and avoids requiring
callers to implement their own recursive serialization logic. It also
introduces a shared helper for dispatching `simdjson::ondemand::value` based on
its JSON type, reducing duplicated type dispatch and extraction logic.
### What changes are included in this PR?
* Add `JsonWriter::WriteValue(simdjson::ondemand::value)`.
* Add `VisitJsonValue` to centralize JSON type dispatch and `simdjson`
value extraction.
* Recursively serialize:
* objects
* arrays
* strings
* booleans
* null values
* numeric values
* Add unit tests covering:
* simple objects
* nested objects
* objects containing arrays
* complex nested values
* empty objects
* Use `simdjson::ondemand::document::get_value()` in tests to obtain the
root `ondemand::value` before serialization.
### Are these changes tested?
Yes.
Added unit tests for `JsonWriter::WriteValue` covering the supported JSON
value types and nested structures.
### Are there any user-facing changes?
No.
Closes: #50724
* GitHub Issue: #50724
Authored-by: Aaditya Srinivasan <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
cpp/src/arrow/json/from_string.cc | 203 +++++---------------
cpp/src/arrow/json/json_writer_internal.cc | 82 ++++++++
cpp/src/arrow/json/json_writer_internal.h | 3 +
cpp/src/arrow/json/json_writer_internal_test.cc | 120 ++++++++++++
cpp/src/arrow/util/simdjson_internal.h | 244 ++++++++++++++++++++++++
5 files changed, 497 insertions(+), 155 deletions(-)
diff --git a/cpp/src/arrow/json/from_string.cc
b/cpp/src/arrow/json/from_string.cc
index 5694fbde4c..c9d9106671 100644
--- a/cpp/src/arrow/json/from_string.cc
+++ b/cpp/src/arrow/json/from_string.cc
@@ -47,6 +47,7 @@
#include "arrow/util/decimal.h"
#include "arrow/util/float16.h"
#include "arrow/util/logging_internal.h"
+#include "arrow/util/simdjson_internal.h"
#include "arrow/util/unreachable.h"
#include "arrow/util/value_parsing.h"
@@ -66,116 +67,6 @@ using ::arrow::internal::checked_pointer_cast;
namespace {
-const char* JsonTypeName(sj::json_type type) {
- switch (type) {
- case sj::json_type::array:
- return "array";
- case sj::json_type::object:
- return "object";
- case sj::json_type::number:
- return "number";
- case sj::json_type::string:
- return "string";
- case sj::json_type::boolean:
- return "boolean";
- case sj::json_type::null:
- return "null";
- default:
- return "unknown";
- }
-}
-
-// Empty struct to represent the type of a simdjson null value
-struct SimdjsonNull {};
-
-template <typename T>
-struct JsonTypeNameOf;
-
-template <>
-struct JsonTypeNameOf<sj::array> {
- static constexpr const char* kValue = "array";
-};
-template <>
-struct JsonTypeNameOf<sj::object> {
- static constexpr const char* kValue = "object";
-};
-template <>
-struct JsonTypeNameOf<std::string_view> {
- static constexpr const char* kValue = "string";
-};
-template <>
-struct JsonTypeNameOf<bool> {
- static constexpr const char* kValue = "boolean";
-};
-template <>
-struct JsonTypeNameOf<SimdjsonNull> {
- static constexpr const char* kValue = "null";
-};
-template <>
-struct JsonTypeNameOf<int64_t> {
- static constexpr const char* kValue = "number";
-};
-template <>
-struct JsonTypeNameOf<uint64_t> {
- static constexpr const char* kValue = "number";
-};
-template <>
-struct JsonTypeNameOf<double> {
- static constexpr const char* kValue = "number";
-};
-
-template <typename T>
-constexpr const char* JsonTypeName() {
- return JsonTypeNameOf<T>::kValue;
-}
-
-template <typename SimdjsonValueType>
-Result<SimdjsonValueType> GetJsonAs(sj::value& value) {
- SimdjsonValueType typed_value{};
- simdjson::error_code error_code;
- if constexpr (std::is_same_v<SimdjsonValueType, SimdjsonNull>) {
- // simdjson has no get<>() for null; probe it explicitly
- bool is_null;
- error_code = value.is_null().get(is_null);
- if (error_code == simdjson::SUCCESS && !is_null) {
- error_code = simdjson::INCORRECT_TYPE;
- }
- } else {
- error_code = value.get(typed_value);
- }
- if (error_code != simdjson::SUCCESS) {
- sj::json_type json_type;
- if (value.type().get(json_type) != simdjson::SUCCESS) {
- return Status::Invalid("Expected ", JsonTypeName<SimdjsonValueType>(),
- " or null, got malformed JSON value");
- }
- return Status::Invalid("Expected ", JsonTypeName<SimdjsonValueType>(),
- " or null, got JSON type ",
JsonTypeName(json_type));
- }
- return typed_value;
-}
-
-template <typename SimdjsonValueType>
-Result<SimdjsonValueType> GetJsonResult(
- simdjson::simdjson_result<SimdjsonValueType> element, std::string_view
error) {
- SimdjsonValueType typed_value;
- if (auto error_code = std::move(element).get(typed_value);
- error_code != simdjson::SUCCESS) {
- return Status::Invalid(error, simdjson::error_message(error_code));
- }
- return typed_value;
-}
-
-// Result<bool> because peeking the nonRootScalar can fail (parsed lazily)
-Result<bool> IsJsonNull(sj::value& value) {
- bool is_null;
- if (auto error_code = value.is_null().get(is_null); error_code !=
simdjson::SUCCESS) {
- return Status::Invalid("Error checking for JSON null: ",
- simdjson::error_message(error_code));
- }
- return is_null;
-}
-
class JSONConverter {
public:
virtual ~JSONConverter() = default;
@@ -215,7 +106,7 @@ class ConcreteConverter : public JSONConverter {
int32_t num_elements = 0;
for (auto element : json_array) {
ARROW_ASSIGN_OR_RAISE(auto value,
- GetJsonResult<sj::value>(
+ internal::GetSimdjsonResult<sj::value>(
element, "Could not iterate elements of JSON
array: "));
RETURN_NOT_OK(self->AppendValue(value));
num_elements++;
@@ -251,7 +142,7 @@ class NullConverter final : public
ConcreteConverter<NullConverter> {
}
Status AppendValue(sj::value& json_obj) override {
- ARROW_RETURN_NOT_OK(GetJsonAs<SimdjsonNull>(json_obj));
+ ARROW_RETURN_NOT_OK(internal::GetJsonAs<internal::SimdjsonNull>(json_obj));
return AppendNull();
}
@@ -272,7 +163,7 @@ class BooleanConverter final : public
ConcreteConverter<BooleanConverter> {
}
Status AppendValue(sj::value& json_obj) override {
- ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj));
+ ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj));
if (is_null) {
return AppendNull();
}
@@ -280,7 +171,7 @@ class BooleanConverter final : public
ConcreteConverter<BooleanConverter> {
if (json_obj.get(int_value) == simdjson::SUCCESS) {
return builder_->Append(int_value != 0);
}
- ARROW_ASSIGN_OR_RAISE(bool bool_value, GetJsonAs<bool>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(bool bool_value,
internal::GetJsonAs<bool>(json_obj));
return builder_->Append(bool_value);
}
@@ -299,7 +190,7 @@ enable_if_physical_signed_integer<T, Status>
ConvertNumber(sj::value& json_obj,
const DataType&
type,
typename T::c_type*
out) {
*out = static_cast<typename T::c_type>(0);
- ARROW_ASSIGN_OR_RAISE(int64_t v64, GetJsonAs<int64_t>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(int64_t v64, internal::GetJsonAs<int64_t>(json_obj));
*out = static_cast<typename T::c_type>(v64);
if (*out == v64) {
return Status::OK();
@@ -314,7 +205,7 @@ enable_if_unsigned_integer<T, Status>
ConvertNumber(sj::value& json_obj,
const DataType& type,
typename T::c_type* out) {
*out = static_cast<typename T::c_type>(0);
- ARROW_ASSIGN_OR_RAISE(uint64_t v64, GetJsonAs<uint64_t>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(uint64_t v64, internal::GetJsonAs<uint64_t>(json_obj));
*out = static_cast<typename T::c_type>(v64);
if (*out == v64) {
return Status::OK();
@@ -356,7 +247,7 @@ enable_if_half_float<T, Status> ConvertNumber(sj::value&
json_obj, const DataTyp
*out = Float16(f64.value()).bits();
return Status::OK();
}
- ARROW_ASSIGN_OR_RAISE(auto f64, GetJsonAs<double>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(auto f64, internal::GetJsonAs<double>(json_obj));
*out = Float16(f64).bits();
return arrow::Status::OK();
}
@@ -371,7 +262,7 @@ enable_if_physical_floating_point<T, Status>
ConvertNumber(sj::value& json_obj,
*out = static_cast<typename T::c_type>(f64.value());
return Status::OK();
}
- ARROW_ASSIGN_OR_RAISE(auto f64, GetJsonAs<double>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(auto f64, internal::GetJsonAs<double>(json_obj));
*out = static_cast<typename T::c_type>(f64);
return arrow::Status::OK();
}
@@ -395,9 +286,9 @@ Status ProcessJsonArrayElements(
" elements, had ", index);
}
- ARROW_ASSIGN_OR_RAISE(
- sj::value element,
- GetJsonResult<sj::value>(*it, "Could not iterate elements of JSON
array: "));
+ ARROW_ASSIGN_OR_RAISE(sj::value element,
+ internal::GetSimdjsonResult<sj::value>(
+ *it, "Could not iterate elements of JSON array:
"));
RETURN_NOT_OK(handler(element));
++it;
++index;
@@ -426,7 +317,7 @@ class IntegerConverter final
Status Init() override { return this->MakeConcreteBuilder(&builder_); }
Status AppendValue(sj::value& json_obj) override {
- ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj));
+ ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj));
if (is_null) {
return this->AppendNull();
}
@@ -454,7 +345,7 @@ class FloatConverter final : public
ConcreteConverter<FloatConverter<Type, Build
Status Init() override { return this->MakeConcreteBuilder(&builder_); }
Status AppendValue(sj::value& json_obj) override {
- ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj));
+ ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj));
if (is_null) {
return this->AppendNull();
}
@@ -485,11 +376,12 @@ class DecimalConverter final
Status Init() override { return this->MakeConcreteBuilder(&builder_); }
Status AppendValue(sj::value& json_obj) override {
- ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj));
+ ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj));
if (is_null) {
return this->AppendNull();
}
- ARROW_ASSIGN_OR_RAISE(auto string_value,
GetJsonAs<std::string_view>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(auto string_value,
+ internal::GetJsonAs<std::string_view>(json_obj));
int32_t precision, scale;
DecimalValue d;
RETURN_NOT_OK(DecimalValue::FromString(string_value, &d, &precision,
&scale));
@@ -528,7 +420,7 @@ class TimestampConverter final : public
ConcreteConverter<TimestampConverter> {
}
Status AppendValue(sj::value& json_obj) override {
- ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj));
+ ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj));
if (is_null) {
return this->AppendNull();
}
@@ -563,12 +455,12 @@ class DayTimeIntervalConverter final
}
Status AppendValue(sj::value& json_obj) override {
- ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj));
+ ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj));
if (is_null) {
return this->AppendNull();
}
- ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs<sj::array>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(auto array,
internal::GetJsonAs<sj::array>(json_obj));
DayTimeIntervalType::DayMilliseconds value;
RETURN_NOT_OK(ProcessJsonArrayElements<2>(
@@ -597,12 +489,12 @@ class MonthDayNanoIntervalConverter final
}
Status AppendValue(sj::value& json_obj) override {
- ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj));
+ ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj));
if (is_null) {
return this->AppendNull();
}
- ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs<sj::array>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(auto array,
internal::GetJsonAs<sj::array>(json_obj));
MonthDayNanoIntervalType::MonthDayNanos value;
RETURN_NOT_OK(ProcessJsonArrayElements<3>(
@@ -637,12 +529,12 @@ class StringConverter final
Status Init() override { return this->MakeConcreteBuilder(&builder_); }
Status AppendValue(sj::value& json_obj) override {
- ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj));
+ ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj));
if (is_null) {
return this->AppendNull();
}
- ARROW_ASSIGN_OR_RAISE(auto view, GetJsonAs<std::string_view>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(auto view,
internal::GetJsonAs<std::string_view>(json_obj));
return builder_->Append(view);
}
@@ -666,11 +558,11 @@ class FixedSizeBinaryConverter final
Status Init() override { return this->MakeConcreteBuilder(&builder_); }
Status AppendValue(sj::value& json_obj) override {
- ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj));
+ ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj));
if (is_null) {
return this->AppendNull();
}
- ARROW_ASSIGN_OR_RAISE(auto view, GetJsonAs<std::string_view>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(auto view,
internal::GetJsonAs<std::string_view>(json_obj));
if (view.length() != static_cast<size_t>(builder_->byte_width())) {
std::stringstream ss;
ss << "Invalid string length " << view.length() << " in JSON input for "
@@ -710,11 +602,11 @@ class VarLengthListLikeConverter final
}
Status AppendValue(sj::value& json_obj) override {
- ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj));
+ ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj));
if (is_null) {
return this->AppendNull();
}
- ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs<sj::array>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(auto array,
internal::GetJsonAs<sj::array>(json_obj));
size_t num_elements;
if (array.count_elements().get(num_elements) != simdjson::SUCCESS) {
return Status::Invalid("Malformed JSON array for type ",
this->type_->ToString());
@@ -750,24 +642,25 @@ class MapConverter final : public
ConcreteConverter<MapConverter> {
}
Status AppendValue(sj::value& json_obj) override {
- ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj));
+ ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj));
if (is_null) {
return this->AppendNull();
}
RETURN_NOT_OK(builder_->Append());
- ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs<sj::array>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(auto array,
internal::GetJsonAs<sj::array>(json_obj));
for (auto json_pair_result : array) {
ARROW_ASSIGN_OR_RAISE(
auto json_pair,
- GetJsonResult<sj::value>(json_pair_result,
- "Could not iterate elements of JSON array:
"));
- ARROW_ASSIGN_OR_RAISE(auto json_pair_array,
GetJsonAs<sj::array>(json_pair));
+ internal::GetSimdjsonResult<sj::value>(
+ json_pair_result, "Could not iterate elements of JSON array: "));
+ ARROW_ASSIGN_OR_RAISE(auto json_pair_array,
+ internal::GetJsonAs<sj::array>(json_pair));
RETURN_NOT_OK(ProcessJsonArrayElements<2>(
json_pair_array, "key-item pair",
{[this](sj::value& key) {
- ARROW_ASSIGN_OR_RAISE(bool key_is_null, IsJsonNull(key));
+ ARROW_ASSIGN_OR_RAISE(bool key_is_null,
internal::IsJsonNull(key));
if (key_is_null) {
return Status::Invalid("null key is invalid");
}
@@ -803,13 +696,13 @@ class FixedSizeListConverter final : public
ConcreteConverter<FixedSizeListConve
}
Status AppendValue(sj::value& json_obj) override {
- ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj));
+ ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj));
if (is_null) {
return this->AppendNull();
}
RETURN_NOT_OK(builder_->Append());
// Extend the child converter with this JSON array
- ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs<sj::array>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(auto array,
internal::GetJsonAs<sj::array>(json_obj));
ARROW_ASSIGN_OR_RAISE(int32_t size, child_converter_->AppendValues(array));
if (size != list_size_) {
return Status::Invalid("incorrect list size ", size);
@@ -852,7 +745,7 @@ class StructConverter final : public
ConcreteConverter<StructConverter> {
// or an object mapping struct names to values (omitted struct members
// are mapped to null).
Status AppendValue(sj::value& json_obj) override {
- ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj));
+ ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj));
if (is_null) {
return this->AppendNull();
}
@@ -870,14 +763,14 @@ class StructConverter final : public
ConcreteConverter<StructConverter> {
size_t i = 0;
for (auto child : array) {
ARROW_ASSIGN_OR_RAISE(auto child_value,
- GetJsonResult<sj::value>(
+ internal::GetSimdjsonResult<sj::value>(
child, "Could not iterate elements of JSON
array: "));
RETURN_NOT_OK(child_converters_[i]->AppendValue(child_value));
++i;
}
return builder_->Append();
}
- ARROW_ASSIGN_OR_RAISE(auto object, GetJsonAs<sj::object>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(auto object,
internal::GetJsonAs<sj::object>(json_obj));
// Iterate the object fields in JSON order (the on-demand API is
// forward-only, so per-field lookups would be quadratic and would also
// compare against raw, still-escaped keys). Fields absent from the JSON
@@ -885,9 +778,9 @@ class StructConverter final : public
ConcreteConverter<StructConverter> {
auto num_fields = type_->num_fields();
std::vector<bool> field_seen(num_fields, false);
for (auto field_result : object) {
- ARROW_ASSIGN_OR_RAISE(
- auto field,
- GetJsonResult<sj::field>(field_result, "Error getting field of
object: "));
+ ARROW_ASSIGN_OR_RAISE(auto field,
+ internal::GetSimdjsonResult<sj::field>(
+ field_result, "Error getting field of object:
"));
std::string_view key;
if (field.unescaped_key(/*allow_replacement=*/false).get(key) !=
simdjson::SUCCESS) {
@@ -961,12 +854,12 @@ class UnionConverter final : public
ConcreteConverter<UnionConverter> {
// Append a JSON value that must be a 2-long array, containing the type_id
// and value of the UnionArray's slot.
Status AppendValue(sj::value& json_obj) override {
- ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj));
+ ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj));
if (is_null) {
return this->AppendNull();
}
- ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs<sj::array>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(auto array,
internal::GetJsonAs<sj::array>(json_obj));
int8_t id = 0;
std::shared_ptr<JSONConverter> child_converter;
@@ -974,7 +867,7 @@ class UnionConverter final : public
ConcreteConverter<UnionConverter> {
RETURN_NOT_OK(ProcessJsonArrayElements<2>(
array, "[type_id, value] pair",
{[this, &id, &child_converter](sj::value& id_elem) {
- ARROW_ASSIGN_OR_RAISE(auto id_value, GetJsonAs<int64_t>(id_elem));
+ ARROW_ASSIGN_OR_RAISE(auto id_value,
internal::GetJsonAs<int64_t>(id_elem));
id = static_cast<int8_t>(id_value);
auto child_num = type_id_to_child_num_[id];
if (child_num == -1) {
@@ -1156,7 +1049,7 @@ Result<std::shared_ptr<Array>> ArrayFromJSONString(const
std::shared_ptr<DataTyp
error_code != simdjson::SUCCESS) {
return Status::Invalid("JSON parse error: ",
simdjson::error_message(error_code));
}
- ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs<sj::array>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(auto array, internal::GetJsonAs<sj::array>(json_obj));
// The JSON document should be an array, append it
RETURN_NOT_OK(converter->AppendValues(array));
@@ -1226,7 +1119,7 @@ Result<std::shared_ptr<Scalar>> ScalarFromJSONString(
error_code != simdjson::SUCCESS) {
return Status::Invalid("JSON parse error: ",
simdjson::error_message(error_code));
}
- ARROW_ASSIGN_OR_RAISE(auto singleton_array, GetJsonAs<sj::array>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(auto singleton_array,
internal::GetJsonAs<sj::array>(json_obj));
ARROW_ASSIGN_OR_RAISE(int32_t num_elements,
converter->AppendValues(singleton_array));
if (num_elements != 1) {
diff --git a/cpp/src/arrow/json/json_writer_internal.cc
b/cpp/src/arrow/json/json_writer_internal.cc
index 446c7f06f4..5676949024 100644
--- a/cpp/src/arrow/json/json_writer_internal.cc
+++ b/cpp/src/arrow/json/json_writer_internal.cc
@@ -16,9 +16,12 @@
// under the License.
#include "arrow/json/json_writer_internal.h"
+#include "arrow/util/simdjson_internal.h"
namespace arrow::json {
+namespace sj = simdjson::ondemand;
+
void JsonWriter::StartObject() {
MaybeComma();
builder_.start_object();
@@ -96,6 +99,85 @@ void JsonWriter::Double(double value) {
needs_comma_ = true;
}
+Status JsonWriter::WriteValue(sj::value value) {
+ return internal::VisitJsonValue(
+ value,
+
+ [&](sj::object object) -> Status {
+ StartObject();
+
+ for (auto field : object) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto key, internal::GetSimdjsonResult(field.unescaped_key(),
+ "Failed to get object key:
"));
+
+ Key(key);
+
+ ARROW_ASSIGN_OR_RAISE(
+ auto field_value,
+ internal::GetSimdjsonResult(field.value(), "Failed to get object
value: "));
+
+ RETURN_NOT_OK(WriteValue(field_value));
+ }
+
+ EndObject();
+ return Status::OK();
+ },
+
+ [&](sj::array array) -> Status {
+ StartArray();
+
+ for (auto element : array) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto element_value,
+ internal::GetSimdjsonResult(element, "Failed to iterate JSON
array: "));
+
+ RETURN_NOT_OK(WriteValue(element_value));
+ }
+
+ EndArray();
+ return Status::OK();
+ },
+
+ [&](std::string_view string_value) -> Status {
+ String(string_value);
+ return Status::OK();
+ },
+
+ [&](bool bool_value) -> Status {
+ Bool(bool_value);
+ return Status::OK();
+ },
+
+ [&]() -> Status {
+ Null();
+ return Status::OK();
+ },
+
+ [&](int64_t value) -> Status {
+ Int64(value);
+ return Status::OK();
+ },
+
+ [&](uint64_t value) -> Status {
+ Uint64(value);
+ return Status::OK();
+ },
+
+ [&](double value) -> Status {
+ Double(value);
+ return Status::OK();
+ },
+
+ [&](sj::value value) -> Status {
+ ARROW_ASSIGN_OR_RAISE(auto raw_json,
+
internal::GetSimdjsonResult(simdjson::to_json_string(value),
+ "Failed to get raw
JSON: "));
+ RawValue(raw_json);
+ return Status::OK();
+ });
+}
+
void JsonWriter::Null() {
MaybeComma();
builder_.append_null();
diff --git a/cpp/src/arrow/json/json_writer_internal.h
b/cpp/src/arrow/json/json_writer_internal.h
index e6043bd7e5..4a8200f1df 100644
--- a/cpp/src/arrow/json/json_writer_internal.h
+++ b/cpp/src/arrow/json/json_writer_internal.h
@@ -23,6 +23,7 @@
#include <string_view>
#include "arrow/result.h"
+#include "arrow/status.h"
#include "arrow/util/visibility.h"
namespace arrow::json {
@@ -51,6 +52,8 @@ class ARROW_EXPORT JsonWriter {
void Double(double value);
+ Status WriteValue(simdjson::ondemand::value value);
+
void Null();
void StringField(std::string_view key, std::string_view value);
diff --git a/cpp/src/arrow/json/json_writer_internal_test.cc
b/cpp/src/arrow/json/json_writer_internal_test.cc
index abdf9be7af..7fc12e8adb 100644
--- a/cpp/src/arrow/json/json_writer_internal_test.cc
+++ b/cpp/src/arrow/json/json_writer_internal_test.cc
@@ -20,6 +20,8 @@
#include "arrow/json/json_writer_internal.h"
#include "arrow/testing/gtest_util.h"
+namespace sj = simdjson::ondemand;
+
namespace arrow::json {
TEST(JsonWriter, SimpleObject) {
@@ -170,4 +172,122 @@ TEST(JsonWriter, StringWithExplicitLength) {
EXPECT_EQ(json, R"({"value":"abc"})");
}
+TEST(JsonWriter, WriteValueSimpleObject) {
+ sj::parser parser;
+ std::string json_str = R"({"a":42,"b":"hello"})";
+ simdjson::padded_string json(json_str);
+
+ sj::document doc;
+ ASSERT_EQ(parser.iterate(json).get(doc), simdjson::SUCCESS);
+
+ sj::value value;
+ ASSERT_EQ(doc.get_value().get(value), simdjson::SUCCESS);
+
+ JsonWriter writer;
+ ASSERT_OK(writer.WriteValue(value));
+
+ ASSERT_OK_AND_ASSIGN(std::string_view out, writer.GetString());
+ EXPECT_EQ(out, R"({"a":42,"b":"hello"})");
+}
+
+TEST(JsonWriter, WriteValueNestedObject) {
+ sj::parser parser;
+ std::string json_str = R"({"child":{"x":true}})";
+ simdjson::padded_string json(json_str);
+
+ sj::document doc;
+ ASSERT_EQ(parser.iterate(json).get(doc), simdjson::SUCCESS);
+
+ sj::value value;
+ ASSERT_EQ(doc.get_value().get(value), simdjson::SUCCESS);
+
+ JsonWriter writer;
+ ASSERT_OK(writer.WriteValue(value));
+
+ ASSERT_OK_AND_ASSIGN(std::string_view out, writer.GetString());
+ EXPECT_EQ(out, R"({"child":{"x":true}})");
+}
+
+TEST(JsonWriter, WriteValueObjectWithArray) {
+ sj::parser parser;
+ std::string json_str = R"({"values":[1,2,3]})";
+ simdjson::padded_string json(json_str);
+
+ sj::document doc;
+ ASSERT_EQ(parser.iterate(json).get(doc), simdjson::SUCCESS);
+
+ sj::value value;
+ ASSERT_EQ(doc.get_value().get(value), simdjson::SUCCESS);
+
+ JsonWriter writer;
+ ASSERT_OK(writer.WriteValue(value));
+
+ ASSERT_OK_AND_ASSIGN(std::string_view out, writer.GetString());
+ EXPECT_EQ(out, R"({"values":[1,2,3]})");
+}
+
+TEST(JsonWriter, WriteValueComplexObject) {
+ sj::parser parser;
+ std::string json_str =
+
R"({"name":"arrow","version":1,"enabled":true,"values":[1,2.5,null,{"nested":[false,{"x":10}]}]})";
+ simdjson::padded_string json(json_str);
+
+ sj::document doc;
+ ASSERT_EQ(parser.iterate(json).get(doc), simdjson::SUCCESS);
+
+ sj::value value;
+ ASSERT_EQ(doc.get_value().get(value), simdjson::SUCCESS);
+
+ JsonWriter writer;
+ ASSERT_OK(writer.WriteValue(value));
+
+ ASSERT_OK_AND_ASSIGN(std::string_view out, writer.GetString());
+ EXPECT_EQ(
+ out,
+
R"({"name":"arrow","version":1,"enabled":true,"values":[1,2.5,null,{"nested":[false,{"x":10}]}]})");
+}
+
+TEST(JsonWriter, WriteValueEmptyObject) {
+ sj::parser parser;
+ std::string json_str = "{}";
+ simdjson::padded_string json(json_str);
+
+ sj::document doc;
+ ASSERT_EQ(parser.iterate(json).get(doc), simdjson::SUCCESS);
+
+ sj::value value;
+ ASSERT_EQ(doc.get_value().get(value), simdjson::SUCCESS);
+
+ JsonWriter writer;
+ ASSERT_OK(writer.WriteValue(value));
+
+ ASSERT_OK_AND_ASSIGN(std::string_view out, writer.GetString());
+ EXPECT_EQ(out, "{}");
+}
+
+TEST(JsonWriter, WriteValueAllNumberTypes) {
+ sj::parser parser;
+ std::string json_str = R"({
+ "signed":-42,
+ "unsigned":18446744073709551615,
+ "double":2.5,
+ "big":184467440737095516161234567890
+ })";
+ simdjson::padded_string json(json_str);
+
+ sj::document doc;
+ ASSERT_EQ(parser.iterate(json).get(doc), simdjson::SUCCESS);
+
+ sj::value value;
+ ASSERT_EQ(doc.get_value().get(value), simdjson::SUCCESS);
+
+ JsonWriter writer;
+ ASSERT_OK(writer.WriteValue(value));
+
+ ASSERT_OK_AND_ASSIGN(std::string_view out, writer.GetString());
+ EXPECT_EQ(
+ out,
+
R"({"signed":-42,"unsigned":18446744073709551615,"double":2.5,"big":184467440737095516161234567890})");
+}
+
} // namespace arrow::json
diff --git a/cpp/src/arrow/util/simdjson_internal.h
b/cpp/src/arrow/util/simdjson_internal.h
new file mode 100644
index 0000000000..8ffb741da4
--- /dev/null
+++ b/cpp/src/arrow/util/simdjson_internal.h
@@ -0,0 +1,244 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <concepts>
+#include <cstdint>
+#include <string_view>
+#include <utility>
+
+#include <simdjson.h>
+
+#include "arrow/result.h"
+#include "arrow/status.h"
+
+namespace arrow {
+namespace internal {
+
+// Empty struct to represent the type of a simdjson null value
+struct SimdjsonNull {};
+
+template <typename T>
+struct JsonTypeNameOf;
+
+template <>
+struct JsonTypeNameOf<simdjson::ondemand::array> {
+ static constexpr const char* kValue = "array";
+};
+
+template <>
+struct JsonTypeNameOf<simdjson::ondemand::object> {
+ static constexpr const char* kValue = "object";
+};
+
+template <>
+struct JsonTypeNameOf<std::string_view> {
+ static constexpr const char* kValue = "string";
+};
+
+template <>
+struct JsonTypeNameOf<bool> {
+ static constexpr const char* kValue = "boolean";
+};
+
+template <>
+struct JsonTypeNameOf<SimdjsonNull> {
+ static constexpr const char* kValue = "null";
+};
+
+template <>
+struct JsonTypeNameOf<int64_t> {
+ static constexpr const char* kValue = "number";
+};
+
+template <>
+struct JsonTypeNameOf<uint64_t> {
+ static constexpr const char* kValue = "number";
+};
+
+template <>
+struct JsonTypeNameOf<double> {
+ static constexpr const char* kValue = "number";
+};
+
+template <typename T>
+constexpr const char* JsonTypeName() {
+ return JsonTypeNameOf<T>::kValue;
+}
+
+template <typename T>
+Result<T> GetSimdjsonResult(simdjson::simdjson_result<T> result,
std::string_view error) {
+ T value;
+ if (auto error_code = std::move(result).get(value); error_code !=
simdjson::SUCCESS) {
+ return Status::Invalid(error, simdjson::error_message(error_code));
+ }
+ return value;
+}
+
+template <typename ObjectFn, typename ArrayFn, typename StringFn, typename
BoolFn,
+ typename NullFn, typename Int64Fn, typename Uint64Fn, typename
DoubleFn,
+ typename BigIntegerFn>
+Status VisitJsonValue(simdjson::ondemand::value value, ObjectFn&& object_fn,
+ ArrayFn&& array_fn, StringFn&& string_fn, BoolFn&&
bool_fn,
+ NullFn&& null_fn, Int64Fn&& int64_fn, Uint64Fn&&
uint64_fn,
+ DoubleFn&& double_fn, BigIntegerFn&& big_integer_fn) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto type, GetSimdjsonResult(value.type(), "Failed to determine JSON
type: "));
+
+ switch (type) {
+ case simdjson::ondemand::json_type::object: {
+ ARROW_ASSIGN_OR_RAISE(
+ auto object,
+ GetSimdjsonResult(value.get_object(), "Failed to get JSON object:
"));
+ return object_fn(object);
+ }
+
+ case simdjson::ondemand::json_type::array: {
+ ARROW_ASSIGN_OR_RAISE(
+ auto array, GetSimdjsonResult(value.get_array(), "Failed to get JSON
array: "));
+ return array_fn(array);
+ }
+
+ case simdjson::ondemand::json_type::string: {
+ ARROW_ASSIGN_OR_RAISE(
+ auto string,
+ GetSimdjsonResult(value.get_string(), "Failed to get JSON string:
"));
+ return string_fn(string);
+ }
+
+ case simdjson::ondemand::json_type::boolean: {
+ ARROW_ASSIGN_OR_RAISE(
+ auto boolean,
+ GetSimdjsonResult(value.get_bool(), "Failed to get JSON boolean: "));
+ return bool_fn(boolean);
+ }
+
+ case simdjson::ondemand::json_type::null:
+ return null_fn();
+
+ case simdjson::ondemand::json_type::number: {
+ ARROW_ASSIGN_OR_RAISE(auto number_type,
+ GetSimdjsonResult(value.get_number_type(),
+ "Failed to determine JSON number
type: "));
+
+ switch (number_type) {
+ case simdjson::ondemand::number_type::signed_integer: {
+ ARROW_ASSIGN_OR_RAISE(
+ auto number,
+ GetSimdjsonResult(value.get_int64(), "Failed to get signed
integer: "));
+ return int64_fn(number);
+ }
+
+ case simdjson::ondemand::number_type::unsigned_integer: {
+ ARROW_ASSIGN_OR_RAISE(
+ auto number,
+ GetSimdjsonResult(value.get_uint64(), "Failed to get unsigned
integer: "));
+ return uint64_fn(number);
+ }
+
+ case simdjson::ondemand::number_type::floating_point_number: {
+ ARROW_ASSIGN_OR_RAISE(
+ auto number, GetSimdjsonResult(value.get_double(),
+ "Failed to get floating-point
number: "));
+ return double_fn(number);
+ }
+
+ case simdjson::ondemand::number_type::big_integer:
+ return big_integer_fn(value);
+ }
+
+ return Status::Invalid("Unknown JSON number type");
+ }
+
+ case simdjson::ondemand::json_type::unknown:
+ return Status::Invalid("Unknown JSON type");
+ }
+
+ return Status::Invalid("Unreachable");
+}
+
+inline const char* JsonTypeName(simdjson::ondemand::json_type type) {
+ switch (type) {
+ case simdjson::ondemand::json_type::array:
+ return "array";
+ case simdjson::ondemand::json_type::object:
+ return "object";
+ case simdjson::ondemand::json_type::number:
+ return "number";
+ case simdjson::ondemand::json_type::string:
+ return "string";
+ case simdjson::ondemand::json_type::boolean:
+ return "boolean";
+ case simdjson::ondemand::json_type::null:
+ return "null";
+ default:
+ return "unknown";
+ }
+}
+
+// Result<bool> because peeking the nonRootScalar can fail (parsed lazily)
+inline Result<bool> IsJsonNull(simdjson::ondemand::value& value) {
+ bool is_null;
+ auto error_code = value.is_null().get(is_null);
+ if (error_code != simdjson::SUCCESS) {
+ return Status::Invalid("Error checking for JSON null: ",
+ simdjson::error_message(error_code));
+ }
+ return is_null;
+}
+
+template <typename SimdjsonValueType>
+Result<SimdjsonValueType> GetJsonAs(simdjson::ondemand::value& value) {
+ SimdjsonValueType typed_value{};
+ simdjson::error_code error_code;
+
+ if constexpr (std::same_as<SimdjsonValueType, SimdjsonNull>) {
+ // simdjson has no get<>() for null; probe it explicitly
+ bool is_null;
+ error_code = value.is_null().get(is_null);
+ if (error_code == simdjson::SUCCESS && !is_null) {
+ error_code = simdjson::INCORRECT_TYPE;
+ }
+ } else {
+ error_code = value.get(typed_value);
+ }
+
+ if (error_code != simdjson::SUCCESS) {
+ simdjson::ondemand::json_type json_type;
+ if (value.type().get(json_type) != simdjson::SUCCESS) {
+ if constexpr (std::same_as<SimdjsonValueType, SimdjsonNull>) {
+ return Status::Invalid("Expected null, got malformed JSON value");
+ } else {
+ return Status::Invalid("Expected ", JsonTypeName<SimdjsonValueType>(),
+ ", got malformed JSON value");
+ }
+ }
+
+ if constexpr (std::same_as<SimdjsonValueType, SimdjsonNull>) {
+ return Status::Invalid("Expected null, got JSON type ",
JsonTypeName(json_type));
+ } else {
+ return Status::Invalid("Expected ", JsonTypeName<SimdjsonValueType>(),
+ ", got JSON type ", JsonTypeName(json_type));
+ }
+ }
+
+ return typed_value;
+}
+
+} // namespace internal
+} // namespace arrow
