This is an automated email from the ASF dual-hosted git repository.
pitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new f14ae5b3d98 GH-51013: [C++] Replace RapidJSON in JSON test utilities
(#51014)
f14ae5b3d98 is described below
commit f14ae5b3d9818138636a560c50704ca8bd71afba
Author: Aaditya Srinivasan <[email protected]>
AuthorDate: Tue Sep 8 15:05:36 2026 +0530
GH-51013: [C++] Replace RapidJSON in JSON test utilities (#51014)
### Rationale for this change
This PR continues the simdjson migration by replacing RapidJSON usage in
the C++ JSON test and testing utilities with simdjson and Arrow's existing
`JsonWriter`.
### Changes
- Replace `rapidjson::Writer` and `StringBuffer` usage in JSON test
utilities with `JsonWriter`.
- Replace RapidJSON-based JSON pretty-printing with simdjson's
`fractured_json_string()`.
- Replace RapidJSON DOM parsing in `TensorFromJSON()` with simdjson's DOM
API.
- Add test coverage for `TensorFromJSON()` with strides and dimension names.
- Update test linking to include `arrow::simdjson`.
* GitHub Issue: #51013
Lead-authored-by: Aaditya Srinivasan <[email protected]>
Co-authored-by: Antoine Pitrou <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>
---
cpp/src/arrow/CMakeLists.txt | 4 +--
cpp/src/arrow/json/CMakeLists.txt | 4 ++-
cpp/src/arrow/json/parser_benchmark.cc | 10 ++++--
cpp/src/arrow/json/reader_test.cc | 7 ++--
cpp/src/arrow/json/test_common.h | 61 +++++++++++++++-----------------
cpp/src/arrow/meson.build | 1 +
cpp/src/arrow/testing/gtest_util.cc | 42 ++++++++++------------
cpp/src/arrow/testing/gtest_util_test.cc | 17 +++++++++
8 files changed, 81 insertions(+), 65 deletions(-)
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index d7086773a1f..eead221dbda 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -741,8 +741,8 @@ else()
endif()
set(ARROW_TESTING_SHARED_LINK_LIBS arrow_shared ${ARROW_GTEST_GTEST})
-set(ARROW_TESTING_SHARED_PRIVATE_LINK_LIBS arrow::flatbuffers RapidJSON)
-set(ARROW_TESTING_STATIC_LINK_LIBS arrow::flatbuffers RapidJSON arrow_static
+set(ARROW_TESTING_SHARED_PRIVATE_LINK_LIBS arrow::flatbuffers arrow::simdjson)
+set(ARROW_TESTING_STATIC_LINK_LIBS arrow::flatbuffers arrow::simdjson
arrow_static
${ARROW_GTEST_GTEST})
if(ARROW_ENABLE_THREADING)
list(APPEND ARROW_TESTING_SHARED_PRIVATE_LINK_LIBS arrow::Boost::process)
diff --git a/cpp/src/arrow/json/CMakeLists.txt
b/cpp/src/arrow/json/CMakeLists.txt
index b930034537d..de9115497e2 100644
--- a/cpp/src/arrow/json/CMakeLists.txt
+++ b/cpp/src/arrow/json/CMakeLists.txt
@@ -33,7 +33,9 @@ add_arrow_benchmark(parser_benchmark
PREFIX
"arrow-json"
EXTRA_LINK_LIBS
- RapidJSON)
+ RapidJSON
+ simdjson::simdjson)
+
arrow_install_all_headers("arrow/json")
# pkg-config support
diff --git a/cpp/src/arrow/json/parser_benchmark.cc
b/cpp/src/arrow/json/parser_benchmark.cc
index a5a6eb68e67..f26d6a14431 100644
--- a/cpp/src/arrow/json/parser_benchmark.cc
+++ b/cpp/src/arrow/json/parser_benchmark.cc
@@ -38,10 +38,14 @@ std::string GenerateTestData(const Input& input, int
num_rows,
std::default_random_engine engine(kSeed);
std::string json;
for (int i = 0; i < num_rows; ++i) {
- StringBuffer sb;
- Writer writer(sb);
+ Writer writer;
ABORT_NOT_OK(Generate(input, engine, &writer, options));
- json += pretty ? PrettyPrint(sb.GetString()) : sb.GetString();
+
+ auto json_result = writer.GetString();
+ ABORT_NOT_OK(json_result.status());
+ auto json_view = std::move(json_result).ValueOrDie();
+
+ json += pretty ? PrettyPrint(json_view) : json_view;
json += "\n";
}
return json;
diff --git a/cpp/src/arrow/json/reader_test.cc
b/cpp/src/arrow/json/reader_test.cc
index 2aca602ae9e..ac5bafb5293 100644
--- a/cpp/src/arrow/json/reader_test.cc
+++ b/cpp/src/arrow/json/reader_test.cc
@@ -550,10 +550,11 @@ class StreamingReaderTestBase {
auto options = GenerateOptions::Defaults();
options.null_probability = 0;
for (int i = 0; i < num_rows; ++i) {
- StringBuffer string_buffer;
- Writer writer(string_buffer);
+ Writer writer;
ABORT_NOT_OK(Generate(data_fields, engine, &writer, options));
- std::string json = string_buffer.GetString();
+
+ std::string json(writer.GetString().ValueOrDie());
+
rows[i] = Join({"{\"i\":", std::to_string(i), ",\"d\":", json, "}\n"});
max_row_size = std::max(max_row_size, rows[i].size());
}
diff --git a/cpp/src/arrow/json/test_common.h b/cpp/src/arrow/json/test_common.h
index ab2ce9cdc74..241584959e7 100644
--- a/cpp/src/arrow/json/test_common.h
+++ b/cpp/src/arrow/json/test_common.h
@@ -25,35 +25,30 @@
#include <utility>
#include <vector>
+#include <simdjson.h>
+
#include "arrow/array.h"
#include "arrow/array/builder_binary.h"
#include "arrow/io/memory.h"
#include "arrow/json/converter.h"
#include "arrow/json/options.h"
#include "arrow/json/parser.h"
-#include "arrow/json/rapidjson_defs.h"
+#include "arrow/result.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/testing/random.h"
#include "arrow/type.h"
#include "arrow/util/checked_cast.h"
+#include "arrow/util/simdjson_internal.h"
#include "arrow/visit_type_inline.h"
-#include "rapidjson/document.h"
-#include "rapidjson/prettywriter.h"
-#include "rapidjson/reader.h"
-#include "rapidjson/writer.h"
-
namespace arrow {
using internal::checked_cast;
namespace json {
-namespace rj = arrow::rapidjson;
-
-using rj::StringBuffer;
using std::string_view;
-using Writer = rj::Writer<StringBuffer>;
+using Writer = internal::JsonWriter;
struct GenerateOptions {
// Probability of a field being written
@@ -87,35 +82,43 @@ inline static Status Generate(
template <typename Engine>
struct GenerateImpl {
- Status Visit(const NullType&) { return OK(writer.Null()); }
+ Status Visit(const NullType&) {
+ writer.Null();
+ return Status::OK();
+ }
Status Visit(const BooleanType&) {
- return OK(writer.Bool(std::uniform_int_distribution<uint16_t>{}(e) & 1));
+ writer.Bool(std::uniform_int_distribution<uint16_t>{}(e) & 1);
+ return Status::OK();
}
template <typename T>
enable_if_physical_unsigned_integer<T, Status> Visit(const T&) {
auto val = std::uniform_int_distribution<>{}(e);
- return OK(writer.Uint64(static_cast<typename T::c_type>(val)));
+ writer.Uint64(static_cast<typename T::c_type>(val));
+ return Status::OK();
}
template <typename T>
enable_if_physical_signed_integer<T, Status> Visit(const T&) {
auto val = std::uniform_int_distribution<>{}(e);
- return OK(writer.Int64(static_cast<typename T::c_type>(val)));
+ writer.Int64(static_cast<typename T::c_type>(val));
+ return Status::OK();
}
template <typename T>
enable_if_physical_floating_point<T, Status> Visit(const T&) {
auto val = std::normal_distribution<typename T::c_type>{0, 1 << 10}(e);
- return OK(writer.Double(val));
+ writer.Double(val);
+ return Status::OK();
}
Status GenerateUtf8(const DataType&) {
auto num_codepoints = std::poisson_distribution<>{4}(e);
auto seed = std::uniform_int_distribution<uint32_t>{}(e);
std::string s = RandomUtf8String(seed, num_codepoints);
- return OK(writer.String(s));
+ writer.String(s);
+ return Status::OK();
}
template <typename T>
@@ -132,7 +135,8 @@ struct GenerateImpl {
for (int i = 0; i < size; ++i) {
RETURN_NOT_OK(Generate(t.value_type(), e, &writer, options));
}
- return OK(writer.EndArray(size));
+ writer.EndArray();
+ return Status::OK();
}
Status Visit(const ListViewType& t) { return NotImplemented(t); }
@@ -162,7 +166,7 @@ struct GenerateImpl {
}
Engine& e;
- rj::Writer<rj::StringBuffer>& writer;
+ Writer& writer;
const GenerateOptions& options;
};
@@ -180,12 +184,9 @@ inline static Status Generate(const
std::shared_ptr<DataType>& type, Engine& e,
template <typename Engine>
inline static Status Generate(const std::vector<std::shared_ptr<Field>>&
fields,
Engine& e, Writer* writer, const
GenerateOptions& options) {
- RETURN_NOT_OK(OK(writer->StartObject()));
-
- int num_fields = 0;
+ writer->StartObject();
auto write_field = [&](const Field& f) {
- ++num_fields;
- writer->Key(f.name().c_str());
+ writer->Key(f.name());
return Generate(f.type(), e, writer, options);
};
@@ -210,7 +211,8 @@ inline static Status Generate(const
std::vector<std::shared_ptr<Field>>& fields,
}
}
- return OK(writer->EndObject(num_fields));
+ writer->EndObject();
+ return Status::OK();
}
inline static Status MakeStream(string_view src_str,
@@ -257,15 +259,8 @@ inline static Status ParseFromString(ParseOptions options,
string_view src_str,
return Status::OK();
}
-static inline std::string PrettyPrint(string_view one_line) {
- rj::Document document;
-
- // Must pass size to avoid ASAN issues.
- document.Parse(one_line.data(), one_line.size());
- rj::StringBuffer sb;
- rj::PrettyWriter<rj::StringBuffer> writer(sb);
- document.Accept(writer);
- return sb.GetString();
+static inline std::string PrettyPrint(std::string_view one_line) {
+ return simdjson::fractured_json_string(one_line);
}
template <typename T>
diff --git a/cpp/src/arrow/meson.build b/cpp/src/arrow/meson.build
index f181321aaab..fea26ef4e45 100644
--- a/cpp/src/arrow/meson.build
+++ b/cpp/src/arrow/meson.build
@@ -758,6 +758,7 @@ if needs_testing
filesystem_dep,
gmock_dep,
gtest_dep,
+ simdjson_dep,
],
)
diff --git a/cpp/src/arrow/testing/gtest_util.cc
b/cpp/src/arrow/testing/gtest_util.cc
index b7d2a963d0d..bbbb1f880af 100644
--- a/cpp/src/arrow/testing/gtest_util.cc
+++ b/cpp/src/arrow/testing/gtest_util.cc
@@ -53,7 +53,6 @@
#include "arrow/ipc/reader.h"
#include "arrow/ipc/writer.h"
#include "arrow/json/from_string.h"
-#include "arrow/json/rapidjson_defs.h" // IWYU pragma: keep
#include "arrow/pretty_print.h"
#include "arrow/record_batch.h"
#include "arrow/status.h"
@@ -65,13 +64,10 @@
#include "arrow/util/future.h"
#include "arrow/util/io_util.h"
#include "arrow/util/logging_internal.h"
+#include "arrow/util/simdjson_internal.h"
#include "arrow/util/thread_pool.h"
#include "arrow/util/windows_compatibility.h"
-#include <rapidjson/document.h>
-
-namespace rj = arrow::rapidjson;
-
namespace arrow {
using internal::checked_cast;
@@ -445,24 +441,24 @@ std::shared_ptr<Tensor> TensorFromJSON(const
std::shared_ptr<DataType>& type,
std::string_view dim_names) {
std::shared_ptr<Array> array = arrow::ArrayFromJSON(type, data);
- rj::Document json_shape;
- json_shape.Parse(shape.data(), shape.length());
- std::vector<int64_t> shape_vector;
- for (auto& x : json_shape.GetArray()) {
- shape_vector.emplace_back(x.GetInt64());
- }
- rj::Document json_strides;
- json_strides.Parse(strides.data(), strides.length());
- std::vector<int64_t> strides_vector;
- for (auto& x : json_strides.GetArray()) {
- strides_vector.emplace_back(x.GetInt64());
- }
- rj::Document json_dim_names;
- json_dim_names.Parse(dim_names.data(), dim_names.length());
- std::vector<std::string> dim_names_vector;
- for (auto& x : json_dim_names.GetArray()) {
- dim_names_vector.emplace_back(x.GetString());
- }
+ simdjson::dom::parser parser;
+
+ auto json_shape =
+ internal::ResolveSimdjsonResult(parser.parse(shape), "Failed to parse
shape")
+ .ValueOrDie();
+ auto shape_vector = internal::GetJsonIntArray(json_shape,
"shape").ValueOrDie();
+
+ auto json_strides =
+ internal::ResolveSimdjsonResult(parser.parse(strides), "Failed to parse
strides")
+ .ValueOrDie();
+ auto strides_vector = internal::GetJsonIntArray(json_strides,
"strides").ValueOrDie();
+
+ auto json_dim_names =
internal::ResolveSimdjsonResult(parser.parse(dim_names),
+ "Failed to parse
dimension names")
+ .ValueOrDie();
+ auto dim_names_vector =
+ internal::GetJsonStringArray(json_dim_names, "dimension
names").ValueOrDie();
+
return *Tensor::Make(type, array->data()->buffers[1], shape_vector,
strides_vector,
dim_names_vector);
}
diff --git a/cpp/src/arrow/testing/gtest_util_test.cc
b/cpp/src/arrow/testing/gtest_util_test.cc
index 31b5b9e6628..f8bf694f5c3 100644
--- a/cpp/src/arrow/testing/gtest_util_test.cc
+++ b/cpp/src/arrow/testing/gtest_util_test.cc
@@ -179,6 +179,23 @@ TEST_F(TestTensorFromJSON, FromJSON) {
EXPECT_TRUE(tensor_expected->Equals(*result));
}
+TEST_F(TestTensorFromJSON, FromJSONWithStridesAndDimNames) {
+ std::vector<int64_t> shape = {2, 3};
+ std::vector<int64_t> strides = {sizeof(int64_t) * 3, sizeof(int64_t)};
+ std::vector<std::string> dim_names = {"row", "column"};
+ std::vector<int64_t> values = {1, 2, 3, 4, 5, 6};
+ auto data = Buffer::Wrap(values);
+
+ std::shared_ptr<Tensor> tensor_expected;
+ ASSERT_OK_AND_ASSIGN(tensor_expected,
+ Tensor::Make(int64(), data, shape, strides, dim_names));
+
+ std::shared_ptr<Tensor> result = TensorFromJSON(int64(), "[1, 2, 3, 4, 5,
6]", "[2, 3]",
+ "[24, 8]", R"(["row",
"column"])");
+
+ EXPECT_TRUE(tensor_expected->Equals(*result));
+}
+
TEST(AssertTestWithinUlp, Basics) {
AssertWithinUlp(123.4567, 123.45670000000015, 11);
AssertWithinUlp(123.456f, 123.456085f, 11);
