This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new c44eca6892 GH-50779: [C++][Parquet] Replace remaining RapidJSON usage
with simdjson (#50781)
c44eca6892 is described below
commit c44eca6892364e58489fe564f910c0e1c1c6cb6b
Author: Aaditya Srinivasan <[email protected]>
AuthorDate: Thu Aug 13 14:42:02 2026 +0530
GH-50779: [C++][Parquet] Replace remaining RapidJSON usage with simdjson
(#50781)
### Rationale for this change
This PR continues the simdjson migration by replacing the remaining
RapidJSON usage under `cpp/src/parquet` with simdjson and `JsonWriter`. It also
updates the Meson build to support simdjson and removes unnecessary RapidJSON
dependencies from the Parquet build configuration.
### What changes are included in this PR?
- Replace the remaining RapidJSON parsing logic in `reader_test.cc` with
simdjson.
- Replace RapidJSON parsing and serialization in
`geospatial/util_json_internal.cc` with simdjson and `JsonWriter`.
- Replace RapidJSON string escaping in `types.cc` with `JsonWriter`.
- Add Meson support for simdjson, including a fallback CMake subproject.
- Link simdjson in the relevant Meson targets.
- Remove unnecessary RapidJSON dependencies from the Parquet Meson and
CMake build configuration.
* GitHub Issue: #50779
Authored-by: Aaditya Srinivasan <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
cpp/src/arrow/json/from_string.cc | 10 +-
cpp/src/arrow/json/json_writer_internal.cc | 18 +-
cpp/src/arrow/json/meson.build | 2 +-
cpp/src/arrow/meson.build | 18 +-
cpp/src/arrow/util/simdjson_internal.h | 120 ++++++++++--
cpp/src/parquet/CMakeLists.txt | 9 +-
cpp/src/parquet/geospatial/util_json_internal.cc | 213 ++++++++++++++-------
cpp/src/parquet/geospatial/util_json_internal.h | 8 +-
.../parquet/geospatial/util_json_internal_test.cc | 68 +++++++
cpp/src/parquet/meson.build | 4 +-
cpp/src/parquet/reader_test.cc | 22 +--
cpp/src/parquet/schema_test.cc | 6 +-
cpp/src/parquet/types.cc | 15 +-
cpp/subprojects/simdjson.wrap | 23 +++
14 files changed, 401 insertions(+), 135 deletions(-)
diff --git a/cpp/src/arrow/json/from_string.cc
b/cpp/src/arrow/json/from_string.cc
index c9d9106671..f573ae7106 100644
--- a/cpp/src/arrow/json/from_string.cc
+++ b/cpp/src/arrow/json/from_string.cc
@@ -106,7 +106,7 @@ class ConcreteConverter : public JSONConverter {
int32_t num_elements = 0;
for (auto element : json_array) {
ARROW_ASSIGN_OR_RAISE(auto value,
- internal::GetSimdjsonResult<sj::value>(
+ internal::ResolveSimdjsonResult<sj::value>(
element, "Could not iterate elements of JSON
array: "));
RETURN_NOT_OK(self->AppendValue(value));
num_elements++;
@@ -287,7 +287,7 @@ Status ProcessJsonArrayElements(
}
ARROW_ASSIGN_OR_RAISE(sj::value element,
- internal::GetSimdjsonResult<sj::value>(
+ internal::ResolveSimdjsonResult<sj::value>(
*it, "Could not iterate elements of JSON array:
"));
RETURN_NOT_OK(handler(element));
++it;
@@ -652,7 +652,7 @@ class MapConverter final : public
ConcreteConverter<MapConverter> {
for (auto json_pair_result : array) {
ARROW_ASSIGN_OR_RAISE(
auto json_pair,
- internal::GetSimdjsonResult<sj::value>(
+ internal::ResolveSimdjsonResult<sj::value>(
json_pair_result, "Could not iterate elements of JSON array: "));
ARROW_ASSIGN_OR_RAISE(auto json_pair_array,
internal::GetJsonAs<sj::array>(json_pair));
@@ -763,7 +763,7 @@ class StructConverter final : public
ConcreteConverter<StructConverter> {
size_t i = 0;
for (auto child : array) {
ARROW_ASSIGN_OR_RAISE(auto child_value,
- internal::GetSimdjsonResult<sj::value>(
+ internal::ResolveSimdjsonResult<sj::value>(
child, "Could not iterate elements of JSON
array: "));
RETURN_NOT_OK(child_converters_[i]->AppendValue(child_value));
++i;
@@ -779,7 +779,7 @@ class StructConverter final : public
ConcreteConverter<StructConverter> {
std::vector<bool> field_seen(num_fields, false);
for (auto field_result : object) {
ARROW_ASSIGN_OR_RAISE(auto field,
- internal::GetSimdjsonResult<sj::field>(
+ internal::ResolveSimdjsonResult<sj::field>(
field_result, "Error getting field of object:
"));
std::string_view key;
if (field.unescaped_key(/*allow_replacement=*/false).get(key) !=
diff --git a/cpp/src/arrow/json/json_writer_internal.cc
b/cpp/src/arrow/json/json_writer_internal.cc
index 5676949024..6d3e9ff9d8 100644
--- a/cpp/src/arrow/json/json_writer_internal.cc
+++ b/cpp/src/arrow/json/json_writer_internal.cc
@@ -108,14 +108,14 @@ Status JsonWriter::WriteValue(sj::value value) {
for (auto field : object) {
ARROW_ASSIGN_OR_RAISE(
- auto key, internal::GetSimdjsonResult(field.unescaped_key(),
- "Failed to get object key:
"));
+ auto key, internal::ResolveSimdjsonResult(field.unescaped_key(),
+ "Failed to get object
key"));
Key(key);
- ARROW_ASSIGN_OR_RAISE(
- auto field_value,
- internal::GetSimdjsonResult(field.value(), "Failed to get object
value: "));
+ ARROW_ASSIGN_OR_RAISE(auto field_value,
+ internal::ResolveSimdjsonResult(
+ field.value(), "Failed to get object
value"));
RETURN_NOT_OK(WriteValue(field_value));
}
@@ -130,7 +130,7 @@ Status JsonWriter::WriteValue(sj::value value) {
for (auto element : array) {
ARROW_ASSIGN_OR_RAISE(
auto element_value,
- internal::GetSimdjsonResult(element, "Failed to iterate JSON
array: "));
+ internal::ResolveSimdjsonResult(element, "Failed to iterate JSON
array"));
RETURN_NOT_OK(WriteValue(element_value));
}
@@ -170,9 +170,9 @@ Status JsonWriter::WriteValue(sj::value value) {
},
[&](sj::value value) -> Status {
- ARROW_ASSIGN_OR_RAISE(auto raw_json,
-
internal::GetSimdjsonResult(simdjson::to_json_string(value),
- "Failed to get raw
JSON: "));
+ ARROW_ASSIGN_OR_RAISE(auto raw_json, internal::ResolveSimdjsonResult(
+
simdjson::to_json_string(value),
+ "Failed to get raw JSON"));
RawValue(raw_json);
return Status::OK();
});
diff --git a/cpp/src/arrow/json/meson.build b/cpp/src/arrow/json/meson.build
index edf92a46fd..ee2a26b2cc 100644
--- a/cpp/src/arrow/json/meson.build
+++ b/cpp/src/arrow/json/meson.build
@@ -26,7 +26,7 @@ exc = executable(
'parser_test.cc',
'reader_test.cc',
],
- dependencies: [arrow_test_dep, rapidjson_dep],
+ dependencies: [arrow_test_dep, rapidjson_dep, simdjson_dep],
)
test('arrow-json-test', exc)
diff --git a/cpp/src/arrow/meson.build b/cpp/src/arrow/meson.build
index 955ef78973..b5443c28cf 100644
--- a/cpp/src/arrow/meson.build
+++ b/cpp/src/arrow/meson.build
@@ -318,6 +318,22 @@ else
rapidjson_dep = disabler()
endif
+if needs_json or needs_integration
+ simdjson_dep = dependency('simdjson', allow_fallback: false, required:
false)
+
+ if not simdjson_dep.found()
+ cmake = import('cmake')
+ simdjson_opts = cmake.subproject_options()
+
+ simdjson_opts.add_cmake_defines({'SIMDJSON_EXCEPTIONS': 'OFF'})
+
+ simdjson_proj = cmake.subproject('simdjson', options: simdjson_opts)
+ simdjson_dep = simdjson_proj.dependency('simdjson')
+ endif
+else
+ simdjson_dep = disabler()
+endif
+
azure_dep = disabler()
gcs_dep = disabler()
s3_dep = disabler()
@@ -520,7 +536,7 @@ if needs_json
'json/parser.cc',
'json/reader.cc',
],
- 'dependencies': [rapidjson_dep],
+ 'dependencies': [rapidjson_dep, simdjson_dep],
},
}
endif
diff --git a/cpp/src/arrow/util/simdjson_internal.h
b/cpp/src/arrow/util/simdjson_internal.h
index 8ffb741da4..1badd99938 100644
--- a/cpp/src/arrow/util/simdjson_internal.h
+++ b/cpp/src/arrow/util/simdjson_internal.h
@@ -82,10 +82,11 @@ constexpr const char* JsonTypeName() {
}
template <typename T>
-Result<T> GetSimdjsonResult(simdjson::simdjson_result<T> result,
std::string_view error) {
+Result<T> ResolveSimdjsonResult(simdjson::simdjson_result<T> result,
+ std::string_view error) {
T value;
if (auto error_code = std::move(result).get(value); error_code !=
simdjson::SUCCESS) {
- return Status::Invalid(error, simdjson::error_message(error_code));
+ return Status::Invalid(error, ": ", simdjson::error_message(error_code));
}
return value;
}
@@ -98,33 +99,34 @@ Status VisitJsonValue(simdjson::ondemand::value value,
ObjectFn&& object_fn,
NullFn&& null_fn, Int64Fn&& int64_fn, Uint64Fn&&
uint64_fn,
DoubleFn&& double_fn, BigIntegerFn&& big_integer_fn) {
ARROW_ASSIGN_OR_RAISE(
- auto type, GetSimdjsonResult(value.type(), "Failed to determine JSON
type: "));
+ auto type, ResolveSimdjsonResult(value.type(), "Failed to determine JSON
type"));
switch (type) {
case simdjson::ondemand::json_type::object: {
ARROW_ASSIGN_OR_RAISE(
auto object,
- GetSimdjsonResult(value.get_object(), "Failed to get JSON object:
"));
+ ResolveSimdjsonResult(value.get_object(), "Failed to get JSON
object"));
return object_fn(object);
}
case simdjson::ondemand::json_type::array: {
ARROW_ASSIGN_OR_RAISE(
- auto array, GetSimdjsonResult(value.get_array(), "Failed to get JSON
array: "));
+ auto array,
+ ResolveSimdjsonResult(value.get_array(), "Failed to get JSON
array"));
return array_fn(array);
}
case simdjson::ondemand::json_type::string: {
ARROW_ASSIGN_OR_RAISE(
auto string,
- GetSimdjsonResult(value.get_string(), "Failed to get JSON string:
"));
+ ResolveSimdjsonResult(value.get_string(), "Failed to get JSON
string"));
return string_fn(string);
}
case simdjson::ondemand::json_type::boolean: {
ARROW_ASSIGN_OR_RAISE(
auto boolean,
- GetSimdjsonResult(value.get_bool(), "Failed to get JSON boolean: "));
+ ResolveSimdjsonResult(value.get_bool(), "Failed to get JSON
boolean"));
return bool_fn(boolean);
}
@@ -132,29 +134,30 @@ Status VisitJsonValue(simdjson::ondemand::value value,
ObjectFn&& object_fn,
return null_fn();
case simdjson::ondemand::json_type::number: {
- ARROW_ASSIGN_OR_RAISE(auto number_type,
- GetSimdjsonResult(value.get_number_type(),
- "Failed to determine JSON number
type: "));
+ ARROW_ASSIGN_OR_RAISE(
+ auto number_type,
+ ResolveSimdjsonResult(value.get_number_type(),
+ "Failed to determine JSON number type"));
switch (number_type) {
case simdjson::ondemand::number_type::signed_integer: {
ARROW_ASSIGN_OR_RAISE(
auto number,
- GetSimdjsonResult(value.get_int64(), "Failed to get signed
integer: "));
+ ResolveSimdjsonResult(value.get_int64(), "Failed to get signed
integer"));
return int64_fn(number);
}
case simdjson::ondemand::number_type::unsigned_integer: {
- ARROW_ASSIGN_OR_RAISE(
- auto number,
- GetSimdjsonResult(value.get_uint64(), "Failed to get unsigned
integer: "));
+ ARROW_ASSIGN_OR_RAISE(auto number,
+ ResolveSimdjsonResult(value.get_uint64(),
+ "Failed to get unsigned
integer"));
return uint64_fn(number);
}
case simdjson::ondemand::number_type::floating_point_number: {
ARROW_ASSIGN_OR_RAISE(
- auto number, GetSimdjsonResult(value.get_double(),
- "Failed to get floating-point
number: "));
+ auto number, ResolveSimdjsonResult(value.get_double(),
+ "Failed to get floating-point
number"));
return double_fn(number);
}
@@ -240,5 +243,90 @@ Result<SimdjsonValueType>
GetJsonAs(simdjson::ondemand::value& value) {
return typed_value;
}
+template <typename T>
+Result<T> GetJsonField(simdjson::ondemand::object& object, std::string_view
key) {
+ for (auto field_result : object) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto field, ResolveSimdjsonResult(field_result, "Failed to iterate
JSON object"));
+
+ ARROW_ASSIGN_OR_RAISE(
+ auto field_key,
+ ResolveSimdjsonResult(field.unescaped_key(), "Failed to get JSON
object key"));
+
+ if (field_key == key) {
+ auto value = field.value();
+
+ if constexpr (std::is_same_v<T, simdjson::ondemand::value>) {
+ return value;
+ } else {
+ return GetJsonAs<T>(value);
+ }
+ }
+ }
+
+ return Status::KeyError("Missing JSON field: ", key);
+}
+
+inline Result<std::string> MinifyJson(std::string_view json) {
+ std::string minified(json.size(), '\0');
+ size_t minified_len = 0;
+
+ if (auto error =
+ simdjson::minify(json.data(), json.size(), minified.data(),
minified_len);
+ error != simdjson::SUCCESS) {
+ return Status::Invalid("Failed to minify JSON: ",
simdjson::error_message(error));
+ }
+
+ minified.resize(minified_len);
+ return minified;
+}
+
+inline Status ValidateJsonObject(simdjson::ondemand::object object);
+
+inline Status ValidateJsonArray(simdjson::ondemand::array array);
+
+inline Status ConsumeJsonValue(simdjson::ondemand::value value) {
+ return VisitJsonValue(
+ value, ValidateJsonObject, ValidateJsonArray,
+ [](std::string_view) { return Status::OK(); }, [](bool) { return
Status::OK(); },
+ []() { return Status::OK(); }, [](int64_t) { return Status::OK(); },
+ [](uint64_t) { return Status::OK(); }, [](double) { return Status::OK();
},
+ [](simdjson::ondemand::value) { return Status::OK(); });
+}
+
+inline Status ValidateJsonObject(simdjson::ondemand::object object) {
+ for (auto field_result : object) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto field, ResolveSimdjsonResult(field_result, "Failed to iterate
JSON object"));
+
+ RETURN_NOT_OK(ConsumeJsonValue(field.value()));
+ }
+
+ return Status::OK();
+}
+
+inline Status ValidateJsonArray(simdjson::ondemand::array array) {
+ for (auto element_result : array) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto value,
+ ResolveSimdjsonResult(element_result, "Failed to iterate JSON array"));
+
+ RETURN_NOT_OK(ConsumeJsonValue(value));
+ }
+
+ return Status::OK();
+}
+
+inline Status ValidateJsonDocument(simdjson::ondemand::parser& parser,
+ simdjson::padded_string& json) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto document, ResolveSimdjsonResult(parser.iterate(json), "Failed to
parse JSON"));
+
+ ARROW_ASSIGN_OR_RAISE(auto value, ResolveSimdjsonResult(document.get_value(),
+ "Failed to get JSON
value"));
+
+ return ConsumeJsonValue(value);
+}
+
} // namespace internal
} // namespace arrow
diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt
index e5860d8919..606dcdc0a9 100644
--- a/cpp/src/parquet/CMakeLists.txt
+++ b/cpp/src/parquet/CMakeLists.txt
@@ -263,9 +263,9 @@ endif()
list(APPEND PARQUET_SHARED_LINK_LIBS arrow_shared)
-# Add RapidJSON & simdjson libraries
-list(APPEND PARQUET_SHARED_PRIVATE_LINK_LIBS RapidJSON simdjson::simdjson)
-list(APPEND PARQUET_STATIC_LINK_LIBS RapidJSON simdjson::simdjson)
+# Add simdjson libraries
+list(APPEND PARQUET_SHARED_PRIVATE_LINK_LIBS simdjson::simdjson)
+list(APPEND PARQUET_STATIC_LINK_LIBS simdjson::simdjson)
# These are libraries that we will link privately with parquet_shared (as they
# do not need to be linked transitively by other linkers)
@@ -323,7 +323,7 @@ if(ARROW_TESTING)
# "link" our dependencies so that include paths are configured
# correctly
target_link_libraries(parquet_testing PUBLIC ${ARROW_GTEST_GMOCK})
- list(APPEND PARQUET_TEST_LINK_LIBS parquet_testing RapidJSON)
+ list(APPEND PARQUET_TEST_LINK_LIBS parquet_testing simdjson::simdjson)
endif()
if(NOT ARROW_BUILD_SHARED)
@@ -379,6 +379,7 @@ add_parquet_test(internals-test
bloom_filter_test.cc
geospatial/statistics_test.cc
geospatial/util_internal_test.cc
+ geospatial/util_json_internal_test.cc
metadata_test.cc
page_index_test.cc
properties_test.cc
diff --git a/cpp/src/parquet/geospatial/util_json_internal.cc
b/cpp/src/parquet/geospatial/util_json_internal.cc
index 6278ab8873..44efcfc8d7 100644
--- a/cpp/src/parquet/geospatial/util_json_internal.cc
+++ b/cpp/src/parquet/geospatial/util_json_internal.cc
@@ -20,13 +20,11 @@
#include <string>
#include "arrow/extension_type.h"
-#include "arrow/json/rapidjson_defs.h" // IWYU pragma: keep
+#include "arrow/json/json_writer_internal.h"
#include "arrow/result.h"
+#include "arrow/util/simdjson_internal.h"
#include "arrow/util/string.h"
-#include <rapidjson/document.h>
-#include <rapidjson/writer.h>
-
#include "parquet/exception.h"
#include "parquet/types.h"
@@ -34,35 +32,94 @@ namespace parquet {
namespace {
::arrow::Result<std::string> GeospatialGeoArrowCrsToParquetCrs(
- const ::arrow::rapidjson::Document& document) {
- namespace rj = ::arrow::rapidjson;
+ simdjson::ondemand::object object) {
+ auto json_crs_result =
+ ::arrow::internal::GetJsonField<simdjson::ondemand::value>(object,
"crs");
- if (!document.HasMember("crs") || document["crs"].IsNull()) {
- // Parquet GEOMETRY/GEOGRAPHY do not have a concept of a null/missing
- // CRS, but an omitted one is more likely to have meant "lon/lat" than
- // a truly unspecified one (i.e., Engineering CRS with arbitrary XY units)
- return "";
+ if (!json_crs_result.ok()) {
+ if (json_crs_result.status().IsKeyError()) {
+ // Parquet GEOMETRY/GEOGRAPHY do not have a concept of a null/missing
+ // CRS, but an omitted one is more likely to have meant "lon/lat" than
+ // a truly unspecified one (i.e., Engineering CRS with arbitrary XY
units)
+ return "";
+ }
+
+ return json_crs_result.status();
}
- const auto& json_crs = document["crs"];
- if (json_crs.IsString() && (json_crs == "EPSG:4326" || json_crs ==
"OGC:CRS84")) {
- // crs can be left empty because these cases both correspond to
- // longitude/latitude in WGS84 according to the Parquet specification
+ auto json_crs = *std::move(json_crs_result);
+
+ ARROW_ASSIGN_OR_RAISE(bool is_null, ::arrow::internal::IsJsonNull(json_crs));
+ if (is_null) {
return "";
- } else if (json_crs.IsObject()) {
- // Attempt to detect common PROJJSON representations of longitude/latitude
and return
- // an empty crs to maximize compatibility with readers that do not
implement CRS
- // support. PROJJSON stores this in the "id" member like:
- // {..., "id": {"authority": "...", "code": "..."}}
- if (json_crs.HasMember("id")) {
- const auto& identifier = json_crs["id"];
- if (identifier.HasMember("authority") && identifier.HasMember("code")) {
- if (identifier["authority"] == "OGC" && identifier["code"] == "CRS84")
{
- return "";
- } else if (identifier["authority"] == "EPSG" && identifier["code"] ==
"4326") {
+ }
+
+ auto crs_string_result =
::arrow::internal::GetJsonAs<std::string_view>(json_crs);
+
+ if (crs_string_result.ok()) {
+ auto crs_string = *crs_string_result;
+
+ if (crs_string == "EPSG:4326" || crs_string == "OGC:CRS84") {
+ // crs can be left empty because these cases both correspond to
+ // longitude/latitude in WGS84 according to the Parquet specification
+ return "";
+ }
+
+ // If we could not detect a longitude/latitude CRS, just write the string
to the
+ // LogicalType crs (being sure to unescape a JSON string into a regular
string)
+ return std::string(crs_string);
+ }
+
+ ARROW_ASSIGN_OR_RAISE(
+ auto crs_object,
+ ::arrow::internal::GetJsonAs<simdjson::ondemand::object>(json_crs));
+
+ // Attempt to detect common PROJJSON representations of longitude/latitude
and return
+ // an empty crs to maximize compatibility with readers that do not implement
CRS
+ // support. PROJJSON stores this in the "id" member like:
+ // {..., "id": {"authority": "...", "code": "..."}}
+ auto identifier_result =
+ ::arrow::internal::GetJsonField<simdjson::ondemand::object>(crs_object,
"id");
+
+ if (identifier_result.ok()) {
+ auto identifier = *std::move(identifier_result);
+
+ std::optional<std::string_view> authority_string;
+ std::optional<simdjson::ondemand::value> code;
+
+ for (auto field_result : identifier) {
+ ARROW_ASSIGN_OR_RAISE(auto field,
+ ::arrow::internal::ResolveSimdjsonResult(
+ field_result, "Failed to iterate JSON
object"));
+
+ ARROW_ASSIGN_OR_RAISE(auto key,
+ ::arrow::internal::ResolveSimdjsonResult(
+ field.unescaped_key(), "Failed to get JSON
object key"));
+
+ if (key == "authority") {
+ ARROW_ASSIGN_OR_RAISE(
+ auto authority,
+ ::arrow::internal::GetJsonAs<std::string_view>(field.value()));
+ authority_string = authority;
+ } else if (key == "code") {
+ code = field.value();
+ }
+ }
+
+ if (authority_string && code) {
+ auto code_string_result =
::arrow::internal::GetJsonAs<std::string_view>(*code);
+
+ if (code_string_result.ok()) {
+ auto code_string = *code_string_result;
+
+ if ((*authority_string == "OGC" && code_string == "CRS84") ||
+ (*authority_string == "EPSG" && code_string == "4326")) {
return "";
- } else if (identifier["authority"] == "EPSG" &&
identifier["code"].IsInt() &&
- identifier["code"].GetInt() == 4326) {
+ }
+ } else if (*authority_string == "EPSG") {
+ auto code_int_result = ::arrow::internal::GetJsonAs<int64_t>(*code);
+
+ if (code_int_result.ok() && *code_int_result == 4326) {
return "";
}
}
@@ -71,20 +128,21 @@ namespace {
// If we could not detect a longitude/latitude CRS, just write the string to
the
// LogicalType crs (being sure to unescape a JSON string into a regular
string)
- if (json_crs.IsString()) {
- return json_crs.GetString();
- } else {
- rj::StringBuffer buffer;
- rj::Writer<rj::StringBuffer> writer(buffer);
- json_crs.Accept(writer);
- return buffer.GetString();
- }
+ RETURN_NOT_OK(::arrow::internal::ResolveSimdjsonResult(crs_object.reset(),
+ "Failed to reset
'crs' object")
+ .status());
+
+ ARROW_ASSIGN_OR_RAISE(auto raw_crs,
+ ::arrow::internal::ResolveSimdjsonResult(
+ crs_object.raw_json(), "Failed to get raw 'crs'
JSON"));
+
+ return ::arrow::internal::MinifyJson(raw_crs);
}
// Utility for ensuring that a Parquet CRS is valid JSON when written to
// GeoArrow metadata (without escaping it if it is already valid JSON such as
// a PROJJSON string)
-std::string EscapeCrsAsJsonIfRequired(std::string_view crs);
+::arrow::Result<std::string> EscapeCrsAsJsonIfRequired(std::string_view crs);
::arrow::Result<std::string> MakeGeoArrowCrsMetadata(
std::string_view crs,
@@ -113,64 +171,89 @@ std::string EscapeCrsAsJsonIfRequired(std::string_view
crs);
ARROW_ASSIGN_OR_RAISE(std::string projjson_value,
metadata->Get(metadata_field));
// This value should be valid JSON, but if it is not, we escape it as a
string such
// that it can be inspected by the consumer of GeoArrow.
- return R"("crs": )" + EscapeCrsAsJsonIfRequired(projjson_value) +
- R"(, "crs_type": "projjson")";
+ ARROW_ASSIGN_OR_RAISE(auto escaped,
EscapeCrsAsJsonIfRequired(projjson_value));
+ return R"("crs": )" + escaped + R"(, "crs_type": "projjson")";
}
}
// Pass on the string directly to GeoArrow. If the string is already valid
JSON,
// insert it directly into GeoArrow's "crs" field. Otherwise, escape it and
pass it as a
// string value.
- return R"("crs": )" + EscapeCrsAsJsonIfRequired(crs);
+ ARROW_ASSIGN_OR_RAISE(auto escaped, EscapeCrsAsJsonIfRequired(crs));
+
+ return R"("crs": )" + escaped;
}
-std::string EscapeCrsAsJsonIfRequired(std::string_view crs) {
- namespace rj = ::arrow::rapidjson;
- rj::Document document;
- if (document.Parse(crs.data(), crs.length()).HasParseError()) {
- rj::StringBuffer buffer;
- rj::Writer<rj::StringBuffer> writer(buffer);
- rj::Value v;
- v.SetString(crs.data(), static_cast<int32_t>(crs.size()));
- v.Accept(writer);
- return std::string(buffer.GetString());
- } else {
- return std::string(crs);
+::arrow::Result<std::string> EscapeJsonString(std::string_view value) {
+ ::arrow::json::JsonWriter writer;
+ writer.String(value);
+
+ ARROW_ASSIGN_OR_RAISE(auto escaped, writer.GetString());
+ return ::arrow::internal::MinifyJson(escaped);
+}
+
+::arrow::Result<std::string> EscapeCrsAsJsonIfRequired(std::string_view crs) {
+ simdjson::ondemand::parser parser;
+ simdjson::padded_string json(crs);
+
+ if (!::arrow::internal::ValidateJsonDocument(parser, json).ok()) {
+ return EscapeJsonString(crs);
}
+
+ return std::string(crs);
}
} // namespace
-::arrow::Result<std::shared_ptr<const LogicalType>>
LogicalTypeFromGeoArrowMetadata(
- std::string_view serialized_data) {
+::arrow::Result<std::shared_ptr<const parquet::LogicalType>>
+LogicalTypeFromGeoArrowMetadata(std::string_view serialized_data) {
// Parquet has no way to interpret a null or missing CRS, so we choose the
most likely
// intent here (that the user meant to use the default Parquet CRS)
if (serialized_data.empty() || serialized_data == "{}") {
return LogicalType::Geometry();
}
- namespace rj = ::arrow::rapidjson;
- rj::Document document;
- if (document.Parse(serialized_data.data(),
serialized_data.length()).HasParseError()) {
- return ::arrow::Status::Invalid("Invalid serialized JSON data: ",
serialized_data);
+ simdjson::ondemand::parser parser;
+ simdjson::padded_string json(serialized_data);
+
+ RETURN_NOT_OK(::arrow::internal::ValidateJsonDocument(parser, json));
+
+ // Reparse because validation consumes the On-Demand document.
+ ARROW_ASSIGN_OR_RAISE(auto document,
::arrow::internal::ResolveSimdjsonResult(
+ parser.iterate(json), "Failed to
parse JSON"));
+
+ ARROW_ASSIGN_OR_RAISE(
+ auto object,
::arrow::internal::ResolveSimdjsonResult(document.get_object(),
+ "Failed to get
JSON object"));
+
+ ARROW_ASSIGN_OR_RAISE(std::string crs,
GeospatialGeoArrowCrsToParquetCrs(object));
+
+ auto edges_field = object["edges"];
+
+ if (edges_field.error() == simdjson::NO_SUCH_FIELD) {
+ return LogicalType::Geometry(crs);
}
- ARROW_ASSIGN_OR_RAISE(std::string crs,
GeospatialGeoArrowCrsToParquetCrs(document));
+ ARROW_ASSIGN_OR_RAISE(auto edges, ::arrow::internal::ResolveSimdjsonResult(
+ edges_field, "Failed to get 'edges'
field"));
- if (document.HasMember("edges") && document["edges"] == "planar") {
+ ARROW_ASSIGN_OR_RAISE(auto edges_string,
+ ::arrow::internal::GetJsonAs<std::string_view>(edges));
+
+ if (edges_string == "planar") {
return LogicalType::Geometry(crs);
- } else if (document.HasMember("edges") && document["edges"] == "spherical") {
+ }
+
+ if (edges_string == "spherical") {
return LogicalType::Geography(crs,
LogicalType::EdgeInterpolationAlgorithm::SPHERICAL);
- } else if (document.HasMember("edges")) {
- return ::arrow::Status::Invalid("Unsupported GeoArrow edge type: ",
serialized_data);
}
- return LogicalType::Geometry(crs);
+ return ::arrow::Status::Invalid("Unsupported GeoArrow edge type: ",
serialized_data);
}
::arrow::Result<std::shared_ptr<::arrow::DataType>>
GeoArrowTypeFromLogicalType(
- const LogicalType& logical_type,
+ const parquet::LogicalType& logical_type,
const std::shared_ptr<const ::arrow::KeyValueMetadata>& metadata,
const std::shared_ptr<::arrow::DataType>& storage_type) {
// Check if we have a registered GeoArrow type to read into
diff --git a/cpp/src/parquet/geospatial/util_json_internal.h
b/cpp/src/parquet/geospatial/util_json_internal.h
index 1d43b320f8..5c18c55366 100644
--- a/cpp/src/parquet/geospatial/util_json_internal.h
+++ b/cpp/src/parquet/geospatial/util_json_internal.h
@@ -21,6 +21,7 @@
#include "arrow/util/key_value_metadata.h"
+#include "parquet/platform.h"
#include "parquet/types.h"
namespace parquet {
@@ -29,8 +30,8 @@ namespace parquet {
/// GeoArrow `ARROW:extension:metadata` (JSON-encoded extension type metadata)
///
/// Returns the appropriate LogicalType or Invalid if the metadata was invalid.
-::arrow::Result<std::shared_ptr<const LogicalType>>
LogicalTypeFromGeoArrowMetadata(
- std::string_view serialized_data);
+PARQUET_EXPORT ::arrow::Result<std::shared_ptr<const LogicalType>>
+LogicalTypeFromGeoArrowMetadata(std::string_view serialized_data);
/// \brief Compute a suitable DataType into which a GEOMETRY or GEOGRAPHY type
should be
/// read
@@ -38,7 +39,8 @@ namespace parquet {
/// The result of this function depends on whether or not "geoarrow.wkb" has
been
/// registered: if it has, the result will be the registered ExtensionType; if
it has not,
/// the result will be the given storage_type.
-::arrow::Result<std::shared_ptr<::arrow::DataType>>
GeoArrowTypeFromLogicalType(
+PARQUET_EXPORT ::arrow::Result<std::shared_ptr<::arrow::DataType>>
+GeoArrowTypeFromLogicalType(
const LogicalType& logical_type,
const std::shared_ptr<const ::arrow::KeyValueMetadata>& metadata,
const std::shared_ptr<::arrow::DataType>& storage_type);
diff --git a/cpp/src/parquet/geospatial/util_json_internal_test.cc
b/cpp/src/parquet/geospatial/util_json_internal_test.cc
new file mode 100644
index 0000000000..ae86a0cdfb
--- /dev/null
+++ b/cpp/src/parquet/geospatial/util_json_internal_test.cc
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "parquet/geospatial/util_json_internal.h"
+
+#include <memory>
+
+#include <gtest/gtest.h>
+
+#include "arrow/testing/extension_type.h"
+#include "arrow/testing/gtest_util.h"
+#include "arrow/type.h"
+#include "arrow/util/simdjson_internal.h"
+
+#include "parquet/test_util.h"
+
+namespace parquet {
+
+TEST(UtilJsonInternal, InvalidProjJsonIsEscaped) {
+ ::arrow::ExtensionTypeGuard guard(test::geoarrow_wkb());
+
+ auto metadata = ::arrow::key_value_metadata(
+ {"proj"}, {R"({"a":[1,2,]})"}); // Invalid JSON (trailing comma)
+
+ auto logical_type = LogicalType::Geometry("projjson:proj");
+
+ ASSERT_OK_AND_ASSIGN(
+ auto type, GeoArrowTypeFromLogicalType(*logical_type, metadata,
::arrow::binary()));
+
+ auto extension = std::dynamic_pointer_cast<::arrow::ExtensionType>(type);
+ ASSERT_NE(extension, nullptr);
+
+ ASSERT_OK_AND_ASSIGN(auto actual,
+ ::arrow::internal::MinifyJson(extension->Serialize()));
+
+ EXPECT_EQ(actual,
"{\"crs\":\"{\\\"a\\\":[1,2,]}\",\"crs_type\":\"projjson\"}");
+}
+
+TEST(UtilJsonInternal, EscapedCrsKeyIsRecognized) {
+ std::string metadata =
R"({"cr\u0073":"EPSG:3857","crs_type":"authority_code"})";
+
+ ASSERT_OK_AND_ASSIGN(auto logical_type,
LogicalTypeFromGeoArrowMetadata(metadata));
+
+ ASSERT_EQ(logical_type->ToString(), "Geometry(crs=EPSG:3857)");
+}
+
+TEST(UtilJsonInternal, InvalidTrailingMetadataIsRejected) {
+ auto result = LogicalTypeFromGeoArrowMetadata(
+ R"({"crs":"EPSG:3857","edges":"planar","unused":[1,2,]})");
+
+ ASSERT_RAISES(Invalid, result);
+}
+
+} // namespace parquet
diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build
index 9069ccb5fd..f2aff7dfa1 100644
--- a/cpp/src/parquet/meson.build
+++ b/cpp/src/parquet/meson.build
@@ -88,7 +88,7 @@ if not thrift_dep.found()
thrift_dep = thrift_proj.dependency('thrift')
endif
-parquet_deps = [arrow_dep, rapidjson_dep, thrift_dep]
+parquet_deps = [arrow_dep, simdjson_dep, thrift_dep]
if needs_parquet_encryption or get_option('parquet_require_encryption').auto()
openssl_dep = dependency('openssl', required: needs_parquet_encryption)
@@ -198,6 +198,7 @@ parquet_tests = {
'encoding_test.cc',
'geospatial/statistics_test.cc',
'geospatial/util_internal_test.cc',
+ 'geospatial/util_json_internal_test.cc',
'metadata_test.cc',
'page_index_test.cc',
'properties_test.cc',
@@ -291,6 +292,7 @@ parquet_test_dep = [
parquet_dep,
parquet_test_support_dep,
arrow_test_dep,
+ simdjson_dep,
thrift_dep,
]
diff --git a/cpp/src/parquet/reader_test.cc b/cpp/src/parquet/reader_test.cc
index 7ae9021e35..cdeee116fb 100644
--- a/cpp/src/parquet/reader_test.cc
+++ b/cpp/src/parquet/reader_test.cc
@@ -27,12 +27,6 @@
#include <gmock/gmock.h>
#include <gtest/gtest.h>
-#include "arrow/json/rapidjson_defs.h" // IWYU pragma: keep
-
-#include <rapidjson/document.h>
-#include <rapidjson/error/en.h>
-#include <rapidjson/stringbuffer.h>
-
#include "arrow/array.h"
#include "arrow/array/array_binary.h"
#include "arrow/array/builder_binary.h"
@@ -44,6 +38,7 @@
#include "arrow/util/checked_cast.h"
#include "arrow/util/config.h"
#include "arrow/util/range.h"
+#include "arrow/util/simdjson_internal.h"
#include "parquet/column_reader.h"
#include "parquet/column_scanner.h"
@@ -59,8 +54,6 @@
#include "parquet/test_util.h"
#include "parquet/types.h"
-namespace rj = arrow::rapidjson;
-
using arrow::internal::checked_pointer_cast;
using arrow::internal::Zip;
@@ -1230,14 +1223,11 @@ TEST_F(TestJSONWithLocalFile, JSONOutputSortColumns) {
namespace {
::arrow::Status CheckJsonValid(std::string_view json_string) {
- rj::Document json_doc;
- constexpr auto kParseFlags = rj::kParseFullPrecisionFlag |
rj::kParseNanAndInfFlag;
- json_doc.Parse<kParseFlags>(json_string.data(), json_string.length());
- if (json_doc.HasParseError()) {
- return ::arrow::Status::Invalid("JSON parse error at offset ",
- json_doc.GetErrorOffset(), ": ",
-
rj::GetParseError_En(json_doc.GetParseError()));
- }
+ simdjson::ondemand::parser parser;
+ auto padded_json = simdjson::padded_string(json_string);
+
+ RETURN_NOT_OK(::arrow::internal::ValidateJsonDocument(parser, padded_json));
+
return ::arrow::Status::OK();
}
diff --git a/cpp/src/parquet/schema_test.cc b/cpp/src/parquet/schema_test.cc
index 859f14a34d..704b2da79c 100644
--- a/cpp/src/parquet/schema_test.cc
+++ b/cpp/src/parquet/schema_test.cc
@@ -1581,9 +1581,9 @@ TEST(TestLogicalTypeOperation, LogicalTypeRepresentation)
{
{LogicalType::Geometry(R"(crs with "quotes" and \backslashes\)"),
R"(Geometry(crs=crs with "quotes" and \backslashes\))",
R"({"Type": "Geometry", "crs": "crs with \"quotes\" and
\\backslashes\\"})"},
- {LogicalType::Geometry("crs with control characters \u0001 and \u001F"),
- "Geometry(crs=crs with control characters \u0001 and \u001F)",
- R"({"Type": "Geometry", "crs": "crs with control characters \u0001 and
\u001F"})"},
+ {LogicalType::Geometry("crs with control characters \u0001 and \u001f"),
+ "Geometry(crs=crs with control characters \u0001 and \u001f)",
+ R"({"Type": "Geometry", "crs": "crs with control characters \u0001 and
\u001f"})"},
{LogicalType::Geography(), "Geography(crs=, algorithm=spherical)",
R"({"Type": "Geography"})"},
{LogicalType::Geography("srid:1234",
diff --git a/cpp/src/parquet/types.cc b/cpp/src/parquet/types.cc
index 9d7604faec..cc3199f367 100644
--- a/cpp/src/parquet/types.cc
+++ b/cpp/src/parquet/types.cc
@@ -24,16 +24,13 @@
#include <sstream>
#include <string>
-#include "arrow/json/rapidjson_defs.h" // IWYU pragma: keep
+#include "arrow/json/json_writer_internal.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/compression.h"
#include "arrow/util/decimal.h"
#include "arrow/util/float16.h"
#include "arrow/util/logging_internal.h"
-#include <rapidjson/document.h>
-#include <rapidjson/writer.h>
-
#include "parquet/exception.h"
#include "parquet/thrift_internal.h"
#include "parquet/types.h"
@@ -1785,13 +1782,9 @@ namespace {
void WriteCrsKeyAndValue(const std::string_view crs, std::ostream& json) {
// There is no restriction on the crs value here, and it may contain quotes
// or backslashes that would result in invalid JSON if unescaped.
- namespace rj = ::arrow::rapidjson;
- rj::StringBuffer buffer;
- rj::Writer<rj::StringBuffer> writer(buffer);
- rj::Value v;
- v.SetString(crs.data(), static_cast<int32_t>(crs.size()));
- v.Accept(writer);
- json << R"(, "crs": )" << buffer.GetString();
+ ::arrow::json::JsonWriter writer;
+ writer.String(crs);
+ json << R"(, "crs": )" << writer.GetString().ValueUnsafe();
}
} // namespace
diff --git a/cpp/subprojects/simdjson.wrap b/cpp/subprojects/simdjson.wrap
new file mode 100644
index 0000000000..31d9de3c27
--- /dev/null
+++ b/cpp/subprojects/simdjson.wrap
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[wrap-file]
+source_url =
https://github.com/simdjson/simdjson/archive/refs/tags/v4.6.4.tar.gz
+source_filename = simdjson-v4.6.4.tar.gz
+source_hash = b091107844fe928158c5c2265c20360fff312889ddf7ebc4528a0f0f8f2ff9cd
+directory = simdjson-v4.6.4
+method = cmake
