This is an automated email from the ASF dual-hosted git repository.
pitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 3834d24666 GH-50627: [C++] Migrate from_string.cc to simdjson (#50653)
3834d24666 is described below
commit 3834d24666f48dcb984bacecf72ee7aaddb69999
Author: Alexander Taepper <[email protected]>
AuthorDate: Tue Jul 28 19:00:10 2026 +0200
GH-50627: [C++] Migrate from_string.cc to simdjson (#50653)
### Rationale for this change
This is part of the work for #35460. Back in January I started working on
this change-set, but after discussions came up regarding the use of the
`simdjson::dom` vs `simdjson::ondemand` front-end the development went stale. I
saw that @ Reranko05 started the work on #35460 and already made great
progress, well done!
I still wanted to put up this change-set as a possible view-point, but feel
free to disregard @ Reranko05 if you already started your own work.
### What changes are included in this PR?
This changes `from_string.cc` to use simdjson instead of RapidJSON.
### Are there any user-facing changes?
Yes, the API has changed to no longer allow non-utf8 compatible strings.
`simdjson` does not support non-compliant inputs and does not intent to support
it in the future
[[1]](https://github.com/simdjson/simdjson/issues/1691#issuecomment-895286180)
[[2]](https://github.com/simdjson/simdjson/blob/8e6bac94877f2d3d026000d36ce81e0aaf38d26f/doc/basics.md?plain=1#L3701)
[[3]](https://github.com/simdjson/simdjson/discussions/2125)
All call-sites in tests only used a transformation to non-utf8 json as an
intermediate result to easily go from `string -> Array`, so I changed these to
use different helper methods instead.
Another difficult problem arises in the handling of `NaN` and `Inf`
literals in inputs. `simdjson` follows the json standard, whereas `RapidJSON`
implemented an extension where the following is a valid input:
```
{"x": Inf, "y": -Inf, "z": NaN, "n": Infinity, "m": -Infinity}
```
We need to decide whether to be breaking here or not. This PR now contains
a non-breaking version of the change. This can be quite elegantly added by
using the public `ondemand` api's function to retrieve the raw token string in
the `ConvertNumber` function.
Alternatively, we can wait for the next `simdjson` major release, which
will ship with support for a `SIMDJSON_ENABLE_NAN_INF` compilation flag as per
[this pull request](https://github.com/simdjson/simdjson/pull/2696). (this was
only merged to simdjson main)
### Are these changes tested?
Yes
* GitHub Issue: #50627
Authored-by: Alexander Taepper <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>
---
cpp/src/arrow/compute/kernels/scalar_cast_test.cc | 23 +-
.../arrow/compute/kernels/scalar_string_test.cc | 81 ++-
cpp/src/arrow/extension/json_test.cc | 4 +-
cpp/src/arrow/json/from_string.cc | 713 +++++++++++++--------
cpp/src/arrow/json/from_string_test.cc | 18 +-
cpp/src/arrow/scalar_test.cc | 7 +-
cpp/src/arrow/testing/builder.h | 34 +
cpp/src/parquet/arrow/arrow_reader_writer_test.cc | 27 +-
python/pyarrow/src/arrow/python/gdb.cc | 46 +-
9 files changed, 596 insertions(+), 357 deletions(-)
diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_test.cc
b/cpp/src/arrow/compute/kernels/scalar_cast_test.cc
index 51e6ca534c..364a4bd436 100644
--- a/cpp/src/arrow/compute/kernels/scalar_cast_test.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_cast_test.cc
@@ -31,6 +31,7 @@
#include "arrow/chunked_array.h"
#include "arrow/extension_type.h"
#include "arrow/status.h"
+#include "arrow/testing/builder.h"
#include "arrow/testing/extension_type.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/testing/random.h"
@@ -56,16 +57,7 @@ using internal::checked_pointer_cast;
namespace compute {
static std::shared_ptr<Array> InvalidUtf8(std::shared_ptr<DataType> type) {
- return ArrayFromJSON(type,
- "["
- R"(
- "Hi",
- "olá mundo",
- "你好世界",
- "",
- )"
- "\"\xa0\xa1\""
- "]");
+ return BinaryArrayFromStrings(type, {"Hi", "olá mundo", "你好世界", "",
"\xa0\xa1"});
}
static std::shared_ptr<Array> FixedSizeInvalidUtf8(std::shared_ptr<DataType>
type) {
@@ -73,16 +65,7 @@ static std::shared_ptr<Array>
FixedSizeInvalidUtf8(std::shared_ptr<DataType> typ
// Assume a particular width for testing
EXPECT_EQ(3, checked_cast<const FixedSizeBinaryType&>(*type).byte_width());
}
- return ArrayFromJSON(type,
- "["
- R"(
- "Hi!",
- "lá",
- "你",
- " ",
- )"
- "\"\xa0\xa1\xa2\""
- "]");
+ return BinaryArrayFromStrings(type, {"Hi!", "lá", "你", " ",
"\xa0\xa1\xa2"});
}
static std::vector<std::shared_ptr<DataType>> kNumericTypes = {
diff --git a/cpp/src/arrow/compute/kernels/scalar_string_test.cc
b/cpp/src/arrow/compute/kernels/scalar_string_test.cc
index bf5469ccd8..4969beada6 100644
--- a/cpp/src/arrow/compute/kernels/scalar_string_test.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_string_test.cc
@@ -1217,7 +1217,7 @@ TYPED_TEST(TestStringKernels, Utf8Reverse) {
// inputs with malformed utf8 chars would produce garbage output, but the
end result
// would produce arrays with same lengths. Hence checking offset buffer
equality
- auto malformed_input = ArrayFromJSON(this->type(), "[\"ɑ\xFFɑa\",
\"ɽ\xe1\xbdɽa\"]");
+ auto malformed_input = this->MakeArray({"ɑ\xFFɑa", "ɽ\xe1\xbdɽa"});
const Result<Datum>& res = CallFunction("utf8_reverse", {malformed_input});
ASSERT_TRUE(res->array()->buffers[1]->Equals(*malformed_input->data()->buffers[1]));
}
@@ -1340,7 +1340,7 @@ TYPED_TEST(TestStringKernels, Utf8Upper) {
this->CheckUnary("utf8_upper", "[\"ɑɑɑɑ\"]", this->type(), "[\"ⱭⱭⱭⱭ\"]");
// Test invalid data
- auto invalid_input = ArrayFromJSON(this->type(), "[\"ɑa\xFFɑ\",
\"ɽ\xe1\xbdɽaa\"]");
+ auto invalid_input = this->MakeArray({"ɑa\xFFɑ", "ɽ\xe1\xbdɽaa"});
EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("Invalid UTF8
sequence"),
CallFunction("utf8_upper", {invalid_input}));
}
@@ -1362,7 +1362,7 @@ TYPED_TEST(TestStringKernels, Utf8Lower) {
this->CheckUnary("utf8_lower", "[\"ȺȺȺȺ\"]", this->type(), "[\"ⱥⱥⱥⱥ\"]");
// Test invalid data
- auto invalid_input = ArrayFromJSON(this->type(), "[\"Ⱥa\xFFⱭ\",
\"Ɽ\xe1\xbdⱤaA\"]");
+ auto invalid_input = this->MakeArray({"Ⱥa\xFFⱭ", "Ɽ\xe1\xbdⱤaA"});
EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("Invalid UTF8
sequence"),
CallFunction("utf8_lower", {invalid_input}));
}
@@ -1382,7 +1382,7 @@ TYPED_TEST(TestStringKernels, Utf8SwapCase) {
"[\"HeLLo, wOrLD!\", \"$. a35?\"]");
// Test invalid data
- auto invalid_input = ArrayFromJSON(this->type(), "[\"Ⱥa\xFFⱭ\",
\"Ɽ\xe1\xbdⱤaA\"]");
+ auto invalid_input = this->MakeArray({"Ⱥa\xFFⱭ", "Ɽ\xe1\xbdⱤaA"});
EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("Invalid UTF8
sequence"),
CallFunction("utf8_swapcase",
{invalid_input}));
}
@@ -2672,101 +2672,92 @@ TYPED_TEST(TestBinaryKernels, SliceBytesPosPos) {
SliceOptions options{2, 4};
this->CheckUnary(
"binary_slice",
- "[\"\", \"a\", \"ab\", \"a\xc2\xa2\", \"ab\xc2\xa2\", \"ab\xc2\xffZ\"]",
- this->type(), "[\"\", \"\", \"\", \"\xa2\", \"\xc2\xa2\",
\"\xc2\xff\"]", &options);
+ this->MakeArray({"", "a", "ab", "a\xc2\xa2", "ab\xc2\xa2",
"ab\xc2\xffZ"}),
+ this->MakeArray({"", "", "", "\xa2", "\xc2\xa2", "\xc2\xff"}), &options);
SliceOptions options_step{1, 5, 2};
this->CheckUnary(
"binary_slice",
- "[\"\", \"a\", \"ab\", \"a\xc2\xa2\", \"ab\xc2\xa2\", \"ab\xc2\xffZ\"]",
- this->type(), "[\"\", \"\", \"b\", \"\xc2\", \"b\xa2\", \"b\xff\"]",
&options_step);
+ this->MakeArray({"", "a", "ab", "a\xc2\xa2", "ab\xc2\xa2",
"ab\xc2\xffZ"}),
+ this->MakeArray({"", "", "b", "\xc2", "b\xa2", "b\xff"}), &options_step);
SliceOptions options_step_neg{5, 1, -2};
this->CheckUnary(
"binary_slice",
- "[\"\", \"a\", \"ab\", \"a\xc2\xa2\", \"ab\xc2\xa2\", \"ab\xc2\xffZ\"]",
- this->type(), "[\"\", \"\", \"\", \"\xa2\", \"\xa2\", \"Z\xc2\"]",
- &options_step_neg);
+ this->MakeArray({"", "a", "ab", "a\xc2\xa2", "ab\xc2\xa2",
"ab\xc2\xffZ"}),
+ this->MakeArray({"", "", "", "\xa2", "\xa2", "Z\xc2"}),
&options_step_neg);
options_step_neg.stop = 0;
this->CheckUnary(
"binary_slice",
- "[\"\", \"a\", \"ab\", \"a\xc2\xa2\", \"aZ\xc2\xa2\", \"ab\xc2\xffZ\"]",
- this->type(), "[\"\", \"\", \"b\", \"\xa2\", \"\xa2Z\", \"Z\xc2\"]",
- &options_step_neg);
+ this->MakeArray({"", "a", "ab", "a\xc2\xa2", "aZ\xc2\xa2",
"ab\xc2\xffZ"}),
+ this->MakeArray({"", "", "b", "\xa2", "\xa2Z", "Z\xc2"}),
&options_step_neg);
}
TYPED_TEST(TestBinaryKernels, SliceBytesPosNeg) {
SliceOptions options{2, -1};
this->CheckUnary(
"binary_slice",
- "[\"\", \"a\", \"ab\", \"a\xc2\xa2\", \"aZ\xc2\xa2\", \"ab\xc2\xffZ\"]",
- this->type(), "[\"\", \"\", \"\", \"\", \"\xc2\", \"\xc2\xff\"]",
&options);
+ this->MakeArray({"", "a", "ab", "a\xc2\xa2", "aZ\xc2\xa2",
"ab\xc2\xffZ"}),
+ this->MakeArray({"", "", "", "", "\xc2", "\xc2\xff"}), &options);
SliceOptions options_step{1, -1, 2};
this->CheckUnary(
"binary_slice",
- "[\"\", \"a\", \"ab\", \"a\xc2\xa2\", \"aZ\xc2\xa2\", \"ab\xc2\xffZ\"]",
- this->type(), "[\"\", \"\", \"\", \"\xc2\", \"Z\", \"b\xff\"]",
&options_step);
+ this->MakeArray({"", "a", "ab", "a\xc2\xa2", "aZ\xc2\xa2",
"ab\xc2\xffZ"}),
+ this->MakeArray({"", "", "", "\xc2", "Z", "b\xff"}), &options_step);
SliceOptions options_step_neg{3, -4, -2};
this->CheckUnary(
"binary_slice",
- "[\"\", \"a\", \"ab\", \"Z\xc2\xa2\", \"aZ\xc2\xa2\", \"ab\xc2\xffZ\"]",
- this->type(), "[\"\", \"a\", \"b\", \"\xa2Z\", \"\xa2Z\", \"\xff\"]",
- &options_step_neg);
+ this->MakeArray({"", "a", "ab", "Z\xc2\xa2", "aZ\xc2\xa2",
"ab\xc2\xffZ"}),
+ this->MakeArray({"", "a", "b", "\xa2Z", "\xa2Z", "\xff"}),
&options_step_neg);
options_step_neg.stop = -5;
this->CheckUnary(
"binary_slice",
- "[\"\", \"a\", \"ab\", \"Z\xc2\xa2\", \"aZ\xc2\xa2\", \"aP\xc2\xffZ\"]",
- this->type(), "[\"\", \"a\", \"b\", \"\xa2Z\", \"\xa2Z\", \"\xffP\"]",
- &options_step_neg);
+ this->MakeArray({"", "a", "ab", "Z\xc2\xa2", "aZ\xc2\xa2",
"aP\xc2\xffZ"}),
+ this->MakeArray({"", "a", "b", "\xa2Z", "\xa2Z", "\xffP"}),
&options_step_neg);
}
TYPED_TEST(TestBinaryKernels, SliceBytesNegNeg) {
SliceOptions options{-2, -1};
this->CheckUnary(
"binary_slice",
- "[\"\", \"a\", \"ab\", \"Z\xc2\xa2\", \"aZ\xc2\xa2\", \"ab\xc2\xffZ\"]",
- this->type(), "[\"\", \"\", \"a\", \"\xc2\", \"\xc2\", \"\xff\"]",
&options);
+ this->MakeArray({"", "a", "ab", "Z\xc2\xa2", "aZ\xc2\xa2",
"ab\xc2\xffZ"}),
+ this->MakeArray({"", "", "a", "\xc2", "\xc2", "\xff"}), &options);
SliceOptions options_step{-4, -1, 2};
this->CheckUnary(
"binary_slice",
- "[\"\", \"a\", \"ab\", \"Z\xc2\xa2\", \"aZ\xc2\xa2\", \"aP\xc2\xffZ\"]",
- this->type(), "[\"\", \"\", \"a\", \"Z\", \"a\xc2\", \"P\xff\"]",
&options_step);
+ this->MakeArray({"", "a", "ab", "Z\xc2\xa2", "aZ\xc2\xa2",
"aP\xc2\xffZ"}),
+ this->MakeArray({"", "", "a", "Z", "a\xc2", "P\xff"}), &options_step);
SliceOptions options_step_neg{-1, -3, -2};
this->CheckUnary(
"binary_slice",
- "[\"\", \"a\", \"ab\", \"Z\xc2\xa2\", \"aZ\xc2\xa2\", \"aP\xc2\xffZ\"]",
- this->type(), "[\"\", \"a\", \"b\", \"\xa2\", \"\xa2\", \"Z\"]",
&options_step_neg);
+ this->MakeArray({"", "a", "ab", "Z\xc2\xa2", "aZ\xc2\xa2",
"aP\xc2\xffZ"}),
+ this->MakeArray({"", "a", "b", "\xa2", "\xa2", "Z"}), &options_step_neg);
options_step_neg.stop = -4;
this->CheckUnary(
"binary_slice",
- "[\"\", \"a\", \"ab\", \"Z\xc2\xa2\", \"aZ\xc2\xa2\", \"aP\xc2\xffZ\"]",
- this->type(), "[\"\", \"a\", \"b\", \"\xa2Z\", \"\xa2Z\", \"Z\xc2\"]",
- &options_step_neg);
+ this->MakeArray({"", "a", "ab", "Z\xc2\xa2", "aZ\xc2\xa2",
"aP\xc2\xffZ"}),
+ this->MakeArray({"", "a", "b", "\xa2Z", "\xa2Z", "Z\xc2"}),
&options_step_neg);
}
TYPED_TEST(TestBinaryKernels, SliceBytesNegPos) {
SliceOptions options{-2, 4};
this->CheckUnary(
"binary_slice",
- "[\"\", \"a\", \"ab\", \"Z\xc2\xa2\", \"aZ\xc2\xa2\", \"aP\xc2\xffZ\"]",
- this->type(), "[\"\", \"a\", \"ab\", \"\xc2\xa2\", \"\xc2\xa2\",
\"\xff\"]",
- &options);
+ this->MakeArray({"", "a", "ab", "Z\xc2\xa2", "aZ\xc2\xa2",
"aP\xc2\xffZ"}),
+ this->MakeArray({"", "a", "ab", "\xc2\xa2", "\xc2\xa2", "\xff"}),
&options);
SliceOptions options_step{-4, 4, 2};
this->CheckUnary(
"binary_slice",
- "[\"\", \"a\", \"ab\", \"Z\xc2\xa2\", \"aZ\xc2\xa2\", \"aP\xc2\xffZ\"]",
- this->type(), "[\"\", \"a\", \"a\", \"Z\xa2\", \"a\xc2\", \"P\xff\"]",
- &options_step);
+ this->MakeArray({"", "a", "ab", "Z\xc2\xa2", "aZ\xc2\xa2",
"aP\xc2\xffZ"}),
+ this->MakeArray({"", "a", "a", "Z\xa2", "a\xc2", "P\xff"}),
&options_step);
SliceOptions options_step_neg{-1, 1, -2};
this->CheckUnary(
"binary_slice",
- "[\"\", \"a\", \"ab\", \"Z\xc2\xa2\", \"aZ\xc2\xa2\", \"aP\xc2\xffZ\"]",
- this->type(), "[\"\", \"\", \"\", \"\xa2\", \"\xa2\", \"Z\xc2\"]",
- &options_step_neg);
+ this->MakeArray({"", "a", "ab", "Z\xc2\xa2", "aZ\xc2\xa2",
"aP\xc2\xffZ"}),
+ this->MakeArray({"", "", "", "\xa2", "\xa2", "Z\xc2"}),
&options_step_neg);
options_step_neg.stop = 0;
this->CheckUnary(
"binary_slice",
- "[\"\", \"a\", \"ab\", \"Z\xc2\xa2\", \"aZ\xc2\xa2\", \"aP\xc2\xffZ\"]",
- this->type(), "[\"\", \"\", \"b\", \"\xa2\", \"\xa2Z\", \"Z\xc2\"]",
- &options_step_neg);
+ this->MakeArray({"", "a", "ab", "Z\xc2\xa2", "aZ\xc2\xa2",
"aP\xc2\xffZ"}),
+ this->MakeArray({"", "", "b", "\xa2", "\xa2Z", "Z\xc2"}),
&options_step_neg);
}
TYPED_TEST(TestStringKernels, PadAscii) {
diff --git a/cpp/src/arrow/extension/json_test.cc
b/cpp/src/arrow/extension/json_test.cc
index b938ddb2cf..ae0b34db38 100644
--- a/cpp/src/arrow/extension/json_test.cc
+++ b/cpp/src/arrow/extension/json_test.cc
@@ -20,6 +20,7 @@
#include "arrow/array/validate.h"
#include "arrow/ipc/test_common.h"
#include "arrow/record_batch.h"
+#include "arrow/testing/builder.h"
#include "arrow/testing/gtest_util.h"
#include "parquet/exception.h"
@@ -64,7 +65,8 @@ TEST_F(TestJsonExtensionType, JsonRoundtrip) {
TEST_F(TestJsonExtensionType, InvalidUTF8) {
for (const auto& storage_type : {utf8(), large_utf8(), utf8_view()}) {
auto json_type = json(storage_type);
- auto invalid_input = ArrayFromJSON(storage_type, "[\"Ⱥa\xFFⱭ\",
\"Ɽ\xe1\xbdⱤaA\"]");
+ auto invalid_input =
+ BinaryArrayFromStrings(storage_type, {"Ⱥa\xFFⱭ", "Ɽ\xe1\xbdⱤaA"});
auto ext_arr = ExtensionType::WrapArray(json_type, invalid_input);
ASSERT_RAISES_WITH_MESSAGE(Invalid,
diff --git a/cpp/src/arrow/json/from_string.cc
b/cpp/src/arrow/json/from_string.cc
index e35a362f5a..9cb14a92a6 100644
--- a/cpp/src/arrow/json/from_string.cc
+++ b/cpp/src/arrow/json/from_string.cc
@@ -15,10 +15,16 @@
// specific language governing permissions and limitations
// under the License.
+#include <array>
+#include <cctype>
+#include <cmath>
#include <cstdint>
+#include <functional>
#include <sstream>
+#include <string>
#include <string_view>
#include <type_traits>
+#include <unordered_map>
#include <utility>
#include <vector>
@@ -32,23 +38,21 @@
#include "arrow/array/builder_union.h"
#include "arrow/chunked_array.h"
#include "arrow/json/from_string.h"
+#include "arrow/result.h"
#include "arrow/scalar.h"
+#include "arrow/status.h"
+#include "arrow/type.h"
#include "arrow/type_traits.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/decimal.h"
#include "arrow/util/float16.h"
#include "arrow/util/logging_internal.h"
+#include "arrow/util/unreachable.h"
#include "arrow/util/value_parsing.h"
-#include "arrow/json/rapidjson_defs.h"
+#include <simdjson.h>
-#include <rapidjson/document.h>
-#include <rapidjson/error/en.h>
-#include <rapidjson/rapidjson.h>
-#include <rapidjson/reader.h>
-#include <rapidjson/writer.h>
-
-namespace rj = arrow::rapidjson;
+namespace sj = simdjson::ondemand;
namespace arrow {
@@ -62,32 +66,104 @@ using ::arrow::internal::checked_pointer_cast;
namespace {
-constexpr auto kParseFlags = rj::kParseFullPrecisionFlag |
rj::kParseNanAndInfFlag;
-
-const char* JsonTypeName(rj::Type json_type) {
- switch (json_type) {
- case rapidjson::kNullType:
- return "null";
- case rapidjson::kFalseType:
- return "false";
- case rapidjson::kTrueType:
- return "true";
- case rapidjson::kObjectType:
- return "object";
- case rapidjson::kArrayType:
+const char* JsonTypeName(sj::json_type type) {
+ switch (type) {
+ case sj::json_type::array:
return "array";
- case rapidjson::kStringType:
- return "string";
- case rapidjson::kNumberType:
+ case sj::json_type::object:
+ return "object";
+ case sj::json_type::number:
return "number";
+ case sj::json_type::string:
+ return "string";
+ case sj::json_type::boolean:
+ return "boolean";
+ case sj::json_type::null:
+ return "null";
default:
return "unknown";
}
}
-Status JSONTypeError(const char* expected_type, rj::Type json_type) {
- return Status::Invalid("Expected ", expected_type, " or null, got JSON type
",
- JsonTypeName(json_type));
+// Empty struct to represent the type of a simdjson null value
+struct SimdjsonNull {};
+
+template <typename T>
+struct JsonTypeNameOf;
+
+template <>
+struct JsonTypeNameOf<sj::array> {
+ static constexpr const char* kValue = "array";
+};
+template <>
+struct JsonTypeNameOf<sj::object> {
+ static constexpr const char* kValue = "object";
+};
+template <>
+struct JsonTypeNameOf<std::string_view> {
+ static constexpr const char* kValue = "string";
+};
+template <>
+struct JsonTypeNameOf<bool> {
+ static constexpr const char* kValue = "boolean";
+};
+template <>
+struct JsonTypeNameOf<SimdjsonNull> {
+ static constexpr const char* kValue = "null";
+};
+template <>
+struct JsonTypeNameOf<int64_t> {
+ static constexpr const char* kValue = "number";
+};
+template <>
+struct JsonTypeNameOf<uint64_t> {
+ static constexpr const char* kValue = "number";
+};
+template <>
+struct JsonTypeNameOf<double> {
+ static constexpr const char* kValue = "number";
+};
+
+template <typename T>
+constexpr const char* JsonTypeName() {
+ return JsonTypeNameOf<T>::kValue;
+}
+
+template <typename SimdjsonValueType>
+Result<SimdjsonValueType> GetJsonAs(sj::value& value) {
+ SimdjsonValueType typed_value{};
+ simdjson::error_code error_code;
+ if constexpr (std::is_same_v<SimdjsonValueType, SimdjsonNull>) {
+ // simdjson has no get<>() for null; probe it explicitly
+ bool is_null;
+ error_code = value.is_null().get(is_null);
+ if (error_code == simdjson::SUCCESS && !is_null) {
+ error_code = simdjson::INCORRECT_TYPE;
+ }
+ } else {
+ error_code = value.get(typed_value);
+ }
+ if (error_code != simdjson::SUCCESS) {
+ sj::json_type json_type;
+ if (value.type().get(json_type) != simdjson::SUCCESS) {
+ return Status::Invalid("Expected ", JsonTypeName<SimdjsonValueType>(),
+ " or null, got malformed JSON value");
+ }
+ return Status::Invalid("Expected ", JsonTypeName<SimdjsonValueType>(),
+ " or null, got JSON type ",
JsonTypeName(json_type));
+ }
+ return typed_value;
+}
+
+template <typename SimdjsonValueType>
+Result<SimdjsonValueType> GetJsonResult(
+ simdjson::simdjson_result<SimdjsonValueType> element, std::string_view
error) {
+ SimdjsonValueType typed_value;
+ if (auto error_code = std::move(element).get(typed_value);
+ error_code != simdjson::SUCCESS) {
+ return Status::Invalid(error, simdjson::error_message(error_code));
+ }
+ return typed_value;
}
class JSONConverter {
@@ -96,11 +172,11 @@ class JSONConverter {
virtual Status Init() { return Status::OK(); }
- virtual Status AppendValue(const rj::Value& json_obj) = 0;
+ virtual Status AppendValue(sj::value& json_obj) = 0;
Status AppendNull() { return this->builder()->AppendNull(); }
- virtual Status AppendValues(const rj::Value& json_array) = 0;
+ virtual Result<int32_t> AppendValues(sj::array& json_array) = 0;
virtual std::shared_ptr<ArrayBuilder> builder() = 0;
@@ -124,20 +200,17 @@ Status GetConverter(const std::shared_ptr<DataType>&,
template <class Derived>
class ConcreteConverter : public JSONConverter {
public:
- Result<int64_t> SizeOfJSONArray(const rj::Value& json_obj) {
- if (!json_obj.IsArray()) {
- return JSONTypeError("array", json_obj.GetType());
- }
- return json_obj.Size();
- }
-
- Status AppendValues(const rj::Value& json_array) final {
+ Result<int32_t> AppendValues(sj::array& json_array) final {
auto self = static_cast<Derived*>(this);
- ARROW_ASSIGN_OR_RAISE(auto size, SizeOfJSONArray(json_array));
- for (uint32_t i = 0; i < size; ++i) {
- RETURN_NOT_OK(self->AppendValue(json_array[i]));
+ int32_t num_elements = 0;
+ for (auto element : json_array) {
+ ARROW_ASSIGN_OR_RAISE(auto value,
+ GetJsonResult<sj::value>(
+ element, "Could not iterate elements of JSON
array: "));
+ RETURN_NOT_OK(self->AppendValue(value));
+ num_elements++;
}
- return Status::OK();
+ return num_elements;
}
const std::shared_ptr<DataType>& value_type() {
@@ -167,11 +240,9 @@ class NullConverter final : public
ConcreteConverter<NullConverter> {
builder_ = std::make_shared<NullBuilder>();
}
- Status AppendValue(const rj::Value& json_obj) override {
- if (json_obj.IsNull()) {
- return AppendNull();
- }
- return JSONTypeError("null", json_obj.GetType());
+ Status AppendValue(sj::value& json_obj) override {
+ ARROW_RETURN_NOT_OK(GetJsonAs<SimdjsonNull>(json_obj));
+ return AppendNull();
}
std::shared_ptr<ArrayBuilder> builder() override { return builder_; }
@@ -190,17 +261,16 @@ class BooleanConverter final : public
ConcreteConverter<BooleanConverter> {
builder_ = std::make_shared<BooleanBuilder>();
}
- Status AppendValue(const rj::Value& json_obj) override {
- if (json_obj.IsNull()) {
+ Status AppendValue(sj::value& json_obj) override {
+ if (json_obj.is_null()) {
return AppendNull();
}
- if (json_obj.IsBool()) {
- return builder_->Append(json_obj.GetBool());
+ int64_t int_value;
+ if (json_obj.get(int_value) == simdjson::SUCCESS) {
+ return builder_->Append(int_value != 0);
}
- if (json_obj.IsInt()) {
- return builder_->Append(json_obj.GetInt() != 0);
- }
- return JSONTypeError("boolean", json_obj.GetType());
+ ARROW_ASSIGN_OR_RAISE(bool bool_value, GetJsonAs<bool>(json_obj));
+ return builder_->Append(bool_value);
}
std::shared_ptr<ArrayBuilder> builder() override { return builder_; }
@@ -214,78 +284,119 @@ class BooleanConverter final : public
ConcreteConverter<BooleanConverter> {
// Convert single signed integer value (also {Date,Time}{32,64} and Timestamp)
template <typename T>
-enable_if_physical_signed_integer<T, Status> ConvertNumber(const rj::Value&
json_obj,
+enable_if_physical_signed_integer<T, Status> ConvertNumber(sj::value& json_obj,
const DataType&
type,
typename T::c_type*
out) {
- if (json_obj.IsInt64()) {
- int64_t v64 = json_obj.GetInt64();
- *out = static_cast<typename T::c_type>(v64);
- if (*out == v64) {
- return Status::OK();
- } else {
- return Status::Invalid("Value ", v64, " out of bounds for ", type);
- }
+ *out = static_cast<typename T::c_type>(0);
+ ARROW_ASSIGN_OR_RAISE(int64_t v64, GetJsonAs<int64_t>(json_obj));
+ *out = static_cast<typename T::c_type>(v64);
+ if (*out == v64) {
+ return Status::OK();
} else {
- *out = static_cast<typename T::c_type>(0);
- return JSONTypeError("signed int", json_obj.GetType());
+ return Status::Invalid("Value ", v64, " out of bounds for ", type);
}
}
// Convert single unsigned integer value
template <typename T>
-enable_if_unsigned_integer<T, Status> ConvertNumber(const rj::Value& json_obj,
+enable_if_unsigned_integer<T, Status> ConvertNumber(sj::value& json_obj,
const DataType& type,
typename T::c_type* out) {
- if (json_obj.IsUint64()) {
- uint64_t v64 = json_obj.GetUint64();
- *out = static_cast<typename T::c_type>(v64);
- if (*out == v64) {
- return Status::OK();
- } else {
- return Status::Invalid("Value ", v64, " out of bounds for ", type);
- }
+ *out = static_cast<typename T::c_type>(0);
+ ARROW_ASSIGN_OR_RAISE(uint64_t v64, GetJsonAs<uint64_t>(json_obj));
+ *out = static_cast<typename T::c_type>(v64);
+ if (*out == v64) {
+ return Status::OK();
} else {
- *out = static_cast<typename T::c_type>(0);
- return JSONTypeError("unsigned int", json_obj.GetType());
+ return Status::Invalid("Value ", v64, " out of bounds for ", type);
+ }
+}
+
+// Match the std::string_view against NaN, Inf, Infinity with optional leading
minus
+std::optional<double> NonFiniteDoubleFromString(std::string_view str) {
+ if (str == "NaN") {
+ return std::numeric_limits<double>::quiet_NaN();
+ } else if (str == "-NaN") {
+ return -std::numeric_limits<double>::quiet_NaN();
+ } else if (str == "Inf" || str == "Infinity") {
+ return std::numeric_limits<double>::infinity();
+ } else if (str == "-Inf" || str == "-Infinity") {
+ return -std::numeric_limits<double>::infinity();
+ } else {
+ return std::nullopt;
+ }
+}
+
+std::optional<double> NonFiniteDoubleFromRawToken(sj::value& json_obj) {
+ std::string_view token = json_obj.raw_json_token();
+ // The raw token includes any trailing whitespace up to the next token
+ while (!token.empty() && std::isspace(static_cast<unsigned
char>(token.back()))) {
+ token.remove_suffix(1);
}
+ return NonFiniteDoubleFromString(token);
}
// Convert float16/HalfFloatType
template <typename T>
-enable_if_half_float<T, Status> ConvertNumber(const rj::Value& json_obj,
- const DataType& type, uint16_t*
out) {
- if (json_obj.IsDouble()) {
- double f64 = json_obj.GetDouble();
- *out = Float16(f64).bits();
- return Status::OK();
- } else if (json_obj.IsUint()) {
- uint32_t u32t = json_obj.GetUint();
- double f64 = static_cast<double>(u32t);
- *out = Float16(f64).bits();
- return Status::OK();
- } else if (json_obj.IsInt()) {
- int32_t i32t = json_obj.GetInt();
- double f64 = static_cast<double>(i32t);
- *out = Float16(f64).bits();
+enable_if_half_float<T, Status> ConvertNumber(sj::value& json_obj, const
DataType& type,
+ uint16_t* out) {
+ *out = static_cast<uint16_t>(0);
+ if (auto f64 = NonFiniteDoubleFromRawToken(json_obj); f64.has_value()) {
+ *out = Float16(f64.value()).bits();
return Status::OK();
- } else {
- *out = static_cast<uint16_t>(0);
- return JSONTypeError("unsigned int", json_obj.GetType());
}
+ ARROW_ASSIGN_OR_RAISE(auto f64, GetJsonAs<double>(json_obj));
+ *out = Float16(f64).bits();
+ return arrow::Status::OK();
}
// Convert single floating point value
template <typename T>
-enable_if_physical_floating_point<T, Status> ConvertNumber(const rj::Value&
json_obj,
+enable_if_physical_floating_point<T, Status> ConvertNumber(sj::value& json_obj,
const DataType&
type,
typename T::c_type*
out) {
- if (json_obj.IsNumber()) {
- *out = static_cast<typename T::c_type>(json_obj.GetDouble());
+ *out = static_cast<typename T::c_type>(0);
+ if (auto f64 = NonFiniteDoubleFromRawToken(json_obj); f64.has_value()) {
+ *out = static_cast<typename T::c_type>(f64.value());
return Status::OK();
- } else {
- *out = static_cast<typename T::c_type>(0);
- return JSONTypeError("number", json_obj.GetType());
}
+ ARROW_ASSIGN_OR_RAISE(auto f64, GetJsonAs<double>(json_obj));
+ *out = static_cast<typename T::c_type>(f64);
+ return arrow::Status::OK();
+}
+
+// ------------------------------------------------------------------------
+// Helper to process a JSON array with exactly kExpectedSize elements, calling
+// one handler per element, in order.
+using JsonElementHandler = std::function<Status(sj::value&)>;
+
+template <size_t kExpectedSize>
+Status ProcessJsonArrayElements(
+ sj::array& json_array, const char* error_context,
+ const std::array<JsonElementHandler, kExpectedSize>& handlers) {
+ auto it = json_array.begin();
+ auto end = json_array.end();
+
+ size_t index = 0;
+ for (const auto& handler : handlers) {
+ if (it == end) {
+ return Status::Invalid(error_context, " must have exactly ",
kExpectedSize,
+ " elements, had ", index);
+ }
+
+ ARROW_ASSIGN_OR_RAISE(
+ sj::value element,
+ GetJsonResult<sj::value>(*it, "Could not iterate elements of JSON
array: "));
+ RETURN_NOT_OK(handler(element));
+ ++it;
+ ++index;
+ }
+
+ if (it != end) {
+ return Status::Invalid(error_context, " must have exactly ", kExpectedSize,
+ " elements, had more");
+ }
+ return Status::OK();
}
// ------------------------------------------------------------------------
@@ -303,8 +414,8 @@ class IntegerConverter final
Status Init() override { return this->MakeConcreteBuilder(&builder_); }
- Status AppendValue(const rj::Value& json_obj) override {
- if (json_obj.IsNull()) {
+ Status AppendValue(sj::value& json_obj) override {
+ if (json_obj.is_null()) {
return this->AppendNull();
}
c_type value;
@@ -330,8 +441,8 @@ class FloatConverter final : public
ConcreteConverter<FloatConverter<Type, Build
Status Init() override { return this->MakeConcreteBuilder(&builder_); }
- Status AppendValue(const rj::Value& json_obj) override {
- if (json_obj.IsNull()) {
+ Status AppendValue(sj::value& json_obj) override {
+ if (json_obj.is_null()) {
return this->AppendNull();
}
c_type value;
@@ -360,22 +471,19 @@ class DecimalConverter final
Status Init() override { return this->MakeConcreteBuilder(&builder_); }
- Status AppendValue(const rj::Value& json_obj) override {
- if (json_obj.IsNull()) {
+ Status AppendValue(sj::value& json_obj) override {
+ if (json_obj.is_null()) {
return this->AppendNull();
}
- if (json_obj.IsString()) {
- int32_t precision, scale;
- DecimalValue d;
- auto view = std::string_view(json_obj.GetString(),
json_obj.GetStringLength());
- RETURN_NOT_OK(DecimalValue::FromString(view, &d, &precision, &scale));
- if (scale != decimal_type_->scale()) {
- return Status::Invalid("Invalid scale for decimal: expected ",
- decimal_type_->scale(), ", got ", scale);
- }
- return builder_->Append(d);
+ ARROW_ASSIGN_OR_RAISE(auto string_value,
GetJsonAs<std::string_view>(json_obj));
+ int32_t precision, scale;
+ DecimalValue d;
+ RETURN_NOT_OK(DecimalValue::FromString(string_value, &d, &precision,
&scale));
+ if (scale != decimal_type_->scale()) {
+ return Status::Invalid("Invalid scale for decimal: expected ",
+ decimal_type_->scale(), ", got ", scale);
}
- return JSONTypeError("decimal string", json_obj.GetType());
+ return builder_->Append(d);
}
std::shared_ptr<ArrayBuilder> builder() override { return builder_; }
@@ -405,20 +513,18 @@ class TimestampConverter final : public
ConcreteConverter<TimestampConverter> {
builder_ = std::make_shared<TimestampBuilder>(type, default_memory_pool());
}
- Status AppendValue(const rj::Value& json_obj) override {
- if (json_obj.IsNull()) {
+ Status AppendValue(sj::value& json_obj) override {
+ if (json_obj.is_null()) {
return this->AppendNull();
}
int64_t value;
- if (json_obj.IsNumber()) {
- RETURN_NOT_OK(ConvertNumber<Int64Type>(json_obj, *this->type_, &value));
- } else if (json_obj.IsString()) {
- std::string_view view(json_obj.GetString(), json_obj.GetStringLength());
+ std::string_view view;
+ if (json_obj.get(view) == simdjson::SUCCESS) {
if (!ParseValue(*timestamp_type_, view.data(), view.size(), &value)) {
return Status::Invalid("couldn't parse timestamp from ", view);
}
} else {
- return JSONTypeError("timestamp", json_obj.GetType());
+ RETURN_NOT_OK(ConvertNumber<Int64Type>(json_obj, *this->type_, &value));
}
return builder_->Append(value);
}
@@ -441,21 +547,22 @@ class DayTimeIntervalConverter final
builder_ = std::make_shared<DayTimeIntervalBuilder>(default_memory_pool());
}
- Status AppendValue(const rj::Value& json_obj) override {
- if (json_obj.IsNull()) {
+ Status AppendValue(sj::value& json_obj) override {
+ if (json_obj.is_null()) {
return this->AppendNull();
}
+
+ ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs<sj::array>(json_obj));
+
DayTimeIntervalType::DayMilliseconds value;
- if (!json_obj.IsArray()) {
- return JSONTypeError("array", json_obj.GetType());
- }
- if (json_obj.Size() != 2) {
- return Status::Invalid(
- "day time interval pair must have exactly two elements, had ",
json_obj.Size());
- }
- RETURN_NOT_OK(ConvertNumber<Int32Type>(json_obj[0], *this->type_,
&value.days));
- RETURN_NOT_OK(
- ConvertNumber<Int32Type>(json_obj[1], *this->type_,
&value.milliseconds));
+ RETURN_NOT_OK(ProcessJsonArrayElements<2>(
+ array, "day-time interval",
+ {[this, &value](sj::value& elem) {
+ return ConvertNumber<Int32Type>(elem, *this->type_, &value.days);
+ },
+ [this, &value](sj::value& elem) {
+ return ConvertNumber<Int32Type>(elem, *this->type_,
&value.milliseconds);
+ }}));
return builder_->Append(value);
}
@@ -473,23 +580,25 @@ class MonthDayNanoIntervalConverter final
builder_ =
std::make_shared<MonthDayNanoIntervalBuilder>(default_memory_pool());
}
- Status AppendValue(const rj::Value& json_obj) override {
- if (json_obj.IsNull()) {
+ Status AppendValue(sj::value& json_obj) override {
+ if (json_obj.is_null()) {
return this->AppendNull();
}
- MonthDayNanoIntervalType::MonthDayNanos value;
- if (!json_obj.IsArray()) {
- return JSONTypeError("array", json_obj.GetType());
- }
- if (json_obj.Size() != 3) {
- return Status::Invalid(
- "month_day_nano_interval must have exactly 3 elements, had ",
json_obj.Size());
- }
- RETURN_NOT_OK(ConvertNumber<Int32Type>(json_obj[0], *this->type_,
&value.months));
- RETURN_NOT_OK(ConvertNumber<Int32Type>(json_obj[1], *this->type_,
&value.days));
- RETURN_NOT_OK(
- ConvertNumber<Int64Type>(json_obj[2], *this->type_,
&value.nanoseconds));
+ ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs<sj::array>(json_obj));
+
+ MonthDayNanoIntervalType::MonthDayNanos value;
+ RETURN_NOT_OK(ProcessJsonArrayElements<3>(
+ array, "month-day-nano interval",
+ {[this, &value](sj::value& elem) {
+ return ConvertNumber<Int32Type>(elem, *this->type_, &value.months);
+ },
+ [this, &value](sj::value& elem) {
+ return ConvertNumber<Int32Type>(elem, *this->type_, &value.days);
+ },
+ [this, &value](sj::value& elem) {
+ return ConvertNumber<Int64Type>(elem, *this->type_,
&value.nanoseconds);
+ }}));
return builder_->Append(value);
}
@@ -510,16 +619,13 @@ class StringConverter final
Status Init() override { return this->MakeConcreteBuilder(&builder_); }
- Status AppendValue(const rj::Value& json_obj) override {
- if (json_obj.IsNull()) {
+ Status AppendValue(sj::value& json_obj) override {
+ if (json_obj.is_null()) {
return this->AppendNull();
}
- if (json_obj.IsString()) {
- auto view = std::string_view(json_obj.GetString(),
json_obj.GetStringLength());
- return builder_->Append(view);
- } else {
- return JSONTypeError("string", json_obj.GetType());
- }
+
+ ARROW_ASSIGN_OR_RAISE(auto view, GetJsonAs<std::string_view>(json_obj));
+ return builder_->Append(view);
}
std::shared_ptr<ArrayBuilder> builder() override { return builder_; }
@@ -541,22 +647,18 @@ class FixedSizeBinaryConverter final
Status Init() override { return this->MakeConcreteBuilder(&builder_); }
- Status AppendValue(const rj::Value& json_obj) override {
- if (json_obj.IsNull()) {
+ Status AppendValue(sj::value& json_obj) override {
+ if (json_obj.is_null()) {
return this->AppendNull();
}
- if (json_obj.IsString()) {
- auto view = std::string_view(json_obj.GetString(),
json_obj.GetStringLength());
- if (view.length() != static_cast<size_t>(builder_->byte_width())) {
- std::stringstream ss;
- ss << "Invalid string length " << view.length() << " in JSON input for
"
- << this->type_->ToString();
- return Status::Invalid(ss.str());
- }
- return builder_->Append(view);
- } else {
- return JSONTypeError("string", json_obj.GetType());
+ ARROW_ASSIGN_OR_RAISE(auto view, GetJsonAs<std::string_view>(json_obj));
+ if (view.length() != static_cast<size_t>(builder_->byte_width())) {
+ std::stringstream ss;
+ ss << "Invalid string length " << view.length() << " in JSON input for "
+ << this->type_->ToString();
+ return Status::Invalid(ss.str());
}
+ return builder_->Append(view);
}
std::shared_ptr<ArrayBuilder> builder() override { return builder_; }
@@ -588,14 +690,18 @@ class VarLengthListLikeConverter final
return Status::OK();
}
- Status AppendValue(const rj::Value& json_obj) override {
- if (json_obj.IsNull()) {
+ Status AppendValue(sj::value& json_obj) override {
+ if (json_obj.is_null()) {
return this->AppendNull();
}
+ ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs<sj::array>(json_obj));
+ size_t num_elements;
+ if (array.count_elements().get(num_elements) != simdjson::SUCCESS) {
+ return Status::Invalid("Malformed JSON array for type ",
this->type_->ToString());
+ }
// Extend the child converter with this JSON array
- ARROW_ASSIGN_OR_RAISE(auto size, this->SizeOfJSONArray(json_obj));
- RETURN_NOT_OK(builder_->Append(true, size));
- return child_converter_->AppendValues(json_obj);
+ ARROW_RETURN_NOT_OK(builder_->Append(true,
static_cast<int64_t>(num_elements)));
+ return child_converter_->AppendValues(array).status();
}
std::shared_ptr<ArrayBuilder> builder() override { return builder_; }
@@ -623,29 +729,29 @@ class MapConverter final : public
ConcreteConverter<MapConverter> {
return Status::OK();
}
- Status AppendValue(const rj::Value& json_obj) override {
- if (json_obj.IsNull()) {
+ Status AppendValue(sj::value& json_obj) override {
+ if (json_obj.is_null()) {
return this->AppendNull();
}
RETURN_NOT_OK(builder_->Append());
- if (!json_obj.IsArray()) {
- return JSONTypeError("array", json_obj.GetType());
- }
- auto size = json_obj.Size();
- for (uint32_t i = 0; i < size; ++i) {
- const auto& json_pair = json_obj[i];
- if (!json_pair.IsArray()) {
- return JSONTypeError("array", json_pair.GetType());
- }
- if (json_pair.Size() != 2) {
- return Status::Invalid("key item pair must have exactly two elements,
had ",
- json_pair.Size());
- }
- if (json_pair[0].IsNull()) {
- return Status::Invalid("null key is invalid");
- }
- RETURN_NOT_OK(key_converter_->AppendValue(json_pair[0]));
- RETURN_NOT_OK(item_converter_->AppendValue(json_pair[1]));
+ ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs<sj::array>(json_obj));
+
+ for (auto json_pair_result : array) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto json_pair,
+ GetJsonResult<sj::value>(json_pair_result,
+ "Could not iterate elements of JSON array:
"));
+ ARROW_ASSIGN_OR_RAISE(auto json_pair_array,
GetJsonAs<sj::array>(json_pair));
+
+ RETURN_NOT_OK(ProcessJsonArrayElements<2>(
+ json_pair_array, "key-item pair",
+ {[this](sj::value& key) {
+ if (key.is_null()) {
+ return Status::Invalid("null key is invalid");
+ }
+ return key_converter_->AppendValue(key);
+ },
+ [this](sj::value& item) { return
item_converter_->AppendValue(item); }}));
}
return Status::OK();
}
@@ -674,15 +780,16 @@ class FixedSizeListConverter final : public
ConcreteConverter<FixedSizeListConve
return Status::OK();
}
- Status AppendValue(const rj::Value& json_obj) override {
- if (json_obj.IsNull()) {
+ Status AppendValue(sj::value& json_obj) override {
+ if (json_obj.is_null()) {
return this->AppendNull();
}
RETURN_NOT_OK(builder_->Append());
// Extend the child converter with this JSON array
- RETURN_NOT_OK(child_converter_->AppendValues(json_obj));
- if (json_obj.GetArray().Size() != static_cast<rj::SizeType>(list_size_)) {
- return Status::Invalid("incorrect list size ",
json_obj.GetArray().Size());
+ ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs<sj::array>(json_obj));
+ ARROW_ASSIGN_OR_RAISE(int32_t size, child_converter_->AppendValues(array));
+ if (size != list_size_) {
+ return Status::Invalid("incorrect list size ", size);
}
return Status::OK();
}
@@ -710,6 +817,9 @@ class StructConverter final : public
ConcreteConverter<StructConverter> {
child_converters_.push_back(child_converter);
child_builders.push_back(child_converter->builder());
}
+ for (int32_t i = 0; i < type_->num_fields(); ++i) {
+ field_index_.emplace(type_->field(i)->name(), i);
+ }
builder_ = std::make_shared<StructBuilder>(type_, default_memory_pool(),
std::move(child_builders));
return Status::OK();
@@ -718,45 +828,68 @@ class StructConverter final : public
ConcreteConverter<StructConverter> {
// Append a JSON value that is either an array of N elements in order
// or an object mapping struct names to values (omitted struct members
// are mapped to null).
- Status AppendValue(const rj::Value& json_obj) override {
- if (json_obj.IsNull()) {
+ Status AppendValue(sj::value& json_obj) override {
+ if (json_obj.is_null()) {
return this->AppendNull();
}
- if (json_obj.IsArray()) {
- auto size = json_obj.Size();
- auto expected_size = static_cast<uint32_t>(type_->num_fields());
+ sj::array array;
+ if (json_obj.get(array) == simdjson::SUCCESS) {
+ auto expected_size = static_cast<size_t>(type_->num_fields());
+ size_t size;
+ if (array.count_elements().get(size) != simdjson::SUCCESS) {
+ return Status::Invalid("Malformed JSON array for type ",
type_->ToString());
+ }
if (size != expected_size) {
return Status::Invalid("Expected array of size ", expected_size,
", got array of size ", size);
}
- for (uint32_t i = 0; i < size; ++i) {
- RETURN_NOT_OK(child_converters_[i]->AppendValue(json_obj[i]));
+ size_t i = 0;
+ for (auto child : array) {
+ ARROW_ASSIGN_OR_RAISE(auto child_value,
+ GetJsonResult<sj::value>(
+ child, "Could not iterate elements of JSON
array: "));
+ RETURN_NOT_OK(child_converters_[i]->AppendValue(child_value));
+ ++i;
}
return builder_->Append();
}
- if (json_obj.IsObject()) {
- auto remaining = json_obj.MemberCount();
- auto num_children = type_->num_fields();
- for (int32_t i = 0; i < num_children; ++i) {
- const auto& field = type_->field(i);
- auto it = json_obj.FindMember(field->name());
- if (it != json_obj.MemberEnd()) {
- --remaining;
- RETURN_NOT_OK(child_converters_[i]->AppendValue(it->value));
- } else {
- RETURN_NOT_OK(child_converters_[i]->AppendNull());
- }
+ ARROW_ASSIGN_OR_RAISE(auto object, GetJsonAs<sj::object>(json_obj));
+ // Iterate the object fields in JSON order (the on-demand API is
+ // forward-only, so per-field lookups would be quadratic and would also
+ // compare against raw, still-escaped keys). Fields absent from the JSON
+ // are appended as null afterwards.
+ auto num_fields = type_->num_fields();
+ std::vector<bool> field_seen(num_fields, false);
+ for (auto field_result : object) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto field,
+ GetJsonResult<sj::field>(field_result, "Error getting field of
object: "));
+ std::string_view key;
+ if (field.unescaped_key(/*allow_replacement=*/false).get(key) !=
+ simdjson::SUCCESS) {
+ return Status::Invalid("Malformed key in JSON object for type ",
+ type_->ToString());
}
- if (remaining > 0) {
- rj::StringBuffer sb;
- rj::Writer<rj::StringBuffer> writer(sb);
- json_obj.Accept(writer);
- return Status::Invalid("Unexpected members in JSON object for type ",
- type_->ToString(), " Object: ", sb.GetString());
+ auto it = field_index_.find(key);
+ if (it == field_index_.end()) {
+ return Status::Invalid("Unexpected member \"", key, "\" in JSON object
for type ",
+ type_->ToString());
+ }
+ const int32_t field_num = it->second;
+ if (field_seen[field_num]) {
+ return Status::Invalid("Duplicate member \"", key, "\" in JSON object
for type ",
+ type_->ToString());
+ }
+ field_seen[field_num] = true;
+ sj::value value = field.value();
+ RETURN_NOT_OK(child_converters_[field_num]->AppendValue(value));
+ }
+ for (int32_t i = 0; i < num_fields; ++i) {
+ if (!field_seen[i]) {
+ RETURN_NOT_OK(child_converters_[i]->AppendNull());
}
- return builder_->Append();
}
- return JSONTypeError("array or object", json_obj.GetType());
+ return builder_->Append();
}
std::shared_ptr<ArrayBuilder> builder() override { return builder_; }
@@ -764,6 +897,8 @@ class StructConverter final : public
ConcreteConverter<StructConverter> {
private:
std::shared_ptr<StructBuilder> builder_;
std::vector<std::shared_ptr<JSONConverter>> child_converters_;
+ // keys are views into the field names owned by type_
+ std::unordered_map<std::string_view, int32_t> field_index_;
};
// ------------------------------------------------------------------------
@@ -801,40 +936,43 @@ class UnionConverter final : public
ConcreteConverter<UnionConverter> {
// Append a JSON value that must be a 2-long array, containing the type_id
// and value of the UnionArray's slot.
- Status AppendValue(const rj::Value& json_obj) override {
- if (json_obj.IsNull()) {
+ Status AppendValue(sj::value& json_obj) override {
+ if (json_obj.is_null()) {
return this->AppendNull();
}
- if (!json_obj.IsArray()) {
- return JSONTypeError("array", json_obj.GetType());
- }
- if (json_obj.Size() != 2) {
- return Status::Invalid("Expected [type_id, value] pair, got array of
size ",
- json_obj.Size());
- }
- const auto& id_obj = json_obj[0];
- if (!id_obj.IsInt()) {
- return JSONTypeError("int", id_obj.GetType());
- }
-
- auto id = static_cast<int8_t>(id_obj.GetInt());
- auto child_num = type_id_to_child_num_[id];
- if (child_num == -1) {
- return Status::Invalid("type_id ", id, " not found in ", *type_);
- }
- auto child_converter = child_converters_[child_num];
- if (mode_ == UnionMode::SPARSE) {
- RETURN_NOT_OK(checked_cast<SparseUnionBuilder&>(*builder_).Append(id));
- for (auto&& other_converter : child_converters_) {
- if (other_converter != child_converter) {
- RETURN_NOT_OK(other_converter->AppendNull());
- }
- }
- } else {
- RETURN_NOT_OK(checked_cast<DenseUnionBuilder&>(*builder_).Append(id));
- }
- return child_converter->AppendValue(json_obj[1]);
+ ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs<sj::array>(json_obj));
+
+ int8_t id = 0;
+ std::shared_ptr<JSONConverter> child_converter;
+
+ RETURN_NOT_OK(ProcessJsonArrayElements<2>(
+ array, "[type_id, value] pair",
+ {[this, &id, &child_converter](sj::value& id_elem) {
+ ARROW_ASSIGN_OR_RAISE(auto id_value, GetJsonAs<int64_t>(id_elem));
+ id = static_cast<int8_t>(id_value);
+ auto child_num = type_id_to_child_num_[id];
+ if (child_num == -1) {
+ return Status::Invalid("type_id ", id, " not found in ", *type_);
+ }
+ child_converter = child_converters_[child_num];
+
+ if (mode_ == UnionMode::SPARSE) {
+
RETURN_NOT_OK(checked_cast<SparseUnionBuilder&>(*builder_).Append(id));
+ for (auto&& other_converter : child_converters_) {
+ if (other_converter != child_converter) {
+ RETURN_NOT_OK(other_converter->AppendNull());
+ }
+ }
+ } else {
+
RETURN_NOT_OK(checked_cast<DenseUnionBuilder&>(*builder_).Append(id));
+ }
+ return Status::OK();
+ },
+ [&child_converter](sj::value& value_elem) {
+ return child_converter->AppendValue(value_elem);
+ }}));
+ return Status::OK();
}
std::shared_ptr<ArrayBuilder> builder() override { return builder_; }
@@ -980,15 +1118,27 @@ Result<std::shared_ptr<Array>> ArrayFromJSONString(const
std::shared_ptr<DataTyp
std::shared_ptr<JSONConverter> converter;
RETURN_NOT_OK(GetConverter(type, &converter));
- rj::Document json_doc;
- json_doc.Parse<kParseFlags>(json_string.data(), json_string.length());
- if (json_doc.HasParseError()) {
- return Status::Invalid("JSON parse error at offset ",
json_doc.GetErrorOffset(), ": ",
- GetParseError_En(json_doc.GetParseError()));
+ simdjson::padded_string padded_string{json_string};
+
+ sj::parser parser;
+ sj::document json_doc;
+ auto error = parser.iterate(padded_string).get(json_doc);
+ if (error) {
+ return Status::Invalid("JSON parse error: ",
simdjson::error_message(error));
+ }
+ sj::value json_obj;
+ if (auto error_code = json_doc.get_value().get(json_obj);
+ error_code != simdjson::SUCCESS) {
+ return Status::Invalid("JSON parse error: ",
simdjson::error_message(error_code));
}
+ ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs<sj::array>(json_obj));
// The JSON document should be an array, append it
- RETURN_NOT_OK(converter->AppendValues(json_doc));
+ RETURN_NOT_OK(converter->AppendValues(array));
+ // The parser is lazy and does not look past the end of the array by itself
+ if (!json_doc.at_end()) {
+ return Status::Invalid("JSON parse error: trailing content after JSON
array");
+ }
std::shared_ptr<Array> out;
RETURN_NOT_OK(converter->Finish(&out));
return out;
@@ -1036,17 +1186,36 @@ Result<std::shared_ptr<Scalar>> ScalarFromJSONString(
std::shared_ptr<JSONConverter> converter;
RETURN_NOT_OK(GetConverter(type, &converter));
- rj::Document json_doc;
- json_doc.Parse<kParseFlags>(json_string.data(), json_string.length());
- if (json_doc.HasParseError()) {
- return Status::Invalid("JSON parse error at offset ",
json_doc.GetErrorOffset(), ": ",
- GetParseError_En(json_doc.GetParseError()));
+ simdjson::padded_string padded_string{
+ arrow::internal::JoinToString("[", json_string, "]")};
+
+ sj::parser parser;
+ sj::document json_doc;
+ auto error = parser.iterate(padded_string).get(json_doc);
+ if (error) {
+ return Status::Invalid("JSON parse error: ",
simdjson::error_message(error));
+ }
+
+ sj::value json_obj;
+ if (auto error_code = json_doc.get_value().get(json_obj);
+ error_code != simdjson::SUCCESS) {
+ return Status::Invalid("JSON parse error: ",
simdjson::error_message(error_code));
+ }
+ ARROW_ASSIGN_OR_RAISE(auto singleton_array, GetJsonAs<sj::array>(json_obj));
+
+ ARROW_ASSIGN_OR_RAISE(int32_t num_elements,
converter->AppendValues(singleton_array));
+ if (num_elements != 1) {
+ return Status::Invalid("Expected exactly one JSON value, got ",
num_elements,
+ " values");
+ }
+ // The parser is lazy and does not look past the end of the wrapped value by
+ // itself, e.g. the input "1] , [2" would otherwise be accepted as 1
+ if (!json_doc.at_end()) {
+ return Status::Invalid("JSON parse error: trailing content after JSON
value");
}
std::shared_ptr<Array> array;
- RETURN_NOT_OK(converter->AppendValue(json_doc));
RETURN_NOT_OK(converter->Finish(&array));
- DCHECK_EQ(array->length(), 1);
return array->GetScalar(0);
}
diff --git a/cpp/src/arrow/json/from_string_test.cc
b/cpp/src/arrow/json/from_string_test.cc
index 654450462e..817cfe52ee 100644
--- a/cpp/src/arrow/json/from_string_test.cc
+++ b/cpp/src/arrow/json/from_string_test.cc
@@ -217,6 +217,9 @@ TYPED_TEST_P(TestIntegersFromString, Errors) {
ASSERT_RAISES(Invalid, ArrayFromJSONString(type, "[0.0]"));
ASSERT_RAISES(Invalid, ArrayFromJSONString(type, "[\"0\"]"));
ASSERT_RAISES(Invalid, ArrayFromJSONString(type, "[[0]]"));
+ ASSERT_RAISES(Invalid, ArrayFromJSONString(type, "[0]]"));
+ ASSERT_RAISES(Invalid, ArrayFromJSONString(type, "[0], [1]"));
+ ASSERT_RAISES(Invalid, ArrayFromJSONString(type, "[0] junk"));
}
TYPED_TEST_P(TestIntegersFromString, OutOfBounds) {
@@ -305,9 +308,9 @@ TYPED_TEST_P(TestStringsFromString, Basics) {
AssertJSONArray<T, std::string>(type, "[\"\xc3\xa9\"]", {"\xc3\xa9"});
if (!T::is_utf8) {
- // Arbitrary binary (non-UTF8) sequence in string
+ // Arbitrary binary (non-UTF8) sequences cannot be represented
s = "\xff\x9f";
- AssertJSONArray<T, std::string>(type, "[\"" + s + "\"]", {s});
+ ASSERT_RAISES(Invalid, ArrayFromJSONString(type, "[\"" + s + "\"]"));
}
// Bytes < 0x20 can be represented as JSON unicode escapes
@@ -441,7 +444,8 @@ TEST(TestDoubleFromString, Basics) {
AssertJSONArray<DoubleType>(type, "[1, 2.5, -3e4]", {1.0, 2.5, -3.0e4});
AssertJSONArray<DoubleType>(type, "[-0.0, Inf, -Inf, null]", {true, true,
true, false},
{-0.0, INFINITY, -INFINITY, 0.0});
-
+ AssertJSONArray<DoubleType>(type, "[Infinity, -Infinity, null]", {true,
true, false},
+ {INFINITY, -INFINITY, 0.0});
ASSERT_OK_AND_ASSIGN(actual, ArrayFromJSONString(type, "[NaN]"));
ASSERT_OK(actual->ValidateFull());
double value = checked_cast<DoubleArray&>(*actual).Value(0);
@@ -536,9 +540,10 @@ TEST(TestFixedSizeBinaryFromString, Basics) {
{"foo", "bar"});
AssertJSONArray<FixedSizeBinaryType, std::string>(type, "[null, \"foo\"]",
{false, true}, {"",
"foo"});
- // Arbitrary binary (non-UTF8) sequence in string
+ // Arbitrary binary (non-UTF8) sequences cannot be represented: the JSON
+ // parser requires its input to be valid UTF-8
std::string s = "\xff\x9f\xcc";
- AssertJSONArray<FixedSizeBinaryType, std::string>(type, "[\"" + s + "\"]",
{s});
+ ASSERT_RAISES(Invalid, ArrayFromJSONString(type, "[\"" + s + "\"]"));
}
TEST(TestFixedSizeBinaryFromString, Errors) {
@@ -1563,6 +1568,9 @@ TEST(TestScalarFromJSONString, Errors) {
ASSERT_RAISES(Invalid, ScalarFromJSONString(binary(), "[]"));
ASSERT_RAISES(Invalid, ScalarFromJSONString(boolean(), "0.0"));
ASSERT_RAISES(Invalid, ScalarFromJSONString(boolean(), "\"true\""));
+ ASSERT_RAISES(Invalid, ScalarFromJSONString(int64(), "1, 2"));
+ ASSERT_RAISES(Invalid, ScalarFromJSONString(int64(), "1] , [2"));
+ ASSERT_RAISES(Invalid, ScalarFromJSONString(int64(), "1 junk"));
}
TEST(TestDictScalarFromJSONString, Basics) {
diff --git a/cpp/src/arrow/scalar_test.cc b/cpp/src/arrow/scalar_test.cc
index 4096b8cd17..c9e49ab66a 100644
--- a/cpp/src/arrow/scalar_test.cc
+++ b/cpp/src/arrow/scalar_test.cc
@@ -36,6 +36,7 @@
#include "arrow/memory_pool.h"
#include "arrow/scalar.h"
#include "arrow/status.h"
+#include "arrow/testing/builder.h"
#include "arrow/testing/extension_type.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/testing/random.h"
@@ -1395,8 +1396,10 @@ class TestListLikeScalar : public ::testing::Test {
}
{
- // Invalid UTF8 in child data
- ScalarType scalar(ArrayFromJSON(utf8(), "[null, null, \"\xff\"]"));
+ std::shared_ptr<Array> invalid_utf8;
+ ArrayFromVector<StringType, std::string>({false, false, true}, {"", "",
"\xff"},
+ &invalid_utf8);
+ ScalarType scalar(invalid_utf8);
ASSERT_OK(scalar.Validate());
ASSERT_RAISES(Invalid, scalar.ValidateFull());
}
diff --git a/cpp/src/arrow/testing/builder.h b/cpp/src/arrow/testing/builder.h
index 6beb7760e3..2d3cbbe9a4 100644
--- a/cpp/src/arrow/testing/builder.h
+++ b/cpp/src/arrow/testing/builder.h
@@ -92,6 +92,40 @@ void ArrayFromVector(const std::vector<C_TYPE>& values,
std::shared_ptr<Array>*
ArrayFromVector<TYPE, C_TYPE>(type, values, out);
}
+// BinaryArrayFromStrings: construct an Array of any binary-like type from
+// string values, dispatching on the runtime type id
+inline std::shared_ptr<Array> BinaryArrayFromStrings(
+ const std::shared_ptr<DataType>& type, const std::vector<std::string>&
values) {
+ std::shared_ptr<Array> array;
+ switch (type->id()) {
+ case Type::BINARY:
+ ArrayFromVector<BinaryType, std::string>(type, values, &array);
+ break;
+ case Type::STRING:
+ ArrayFromVector<StringType, std::string>(type, values, &array);
+ break;
+ case Type::LARGE_BINARY:
+ ArrayFromVector<LargeBinaryType, std::string>(type, values, &array);
+ break;
+ case Type::LARGE_STRING:
+ ArrayFromVector<LargeStringType, std::string>(type, values, &array);
+ break;
+ case Type::BINARY_VIEW:
+ ArrayFromVector<BinaryViewType, std::string>(type, values, &array);
+ break;
+ case Type::STRING_VIEW:
+ ArrayFromVector<StringViewType, std::string>(type, values, &array);
+ break;
+ case Type::FIXED_SIZE_BINARY:
+ ArrayFromVector<FixedSizeBinaryType, std::string>(type, values, &array);
+ break;
+ default:
+ ADD_FAILURE() << "unsupported type for binary test data: " <<
type->ToString();
+ break;
+ }
+ return array;
+}
+
// ChunkedArrayFromVector: construct a ChunkedArray from vectors of C values
template <typename TYPE, typename C_TYPE = typename TYPE::c_type>
diff --git a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc
b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc
index 3bee0e7928..3fcc4ac49c 100644
--- a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc
+++ b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc
@@ -1495,7 +1495,12 @@ class TestBinaryLikeParquetIO : public ParquetIOTestBase
{
const std::shared_ptr<DataType>& fallback_type) {
const auto specific_array = ::arrow::ArrayFromJSON(specific_type, json);
const auto fallback_array = ::arrow::ArrayFromJSON(fallback_type, json);
+ CheckRoundTrip(specific_array, fallback_array, binary_type);
+ }
+ void CheckRoundTrip(const std::shared_ptr<Array>& specific_array,
+ const std::shared_ptr<Array>& fallback_array,
+ ::arrow::Type::type binary_type) {
// When the original Arrow schema isn't stored, the array is decoded as
// the fallback type (since there is no specific Parquet logical
// type for it).
@@ -1521,13 +1526,27 @@ class TestBinaryLikeParquetIO : public
ParquetIOTestBase {
};
TEST_F(TestBinaryLikeParquetIO, LargeBinary) {
- CheckRoundTrip("[\"foo\", \"\", null, \"\xff\"]",
::arrow::Type::LARGE_BINARY,
- ::arrow::large_binary(), ::arrow::binary());
+ const std::vector<bool> is_valid = {true, true, false, true};
+ const std::vector<std::string> values = {"foo", "", "", "\xff"};
+ std::shared_ptr<Array> specific_array;
+ ::arrow::ArrayFromVector<::arrow::LargeBinaryType, std::string>(is_valid,
values,
+
&specific_array);
+ std::shared_ptr<Array> fallback_array;
+ ::arrow::ArrayFromVector<::arrow::BinaryType, std::string>(is_valid, values,
+ &fallback_array);
+ CheckRoundTrip(specific_array, fallback_array, ::arrow::Type::LARGE_BINARY);
}
TEST_F(TestBinaryLikeParquetIO, BinaryView) {
- CheckRoundTrip("[\"foo\", \"\", null, \"\xff\"]", ::arrow::Type::BINARY_VIEW,
- ::arrow::binary_view(), ::arrow::binary());
+ const std::vector<bool> is_valid = {true, true, false, true};
+ const std::vector<std::string> values = {"foo", "", "", "\xff"};
+ std::shared_ptr<Array> specific_array;
+ ::arrow::ArrayFromVector<::arrow::BinaryViewType, std::string>(is_valid,
values,
+
&specific_array);
+ std::shared_ptr<Array> fallback_array;
+ ::arrow::ArrayFromVector<::arrow::BinaryType, std::string>(is_valid, values,
+ &fallback_array);
+ CheckRoundTrip(specific_array, fallback_array, ::arrow::Type::BINARY_VIEW);
}
TEST_F(TestBinaryLikeParquetIO, LargeString) {
diff --git a/python/pyarrow/src/arrow/python/gdb.cc
b/python/pyarrow/src/arrow/python/gdb.cc
index 2a7d2eda4b..89135239f2 100644
--- a/python/pyarrow/src/arrow/python/gdb.cc
+++ b/python/pyarrow/src/arrow/python/gdb.cc
@@ -17,9 +17,13 @@
#include <cstdlib>
#include <memory>
+#include <optional>
+#include <string>
#include <utility>
+#include <vector>
#include "arrow/array.h"
+#include "arrow/array/builder_binary.h"
#include "arrow/chunked_array.h"
#include "arrow/datum.h"
#include "arrow/extension/uuid.h"
@@ -29,6 +33,7 @@
#include "arrow/scalar.h"
#include "arrow/table.h"
#include "arrow/type.h"
+#include "arrow/util/checked_cast.h"
#include "arrow/util/debug.h"
#include "arrow/util/decimal.h"
#include "arrow/util/key_value_metadata.h"
@@ -43,6 +48,8 @@ using json::ArrayFromJSONString;
using json::ChunkedArrayFromJSONString;
using json::ScalarFromJSONString;
+using ::arrow::internal::checked_cast;
+
namespace gdb {
// Add a nested `arrow` namespace to exercise type lookup from GDB
(ARROW-15652)
@@ -69,6 +76,25 @@ std::shared_ptr<Array> SliceArrayFromJSON(const
std::shared_ptr<DataType>& ty,
}
}
+template <typename BuilderType>
+std::shared_ptr<Array> BinaryArrayFromStrings(
+ const std::shared_ptr<DataType>& type,
+ const std::vector<std::optional<std::string>>& values) {
+ std::unique_ptr<ArrayBuilder> builder;
+ ARROW_CHECK_OK(MakeBuilder(default_memory_pool(), type, &builder));
+ auto& concrete_builder = checked_cast<BuilderType&>(*builder);
+ for (const auto& value : values) {
+ if (value.has_value()) {
+ ARROW_CHECK_OK(concrete_builder.Append(*value));
+ } else {
+ ARROW_CHECK_OK(concrete_builder.AppendNull());
+ }
+ }
+ std::shared_ptr<Array> array;
+ ARROW_CHECK_OK(concrete_builder.Finish(&array));
+ return array;
+}
+
} // namespace
void TestSession() {
@@ -448,18 +474,22 @@ void TestSession() {
decimal256(50, 6), R"([null,
"-123456789012345678901234567890123456789.012345"])");
auto heap_decimal128_array_sliced = heap_decimal128_array->Slice(1, 1);
- auto heap_fixed_size_binary_array =
- SliceArrayFromJSON(fixed_size_binary(3), "[null, \"abc\",
\"\\u0000\\u001f\xff\"]");
+ auto heap_fixed_size_binary_array =
BinaryArrayFromStrings<FixedSizeBinaryBuilder>(
+ fixed_size_binary(3), {std::nullopt, "abc", std::string("\x00\x1f\xff",
3)});
auto heap_fixed_size_binary_array_zero_width =
SliceArrayFromJSON(fixed_size_binary(0), R"([null, ""])");
auto heap_fixed_size_binary_array_sliced =
heap_fixed_size_binary_array->Slice(1, 1);
- const char* json_binary_array = "[null, \"abcd\", \"\\u0000\\u001f\xff\"]";
- auto heap_binary_array = SliceArrayFromJSON(binary(), json_binary_array);
- auto heap_large_binary_array = SliceArrayFromJSON(large_binary(),
json_binary_array);
- const char* json_string_array = "[null, \"héhé\", \"invalid \xff char\"]";
- auto heap_string_array = SliceArrayFromJSON(utf8(), json_string_array);
- auto heap_large_string_array = SliceArrayFromJSON(large_utf8(),
json_string_array);
+ const std::vector<std::optional<std::string>> binary_values = {
+ std::nullopt, "abcd", std::string("\x00\x1f\xff", 3)};
+ auto heap_binary_array = BinaryArrayFromStrings<BinaryBuilder>(binary(),
binary_values);
+ auto heap_large_binary_array =
+ BinaryArrayFromStrings<LargeBinaryBuilder>(large_binary(),
binary_values);
+ const std::vector<std::optional<std::string>> string_values = {std::nullopt,
"héhé",
+ "invalid \xff
char"};
+ auto heap_string_array = BinaryArrayFromStrings<StringBuilder>(utf8(),
string_values);
+ auto heap_large_string_array =
+ BinaryArrayFromStrings<LargeStringBuilder>(large_utf8(), string_values);
auto heap_binary_array_sliced = heap_binary_array->Slice(1, 1);
// ChunkedArray
