This is an automated email from the ASF dual-hosted git repository.
gabriellee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new edcff62dd7f [chore](json)Remove some unnecessary parameters in
JsonbParserTSIMD and JsonbWriterT. (#51239)
edcff62dd7f is described below
commit edcff62dd7f40f3410c5a7169828550a9b39c643
Author: Mryange <[email protected]>
AuthorDate: Wed May 28 09:20:04 2025 +0800
[chore](json)Remove some unnecessary parameters in JsonbParserTSIMD and
JsonbWriterT. (#51239)
JsonbParserTSIMD does not actually need a template parameter, and the
hDictInsert handler does not need to be passed in.
JsonbWriterT also does not need a template parameter (though it will be
refactored in the future, so it is kept for now), and it also does not
need the hDictInsert handler.
---
be/src/util/jsonb_parser_simd.h | 52 ++++++++++++-----------------------------
be/src/util/jsonb_writer.h | 41 +++++++++++---------------------
2 files changed, 28 insertions(+), 65 deletions(-)
diff --git a/be/src/util/jsonb_parser_simd.h b/be/src/util/jsonb_parser_simd.h
index 8684fc19218..1b51b91a9be 100644
--- a/be/src/util/jsonb_parser_simd.h
+++ b/be/src/util/jsonb_parser_simd.h
@@ -72,29 +72,20 @@
namespace doris {
using int128_t = __int128;
-
-/*
- * Template JsonbParserTSIMD
- */
-template <class OS_TYPE>
class JsonbParserTSIMD {
public:
JsonbParserTSIMD() : err_(JsonbErrType::E_NONE) {}
- explicit JsonbParserTSIMD(OS_TYPE& os) : writer_(os),
err_(JsonbErrType::E_NONE) {}
+ explicit JsonbParserTSIMD(JsonbOutStream& os) : writer_(os),
err_(JsonbErrType::E_NONE) {}
// parse a UTF-8 JSON string
- bool parse(const std::string& str, hDictInsert handler = nullptr) {
- return parse(str.c_str(), str.size(), handler);
- }
+ bool parse(const std::string& str) { return parse(str.c_str(),
str.size()); }
// parse a UTF-8 JSON c-style string (NULL terminated)
- bool parse(const char* c_str, hDictInsert handler = nullptr) {
- return parse(c_str, strlen(c_str), handler);
- }
+ bool parse(const char* c_str) { return parse(c_str, strlen(c_str)); }
// parse a UTF-8 JSON string with length
- bool parse(const char* pch, size_t len, hDictInsert handler = nullptr) {
+ bool parse(const char* pch, size_t len) {
// reset state before parse
reset();
@@ -114,7 +105,7 @@ public:
switch (doc.type()) {
case simdjson::ondemand::json_type::object:
case simdjson::ondemand::json_type::array: {
- parse(doc.get_value(), handler);
+ parse(doc.get_value());
break;
}
case simdjson::ondemand::json_type::null: {
@@ -151,7 +142,7 @@ public:
// parse json, recursively if necessary, by simdjson
// and serialize to binary format by writer
- void parse(simdjson::ondemand::value value, hDictInsert handler = nullptr)
{
+ void parse(simdjson::ondemand::value value) {
switch (value.type()) {
case simdjson::ondemand::json_type::null: {
if (writer_.writeNull() == 0) {
@@ -191,27 +182,14 @@ public:
break;
}
- int key_id = -1;
- if (handler) {
- key_id = handler(key.data(), key.size());
- }
-
- if (key_id < 0) {
- if (writer_.writeKey(key.data(), key.size()) == 0) {
- err_ = JsonbErrType::E_OUTPUT_FAIL;
- LOG(WARNING) << "writeKey failed key: " << key;
- break;
- }
- } else {
- if (writer_.writeKey(key_id) == 0) {
- err_ = JsonbErrType::E_OUTPUT_FAIL;
- LOG(WARNING) << "writeKey failed key_id: " << key_id;
- break;
- }
+ if (writer_.writeKey(key.data(), key.size()) == 0) {
+ err_ = JsonbErrType::E_OUTPUT_FAIL;
+ LOG(WARNING) << "writeKey failed key: " << key;
+ break;
}
// parse object value
- parse(kv.value(), handler);
+ parse(kv.value());
if (err_ != JsonbErrType::E_NONE) {
LOG(WARNING) << "parse object value failed";
break;
@@ -238,7 +216,7 @@ public:
for (auto elem : value.get_array()) {
// parse array element
- parse(elem.value(), handler);
+ parse(elem.value());
if (err_ != JsonbErrType::E_NONE) {
LOG(WARNING) << "parse array element failed";
break;
@@ -342,7 +320,7 @@ public:
}
}
- JsonbWriterT<OS_TYPE>& getWriter() { return writer_; }
+ JsonbWriterT<JsonbOutStream>& getWriter() { return writer_; }
JsonbErrType getErrorCode() { return err_; }
@@ -356,11 +334,11 @@ public:
private:
simdjson::ondemand::parser parser_;
- JsonbWriterT<OS_TYPE> writer_;
+ JsonbWriterT<JsonbOutStream> writer_;
JsonbErrType err_;
};
-using JsonbParser = JsonbParserTSIMD<JsonbOutStream>;
+using JsonbParser = JsonbParserTSIMD;
} // namespace doris
diff --git a/be/src/util/jsonb_writer.h b/be/src/util/jsonb_writer.h
index 9da386aa32e..b2499f6f9a8 100644
--- a/be/src/util/jsonb_writer.h
+++ b/be/src/util/jsonb_writer.h
@@ -49,6 +49,9 @@ using int128_t = __int128;
template <class OS_TYPE>
class JsonbWriterT {
+ /// TODO: maybe we should not use a template class here
+ static_assert(std::is_same_v<OS_TYPE, JsonbOutStream>);
+
public:
JsonbWriterT() : alloc_(true), hasHdr_(false), kvState_(WS_Value),
str_pos_(0) {
os_ = new OS_TYPE();
@@ -73,40 +76,22 @@ public:
;
}
- uint32_t writeKey(const char* key, hDictInsert handler = nullptr) {
- return writeKey(key, strlen(key), handler);
- }
+ uint32_t writeKey(const char* key) { return writeKey(key, strlen(key)); }
// write a key string (or key id if an external dict is provided)
- uint32_t writeKey(const char* key, uint8_t len, hDictInsert handler =
nullptr) {
+ uint32_t writeKey(const char* key, uint8_t len) {
if (!stack_.empty() && verifyKeyState()) {
- int key_id = -1;
- if (handler) {
- key_id = handler(key, len);
- }
-
uint32_t size = sizeof(uint8_t);
- if (key_id < 0) {
- os_->put(len);
- if (len == 0) {
- // NOTE: we use sMaxKeyId to represent an empty key
- JsonbKeyValue::keyid_type idx = JsonbKeyValue::sMaxKeyId;
- os_->write((char*)&idx, sizeof(JsonbKeyValue::keyid_type));
- size += sizeof(JsonbKeyValue::keyid_type);
- } else {
- os_->write(key, len);
- size += len;
- }
- } else if (key_id < JsonbKeyValue::sMaxKeyId) {
- JsonbKeyValue::keyid_type idx = key_id;
- os_->put(0);
+ os_->put(len);
+ if (len == 0) {
+ // NOTE: we use sMaxKeyId to represent an empty key
+ JsonbKeyValue::keyid_type idx = JsonbKeyValue::sMaxKeyId;
os_->write((char*)&idx, sizeof(JsonbKeyValue::keyid_type));
size += sizeof(JsonbKeyValue::keyid_type);
- } else { // key id overflow
- assert(0);
- return 0;
+ } else {
+ os_->write(key, len);
+ size += len;
}
-
kvState_ = WS_Key;
return size;
}
@@ -559,7 +544,7 @@ private:
bool first_ = true;
};
-typedef JsonbWriterT<JsonbOutStream> JsonbWriter;
+using JsonbWriter = JsonbWriterT<JsonbOutStream>;
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
