This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e2405875cf6 [chore](be) Remove unused V1 JSONDataParser (#68232)
e2405875cf6 is described below
commit e2405875cf66abdb99d4f2b6520f9a704023b61d
Author: lihangyu <[email protected]>
AuthorDate: Mon Sep 21 10:15:38 2026 +0800
[chore](be) Remove unused V1 JSONDataParser (#68232)
### What problem does this PR solve?
Issue Number: None
Related PR: #66858
Problem Summary: After #66858 enabled Variant V2 by default, the V1 JSON
parser in `be/src/util/json/json_parser.{h,cpp}` (`JSONDataParser`,
`ParseConfig`, `ParseResult` and the
`getValueAsField`/`writeValueAsJsonb` helpers) has no production caller;
only tests still referenced it.
This PR removes:
- the parser itself (`be/src/util/json/json_parser.{h,cpp}`), including
its `extern template class JSONDataParser<SimdJSONParser>` declaration
and the matching explicit instantiation;
- its dedicated test `be/test/core/jsonb/json_parser_test.cpp`, which
covered nothing else;
- the stale `#include "util/json/json_parser.h"` lines left in
`segment_creator.cpp`, `function_variant_parse.cpp`, `variant_util.cpp`,
`cast_to_decimal128_perf.cpp` and `index_storage_test_util.h`;
- the never-read `IndexBatch::parse_to` test field (of type
`ParseConfig::ParseTo`) and its single assignment in
`index_storage_variant_debug_point_test.cpp`.
`be/test/util/variant/variant_field_test.cpp` contained one case
comparing the V1 parser against the V2 encoder; only its V2
`encode_json` assertions are kept, renamed to
`VariantFieldTest.EncodeJsonStableSubset`.
`util/json/simd_json_parser.h` and `util/json/path_in_data.{h,cpp}` are
kept, as both still have production users.
Note: `be/benchmark/benchmark_variant_segment.hpp` also references
`ParseConfig`, but its V1 branch already fails to compile on master (it
calls `ColumnVariant`, `variant_util::parse_json_to_variant` and
`variant_util::select_storage_variant_parse_target`, none of which exist
any more). It is built only with the BE benchmark target and is left for
a separate fix.
### Release note
None
### Check List (For Author)
- Test: Unit Test
- `build-support/check-build-hygiene.sh` passed
- ASAN BE build (`./build.sh --be`) passed
- `./run-be-ut.sh --run --filter='VariantFieldTest.*:IndexStorage*'`
passed: 83 passed, 2 pre-existing skips
- Behavior changed: No
- Does this need documentation: No
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-authored-by: Claude Opus 5 <[email protected]>
---
be/src/exec/common/variant_util.cpp | 1 -
be/src/exprs/function/function_variant_parse.cpp | 1 -
be/src/storage/rowset/segment_creator.cpp | 1 -
be/src/util/json/json_parser.cpp | 414 ------------------
be/src/util/json/json_parser.h | 179 --------
be/test/core/jsonb/json_parser_test.cpp | 481 ---------------------
.../function/cast/cast_to_decimal128_perf.cpp | 1 -
.../index_storage_variant_debug_point_test.cpp | 1 -
be/test/testutil/index_storage_test_util.h | 2 -
be/test/util/variant/variant_field_test.cpp | 41 +-
10 files changed, 3 insertions(+), 1119 deletions(-)
diff --git a/be/src/exec/common/variant_util.cpp
b/be/src/exec/common/variant_util.cpp
index e2b46cbe47d..61c6f5b0a71 100644
--- a/be/src/exec/common/variant_util.cpp
+++ b/be/src/exec/common/variant_util.cpp
@@ -97,7 +97,6 @@
#include "storage/tablet/tablet_schema.h"
#include "util/client_cache.h"
#include "util/defer_op.h"
-#include "util/json/json_parser.h"
#include "util/json/path_in_data.h"
#include "util/json/simd_json_parser.h"
#include "util/jsonb_utils.h"
diff --git a/be/src/exprs/function/function_variant_parse.cpp
b/be/src/exprs/function/function_variant_parse.cpp
index c931cb1738c..2f4dcf69cf7 100644
--- a/be/src/exprs/function/function_variant_parse.cpp
+++ b/be/src/exprs/function/function_variant_parse.cpp
@@ -29,7 +29,6 @@
#include "core/data_type/data_type_variant_v2.h"
#include "exprs/function/parse/variant_string_parse.h"
#include "exprs/function/simple_function_factory.h"
-#include "util/json/json_parser.h"
namespace doris {
namespace {
diff --git a/be/src/storage/rowset/segment_creator.cpp
b/be/src/storage/rowset/segment_creator.cpp
index bcdbea7c7fe..85ccb0aa957 100644
--- a/be/src/storage/rowset/segment_creator.cpp
+++ b/be/src/storage/rowset/segment_creator.cpp
@@ -49,7 +49,6 @@
#include "storage/transform/block_transform.h"
#include "storage/utils.h"
#include "util/debug_points.h"
-#include "util/json/json_parser.h"
#include "util/pretty_printer.h"
#include "util/stopwatch.hpp"
diff --git a/be/src/util/json/json_parser.cpp b/be/src/util/json/json_parser.cpp
deleted file mode 100644
index 3df723c3849..00000000000
--- a/be/src/util/json/json_parser.cpp
+++ /dev/null
@@ -1,414 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-// This file is copied from
-//
https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/JSONParsers/SimdJSONParser.cpp
-// and modified by Doris
-
-#include "util/json/json_parser.h"
-
-#include <fmt/format.h>
-#include <glog/logging.h>
-
-#include <algorithm>
-#include <cassert>
-#include <string_view>
-#include <vector>
-
-#include "common/cast_set.h"
-// IWYU pragma: keep
-#include "common/status.h"
-#include "util/json/path_in_data.h"
-#include "util/json/simd_json_parser.h"
-
-namespace doris {
-
-template <typename ParserImpl>
-std::optional<ParseResult> JSONDataParser<ParserImpl>::parse(const char*
begin, size_t length,
- const
ParseConfig& config) {
- Element document;
- if (!parser.parse(begin, length, document)) {
- return {};
- }
- ParseContext context;
- // deprecated_enable_flatten_nested controls nested path traversal
- // NestedGroup expansion is now handled at storage layer
- context.deprecated_enable_flatten_nested =
config.deprecated_enable_flatten_nested;
- context.check_duplicate_json_path = config.check_duplicate_json_path;
- context.is_top_array = document.isArray();
- traverse(document, context);
- ParseResult result;
- result.values = std::move(context.values);
- result.paths.reserve(context.paths.size());
- for (auto&& path : context.paths) {
- result.paths.emplace_back(std::move(path));
- }
- return result;
-}
-
-template <typename ParserImpl>
-void JSONDataParser<ParserImpl>::traverse(const Element& element,
ParseContext& ctx) {
- // checkStackSize();
- if (element.isObject()) {
- traverseObject(element.getObject(), ctx);
- } else if (element.isArray()) {
- // allow nested arrays (multi-level) for NestedGroup; deeper levels are
- // handled by VariantNestedBuilder with a max-depth guard.
- has_nested = false;
- check_has_nested_object(element);
- ctx.has_nested_in_flatten = has_nested &&
ctx.deprecated_enable_flatten_nested;
- if (has_nested && !ctx.deprecated_enable_flatten_nested) {
- // Parse nested arrays to JsonbField
- JsonbWriter writer;
- traverseArrayAsJsonb(element.getArray(), writer);
- appendValueIfNotDuplicate(
- ctx, ctx.builder.get_parts(),
-
Field::create_field<TYPE_JSONB>(JsonbField(writer.getOutput()->getBuffer(),
-
writer.getOutput()->getSize())));
- } else {
- traverseArray(element.getArray(), ctx);
- }
- // we should set has_nested_in_flatten to false when traverse array
finished for next array otherwise it will be true for next array
- ctx.has_nested_in_flatten = false;
- } else {
- appendValueIfNotDuplicate(ctx, ctx.builder.get_parts(),
getValueAsField(element));
- }
-}
-
-template <typename ParserImpl>
-void JSONDataParser<ParserImpl>::appendValueIfNotDuplicate(ParseContext& ctx,
- const
PathInData::Parts& path,
- Field&& value) {
- if (ctx.check_duplicate_json_path) {
- PathInData path_in_data(path);
- if (!ctx.visited_path_names.emplace(path_in_data.get_path()).second) {
- return;
- }
- }
- ctx.paths.push_back(path);
- ctx.values.push_back(std::move(value));
-}
-
-template <typename ParserImpl>
-void JSONDataParser<ParserImpl>::traverseObject(const JSONObject& object,
ParseContext& ctx) {
- ctx.paths.reserve(ctx.paths.size() + object.size());
- ctx.values.reserve(ctx.values.size() + object.size());
- auto check_key_length = [](const auto& key) {
- const size_t max_key_length =
cast_set<size_t>(config::variant_max_json_key_length);
- if (key.size() > max_key_length) {
- throw doris::Exception(
- doris::ErrorCode::INVALID_ARGUMENT,
- fmt::format("Key length exceeds maximum allowed size of {}
bytes.",
- max_key_length));
- }
- };
- auto traverse_object_member = [&](const auto& key, const auto& value) {
- check_key_length(key);
- ctx.builder.append(key, false);
- traverse(value, ctx);
- ctx.builder.pop_back();
- };
-
- for (auto it = object.begin(); it != object.end(); ++it) {
- const auto& [key, value] = *it;
- traverse_object_member(key, value);
- }
-}
-
-template <typename ParserImpl>
-void JSONDataParser<ParserImpl>::check_has_nested_object(const Element&
element) {
- if (element.isArray()) {
- const JSONArray& array = element.getArray();
- for (auto it = array.begin(); it != array.end(); ++it) {
- check_has_nested_object(*it);
- }
- }
- if (element.isObject()) {
- has_nested = true;
- }
-}
-
-template <typename ParserImpl>
-void JSONDataParser<ParserImpl>::traverseAsJsonb(const Element& element,
JsonbWriter& writer) {
- if (element.isObject()) {
- traverseObjectAsJsonb(element.getObject(), writer);
- } else if (element.isArray()) {
- traverseArrayAsJsonb(element.getArray(), writer);
- } else {
- writeValueAsJsonb(element, writer);
- }
-}
-
-template <typename ParserImpl>
-void JSONDataParser<ParserImpl>::traverseObjectAsJsonb(const JSONObject&
object,
- JsonbWriter& writer) {
- writer.writeStartObject();
- for (auto it = object.begin(); it != object.end(); ++it) {
- const auto& [key, value] = *it;
- const size_t max_key_length =
cast_set<size_t>(config::variant_max_json_key_length);
- if (key.size() > max_key_length) {
- throw doris::Exception(
- doris::ErrorCode::INVALID_ARGUMENT,
- fmt::format("Key length exceeds maximum allowed size of {}
bytes.",
- max_key_length));
- }
- writer.writeKey(key.data(), cast_set<uint8_t>(key.size()));
- traverseAsJsonb(value, writer);
- }
- writer.writeEndObject();
-}
-
-template <typename ParserImpl>
-void JSONDataParser<ParserImpl>::traverseArrayAsJsonb(const JSONArray& array,
JsonbWriter& writer) {
- writer.writeStartArray();
- for (auto it = array.begin(); it != array.end(); ++it) {
- traverseAsJsonb(*it, writer);
- }
- writer.writeEndArray();
-}
-
-// check isPrefix in PathInData::Parts. like : [{"a": {"c": {"b": 1}}}, {"a":
{"c": 2.2}}], "a.c" is prefix of "a.c.b"
-// return true if prefix is a prefix of parts
-static bool is_prefix(const PathInData::Parts& prefix, const
PathInData::Parts& parts) {
- if (prefix.size() >= parts.size()) {
- return false;
- }
- for (size_t i = 0; i < prefix.size(); ++i) {
- if (prefix[i].key != parts[i].key) {
- return false;
- }
- }
- return true;
-}
-
-template <typename ParserImpl>
-void JSONDataParser<ParserImpl>::traverseArray(const JSONArray& array,
ParseContext& ctx) {
- /// Traverse elements of array and collect an array of fields by each path.
- ParseArrayContext array_ctx;
- array_ctx.has_nested_in_flatten = ctx.has_nested_in_flatten;
- array_ctx.is_top_array = ctx.is_top_array;
- array_ctx.check_duplicate_json_path = ctx.check_duplicate_json_path;
- array_ctx.total_size = array.size();
- for (auto it = array.begin(); it != array.end(); ++it) {
- traverseArrayElement(*it, array_ctx);
- ++array_ctx.current_size;
- }
- auto&& arrays_by_path = array_ctx.arrays_by_path;
- if (arrays_by_path.empty()) {
- appendValueIfNotDuplicate(ctx, ctx.builder.get_parts(),
- Field::create_field<TYPE_ARRAY>(Array()));
- } else {
- ctx.paths.reserve(ctx.paths.size() + arrays_by_path.size());
- ctx.values.reserve(ctx.values.size() + arrays_by_path.size());
- for (auto it = arrays_by_path.begin(); it != arrays_by_path.end();
++it) {
- auto&& [path, path_array] = it->second;
- /// Merge prefix path and path of array element.
- ctx.builder.append(path, true);
- appendValueIfNotDuplicate(ctx, ctx.builder.get_parts(),
-
Field::create_field<TYPE_ARRAY>(std::move(path_array)));
- ctx.builder.pop_back(path.size());
- }
- }
-}
-
-template <typename ParserImpl>
-void JSONDataParser<ParserImpl>::traverseArrayElement(const Element& element,
- ParseArrayContext& ctx) {
- ParseContext element_ctx;
- element_ctx.has_nested_in_flatten = ctx.has_nested_in_flatten;
- element_ctx.is_top_array = ctx.is_top_array;
- element_ctx.check_duplicate_json_path = ctx.check_duplicate_json_path;
- traverse(element, element_ctx);
- auto& paths = element_ctx.paths;
- auto& values = element_ctx.values;
-
- if (element_ctx.has_nested_in_flatten && element_ctx.is_top_array) {
- checkAmbiguousStructure(ctx, paths);
- }
-
- size_t size = paths.size();
- size_t keys_to_update = ctx.arrays_by_path.size();
-
- for (size_t i = 0; i < size; ++i) {
- if (values[i].is_null()) {
- continue;
- }
-
- UInt128 hash = PathInData::get_parts_hash(paths[i]);
- auto found = ctx.arrays_by_path.find(hash);
-
- if (found != ctx.arrays_by_path.end()) {
- handleExistingPath(found->second, paths[i], values[i], ctx,
keys_to_update);
- } else {
- handleNewPath(hash, paths[i], values[i], ctx);
- }
- }
-
- // always fill missed values to keep element-level association between
keys.
- if (keys_to_update) {
- fillMissedValuesInArrays(ctx);
- }
-}
-
-// check if the structure of top_array is ambiguous like:
-// [{"a": {"b": {"c": 1}}}, {"a": {"b": 1}}] a.b is ambiguous
-template <typename ParserImpl>
-void JSONDataParser<ParserImpl>::checkAmbiguousStructure(
- const ParseArrayContext& ctx, const std::vector<PathInData::Parts>&
paths) {
- for (auto&& current_path : paths) {
- for (auto it = ctx.arrays_by_path.begin(); it !=
ctx.arrays_by_path.end(); ++it) {
- auto&& [p, _] = it->second;
- if (is_prefix(p, current_path) || is_prefix(current_path, p)) {
- throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
- "Ambiguous structure of top_array
nested subcolumns: {}, {}",
- PathInData(p).to_jsonpath(),
- PathInData(current_path).to_jsonpath());
- }
- }
- }
-}
-
-template <typename ParserImpl>
-void
JSONDataParser<ParserImpl>::handleExistingPath(std::pair<PathInData::Parts,
Array>& path_data,
- const PathInData::Parts&
path, Field& value,
- ParseArrayContext& ctx,
- size_t& keys_to_update) {
- auto& path_array = path_data.second;
- // keep arrays aligned for all keys (including top-level arrays).
- assert(path_array.size() == ctx.current_size);
- // If current element of array is part of Nested,
- // collect its size or check it if the size of
- // the Nested has been already collected.
- auto nested_key = getNameOfNested(path, value);
- if (!nested_key.empty()) {
- size_t array_size = value.get<TYPE_ARRAY>().size();
- auto& current_nested_sizes = ctx.nested_sizes_by_key[nested_key];
- if (current_nested_sizes.size() == ctx.current_size) {
- current_nested_sizes.push_back(array_size);
- } else if (array_size != current_nested_sizes.back()) {
- throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
- "Array sizes mismatched ({} and {})",
array_size,
- current_nested_sizes.back());
- }
- }
-
- path_array.push_back(std::move(value));
- --keys_to_update;
-}
-
-template <typename ParserImpl>
-void JSONDataParser<ParserImpl>::handleNewPath(UInt128 hash, const
PathInData::Parts& path,
- Field& value,
ParseArrayContext& ctx) {
- Array path_array;
- path_array.reserve(ctx.total_size);
-
- // always resize to keep alignment.
- path_array.resize(ctx.current_size);
-
- auto nested_key = getNameOfNested(path, value);
- if (!nested_key.empty()) {
- size_t array_size = value.get<TYPE_ARRAY>().size();
- auto& current_nested_sizes = ctx.nested_sizes_by_key[nested_key];
- if (current_nested_sizes.empty()) {
- current_nested_sizes.resize(ctx.current_size);
- } else {
- // If newly added element is part of the Nested then
- // resize its elements to keep correct sizes of Nested arrays.
- for (size_t j = 0; j < ctx.current_size; ++j) {
- path_array[j] =
Field::create_field<TYPE_ARRAY>(Array(current_nested_sizes[j]));
- }
- }
- if (current_nested_sizes.size() == ctx.current_size) {
- current_nested_sizes.push_back(array_size);
- } else if (array_size != current_nested_sizes.back()) {
- throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
- "Array sizes mismatched ({} and {})",
array_size,
- current_nested_sizes.back());
- }
- }
-
- path_array.push_back(std::move(value));
- auto& elem = ctx.arrays_by_path[hash];
- elem.first = std::move(path);
- elem.second = std::move(path_array);
-}
-
-template <typename ParserImpl>
-void JSONDataParser<ParserImpl>::fillMissedValuesInArrays(ParseArrayContext&
ctx) {
- for (auto it = ctx.arrays_by_path.begin(); it != ctx.arrays_by_path.end();
++it) {
- auto& [path, path_array] = it->second;
- assert(path_array.size() == ctx.current_size || path_array.size() ==
ctx.current_size + 1);
- if (path_array.size() == ctx.current_size) {
- bool inserted = tryInsertDefaultFromNested(ctx, path, path_array);
- if (!inserted) {
- path_array.emplace_back();
- }
- }
- }
-}
-
-template <typename ParserImpl>
-bool JSONDataParser<ParserImpl>::tryInsertDefaultFromNested(ParseArrayContext&
ctx,
- const
PathInData::Parts& path,
- Array& array) {
- /// If there is a collected size of current Nested
- /// then insert array of this size as a default value.
- if (path.empty() || array.empty()) {
- return false;
- }
- /// Last element is not Null, because otherwise this path wouldn't exist.
- auto nested_key = getNameOfNested(path, array.back());
- if (nested_key.empty()) {
- return false;
- }
- auto mapped = ctx.nested_sizes_by_key.find(nested_key);
- if (mapped == ctx.nested_sizes_by_key.end()) {
- return false;
- }
- auto& current_nested_sizes = mapped->second;
- assert(current_nested_sizes.size() == ctx.current_size ||
- current_nested_sizes.size() == ctx.current_size + 1);
- /// If all keys of Nested were missed then add a zero length.
- if (current_nested_sizes.size() == ctx.current_size) {
- current_nested_sizes.push_back(0);
- }
- size_t array_size = current_nested_sizes.back();
- array.push_back(Field::create_field<TYPE_ARRAY>(Array(array_size)));
- return true;
-}
-
-template <typename ParserImpl>
-StringRef JSONDataParser<ParserImpl>::getNameOfNested(const PathInData::Parts&
path,
- const Field& value) {
- if (value.get_type() != PrimitiveType::TYPE_ARRAY || path.empty()) {
- return {};
- }
- /// Find first key that is marked as nested,
- /// because we may have struct of Nested and there could be
- /// several arrays with the same prefix, but with independent sizes.
- /// Consider we have array element with type `k2 Struct(k3 Nested(...), k5
Nested(...))`
- /// Then subcolumns `k2.k3` and `k2.k5` may have indepented sizes and we
should extract
- /// `k3` and `k5` keys instead of `k2`.
- for (const auto& part : path) {
- if (part.is_nested) {
- return {part.key.data(), part.key.size()};
- }
- }
- return {};
-}
-
-template class JSONDataParser<SimdJSONParser>;
-} // namespace doris
diff --git a/be/src/util/json/json_parser.h b/be/src/util/json/json_parser.h
deleted file mode 100644
index ee91765d44d..00000000000
--- a/be/src/util/json/json_parser.h
+++ /dev/null
@@ -1,179 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-// This file is copied from
-//
https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/JSONParsers/SimdJSONParser.h
-// and modified by Doris
-
-#pragma once
-
-#include <parallel_hashmap/phmap.h>
-#include <stddef.h>
-
-#include <optional>
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "core/column/column.h"
-#include "core/data_type/primitive_type.h"
-#include "core/field.h"
-#include "core/string_ref.h"
-#include "core/uint128.h"
-#include "util/json/path_in_data.h"
-#include "util/json/simd_json_parser.h"
-#include "util/jsonb_writer.h"
-
-namespace doris {
-
-template <typename Element>
-Field getValueAsField(const Element& element) {
- // bool will convert to type FiledType::UInt64
- if (element.isBool()) {
- return Field::create_field<TYPE_BOOLEAN>(element.getBool());
- }
- if (element.isInt64()) {
- return Field::create_field<TYPE_BIGINT>(element.getInt64());
- }
- // doris only support signed integers at present
- // use largeint to store unsigned int64
- if (element.isUInt64()) {
- return
Field::create_field<TYPE_LARGEINT>(static_cast<int128_t>(element.getUInt64()));
- }
- if (element.isDouble()) {
- return Field::create_field<TYPE_DOUBLE>(element.getDouble());
- }
- if (element.isString()) {
- return Field::create_field<TYPE_STRING>(String(element.getString()));
- }
- if (element.isNull()) {
- return Field();
- }
- return Field();
-}
-
-template <typename Element>
-void writeValueAsJsonb(const Element& element, JsonbWriter& writer) {
- // bool will convert to type FiledType::UInt64
- if (element.isBool()) {
- writer.writeBool(element.getBool());
- return;
- }
- if (element.isInt64()) {
- writer.writeInt64(element.getInt64());
- return;
- }
- // doris only support signed integers at present
- // use largeint to store unsigned int64
- if (element.isUInt64()) {
- writer.writeInt128(static_cast<int128_t>(element.getUInt64()));
- return;
- }
- if (element.isDouble()) {
- writer.writeDouble(element.getDouble());
- return;
- }
- if (element.isString()) {
- writer.writeStartString();
- std::string_view str = element.getString();
- writer.writeString(str.data(), str.size());
- writer.writeEndString();
- return;
- }
- if (element.isNull()) {
- writer.writeNull();
- return;
- }
-}
-
-struct ParseConfig {
- bool deprecated_enable_flatten_nested = false;
- bool check_duplicate_json_path = false;
- enum class ParseTo {
- OnlySubcolumns = 0,
- OnlyDocValueColumn = 1,
- };
- ParseTo parse_to = ParseTo::OnlySubcolumns;
-};
-/// Result of parsing of a document.
-/// Contains all paths extracted from document
-/// and values which are related to them.
-struct ParseResult {
- std::vector<PathInData> paths;
- std::vector<Field> values;
-};
-template <typename ParserImpl>
-class JSONDataParser {
-public:
- using Element = typename ParserImpl::Element;
- using JSONObject = typename ParserImpl::Object;
- using JSONArray = typename ParserImpl::Array;
- std::optional<ParseResult> parse(const char* begin, size_t length, const
ParseConfig& config);
-
-private:
- struct ParseContext {
- PathInDataBuilder builder;
- std::vector<PathInData::Parts> paths;
- std::vector<Field> values;
- phmap::flat_hash_set<std::string> visited_path_names;
- bool deprecated_enable_flatten_nested = false;
- bool check_duplicate_json_path = false;
- bool has_nested_in_flatten = false;
- bool is_top_array = false;
- };
- using PathPartsWithArray = std::pair<PathInData::Parts, Array>;
- using PathToArray = phmap::flat_hash_map<UInt128, PathPartsWithArray,
UInt128TrivialHash>;
- using KeyToSizes = phmap::flat_hash_map<StringRef, std::vector<size_t>,
StringRefHash>;
- struct ParseArrayContext {
- size_t current_size = 0;
- size_t total_size = 0;
- PathToArray arrays_by_path;
- KeyToSizes nested_sizes_by_key;
- bool has_nested_in_flatten = false;
- bool is_top_array = false;
- bool check_duplicate_json_path = false;
- };
- void traverse(const Element& element, ParseContext& ctx);
- void traverseObject(const JSONObject& object, ParseContext& ctx);
- void traverseArray(const JSONArray& array, ParseContext& ctx);
- void appendValueIfNotDuplicate(ParseContext& ctx, const PathInData::Parts&
path, Field&& value);
- void traverseArrayElement(const Element& element, ParseArrayContext& ctx);
- void checkAmbiguousStructure(const ParseArrayContext& ctx,
- const std::vector<PathInData::Parts>& paths);
- void handleExistingPath(std::pair<PathInData::Parts, Array>& path_data,
- const PathInData::Parts& path, Field& value,
ParseArrayContext& ctx,
- size_t& keys_to_update);
- void handleNewPath(UInt128 hash, const PathInData::Parts& path, Field&
value,
- ParseArrayContext& ctx);
- static void fillMissedValuesInArrays(ParseArrayContext& ctx);
- static bool tryInsertDefaultFromNested(ParseArrayContext& ctx, const
PathInData::Parts& path,
- Array& array);
- static StringRef getNameOfNested(const PathInData::Parts& path, const
Field& value);
-
- bool has_nested = false;
- void check_has_nested_object(const Element& element);
- void traverseAsJsonb(const Element& element, JsonbWriter& writer);
- void traverseObjectAsJsonb(const JSONObject& object, JsonbWriter& writer);
- void traverseArrayAsJsonb(const JSONArray& array, JsonbWriter& writer);
-
- ParserImpl parser;
-};
-
-class SimdJSONParser;
-/// Instantiated once in json_parser.cpp; suppresses per-TU implicit
instantiation.
-extern template class JSONDataParser<SimdJSONParser>;
-
-} // namespace doris
diff --git a/be/test/core/jsonb/json_parser_test.cpp
b/be/test/core/jsonb/json_parser_test.cpp
deleted file mode 100644
index f8cfdf8e162..00000000000
--- a/be/test/core/jsonb/json_parser_test.cpp
+++ /dev/null
@@ -1,481 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "util/json/json_parser.h"
-
-#include <gtest/gtest.h>
-
-#include <vector>
-
-#include "common/config.h"
-#include "core/string_ref.h"
-
-using doris::JSONDataParser;
-using doris::SimdJSONParser;
-using doris::ParseConfig;
-
-TEST(JsonParserTest, ParseSimpleTypes) {
- JSONDataParser<SimdJSONParser> parser;
- ParseConfig config;
-
- // int
- auto result = parser.parse("123", 3, config);
- ASSERT_TRUE(result.has_value());
- EXPECT_EQ(result->values.size(), 1);
-
- auto parse_result_int = result.value();
- EXPECT_EQ(parse_result_int.paths[0].get_path(), "");
- EXPECT_EQ(parse_result_int.values[0].get_type(),
doris::PrimitiveType::TYPE_BIGINT);
-
- // double
- result = parser.parse("1.23", 4, config);
- ASSERT_TRUE(result.has_value());
-
- auto parse_result_double = result.value();
- EXPECT_EQ(parse_result_double.paths[0].get_path(), "");
- EXPECT_EQ(parse_result_double.values[0].get_type(),
doris::PrimitiveType::TYPE_DOUBLE);
-
- // bool
- result = parser.parse("true", 4, config);
- ASSERT_TRUE(result.has_value());
-
- auto parse_result_bool = result.value();
- EXPECT_EQ(parse_result_bool.values[0].get_type(),
doris::PrimitiveType::TYPE_BOOLEAN);
-
- // null
- result = parser.parse("null", 4, config);
- ASSERT_TRUE(result.has_value());
- auto parse_result_null = result.value();
- EXPECT_EQ(parse_result_null.paths[0].get_path(), "");
- EXPECT_EQ(parse_result_null.values[0].get_type(),
doris::PrimitiveType::TYPE_NULL);
-
- // string
- result = parser.parse("\"abc\"", 5, config);
- ASSERT_TRUE(result.has_value());
- auto parse_result_string = result.value();
- EXPECT_EQ(parse_result_string.paths[0].get_path(), "");
- EXPECT_EQ(parse_result_string.values[0].get_type(),
doris::PrimitiveType::TYPE_STRING);
-
- // largeint
- result = parser.parse("12345678901234567890", 20, config);
- ASSERT_TRUE(result.has_value());
- auto parse_result_bigint = result.value();
- EXPECT_EQ(parse_result_bigint.paths[0].get_path(), "");
- EXPECT_EQ(parse_result_bigint.values[0].get_type(),
doris::PrimitiveType::TYPE_LARGEINT);
-}
-
-TEST(JsonParserTest, ParseObjectAndArray) {
- JSONDataParser<SimdJSONParser> parser;
- ParseConfig config;
-
- // Object
- auto result = parser.parse(R"({"a":1,"b":2})", 13, config);
- ASSERT_TRUE(result.has_value());
- auto& parse_result_object = result.value();
- EXPECT_EQ(parse_result_object.values.size(), 2);
- EXPECT_EQ(parse_result_object.paths.size(), 2);
- EXPECT_EQ(parse_result_object.values[0].get_type(),
doris::PrimitiveType::TYPE_BIGINT);
- EXPECT_EQ(parse_result_object.values[1].get_type(),
doris::PrimitiveType::TYPE_BIGINT);
- EXPECT_EQ(parse_result_object.paths[0].get_path(), "a");
- EXPECT_EQ(parse_result_object.paths[1].get_path(), "b");
-
- // Array
- result = parser.parse("[1,2,3]", 7, config);
- ASSERT_TRUE(result.has_value());
- auto& parse_result_array = result.value();
- EXPECT_EQ(parse_result_array.paths[0].get_path(), "");
- EXPECT_EQ(parse_result_array.values[0].get_type(),
doris::PrimitiveType::TYPE_ARRAY);
-
- std::string json = R"([1, "string", null, true, 1.23,
12345678901234567890])";
- result = parser.parse(json.c_str(), json.size(), config);
- ASSERT_TRUE(result.has_value());
- auto& parse_result_array_2 = result.value();
- EXPECT_EQ(parse_result_array_2.paths[0].get_path(), "");
- EXPECT_EQ(parse_result_array_2.values[0].get_type(),
doris::PrimitiveType::TYPE_ARRAY);
- auto& array_field =
parse_result_array_2.values[0].get<doris::PrimitiveType::TYPE_ARRAY>();
-
- EXPECT_EQ(array_field.size(), 6);
- EXPECT_EQ(array_field[0].get_type(), doris::PrimitiveType::TYPE_BIGINT);
- EXPECT_EQ(array_field[1].get_type(), doris::PrimitiveType::TYPE_STRING);
- EXPECT_EQ(array_field[2].get_type(), doris::PrimitiveType::TYPE_NULL);
- EXPECT_EQ(array_field[3].get_type(), doris::PrimitiveType::TYPE_BOOLEAN);
- EXPECT_EQ(array_field[4].get_type(), doris::PrimitiveType::TYPE_DOUBLE);
- EXPECT_EQ(array_field[5].get_type(), doris::PrimitiveType::TYPE_LARGEINT);
-}
-
-TEST(JsonParserTest, ParseMultiLevelNestedArray) {
- JSONDataParser<SimdJSONParser> parser;
- ParseConfig config;
-
- auto result = parser.parse("[[1,2],[3,4]]", 13, config);
- ASSERT_TRUE(result.has_value());
- EXPECT_EQ(result->values.size(), 1);
- EXPECT_EQ(result->paths.size(), 1);
- EXPECT_EQ(result->values[0].get_type(), doris::PrimitiveType::TYPE_ARRAY);
-
- result = parser.parse("[[[1],[2]],[[3],[4]]]", 21, config);
- ASSERT_TRUE(result.has_value());
- EXPECT_EQ(result->values.size(), 1);
- EXPECT_EQ(result->paths.size(), 1);
- EXPECT_EQ(result->values[0].get_type(), doris::PrimitiveType::TYPE_ARRAY);
-
- result = parser.parse("[[1,2],[3],[4,5,6]]", 19, config);
- ASSERT_TRUE(result.has_value());
- EXPECT_EQ(result->values.size(), 1);
- EXPECT_EQ(result->paths.size(), 1);
-
- // Test complex nested structure
- config.deprecated_enable_flatten_nested = false;
- std::string json1 = R"({"a":[[1,2],[3],[4,5,6]]})";
- // multi level nested array in object
- result = parser.parse(json1.c_str(), json1.size(), config);
- ASSERT_TRUE(result.has_value());
- EXPECT_EQ(result->values.size(), 1);
- EXPECT_EQ(result->paths.size(), 1);
- EXPECT_EQ(result->values[0].get_type(), doris::PrimitiveType::TYPE_ARRAY);
-
- std::string json = R"({"nested": [{"a": [1,2,3]}]})";
- // result should be jsonbField
- result = parser.parse(json.c_str(), json.size(), config);
- ASSERT_TRUE(result.has_value());
- EXPECT_EQ(result->values.size(), 1);
- EXPECT_EQ(result->paths.size(), 1);
- EXPECT_EQ(result->values[0].get_type(), doris::PrimitiveType::TYPE_JSONB);
-
- // multi level nested array in nested array object
- std::string json2 = R"({"a":[{"b":[[1,2,3]]}]})";
- result = parser.parse(json2.c_str(), json2.size(), config);
- ASSERT_TRUE(result.has_value());
- EXPECT_EQ(result->values.size(), 1);
- EXPECT_EQ(result->paths.size(), 1);
- EXPECT_EQ(result->values[0].get_type(), doris::PrimitiveType::TYPE_JSONB);
-
- // test flatten nested
- config.deprecated_enable_flatten_nested = true;
- // TODO: checkAmbiguousStructure is only called when has_nested_in_flatten
&& is_top_array.
- // These JSONs are objects (not top-level arrays), so is_top_array=false
and the check is skipped.
- // EXPECT_ANY_THROW(parser.parse(json.c_str(), json.size(), config));
- // test flatten nested with multi level nested array
- // no throw because it is not nested object array
- result = parser.parse(json1.c_str(), json1.size(), config);
- ASSERT_TRUE(result.has_value());
- EXPECT_EQ(result->values.size(), 1);
- EXPECT_EQ(result->paths.size(), 1);
- EXPECT_EQ(result->values[0].get_type(), doris::PrimitiveType::TYPE_ARRAY);
-
- // TODO: Same reason as above — object-level array, is_top_array=false,
check skipped.
- // EXPECT_ANY_THROW(parser.parse(json2.c_str(), json2.size(), config));
-}
-
-TEST(JsonParserTest, ParseNestedAndFlatten) {
- JSONDataParser<SimdJSONParser> parser;
- ParseConfig config;
- config.deprecated_enable_flatten_nested = true;
-
- std::string json = R"({"a":[{"b":1},{"b":2}]})";
- auto result = parser.parse(json.c_str(), json.size(), config);
- ASSERT_TRUE(result.has_value());
- EXPECT_GT(result->values.size(), 0);
-
- config.deprecated_enable_flatten_nested = false;
- std::string json2 = R"({"a":[{"b":1},{"b":2}]})";
- result = parser.parse(json2.c_str(), json2.size(), config);
- ASSERT_TRUE(result.has_value());
-}
-
-TEST(JsonParserTest, ParseInvalidJson) {
- JSONDataParser<SimdJSONParser> parser;
- ParseConfig config;
-
- auto result = parser.parse("{a:1}", 5, config);
- ASSERT_FALSE(result.has_value());
-
- result = parser.parse("", 0, config);
- ASSERT_FALSE(result.has_value());
-}
-
-TEST(JsonParserTest, ParseCornerCases) {
- JSONDataParser<SimdJSONParser> parser;
- ParseConfig config;
-
- auto result = parser.parse("{}", 2, config);
- ASSERT_TRUE(result.has_value());
-
- result = parser.parse("[]", 2, config);
- ASSERT_TRUE(result.has_value());
-
- result = parser.parse(R"({"a":"\n\t"})", 12, config);
- ASSERT_TRUE(result.has_value());
-}
-
-// Test cases for the selected code functionality
-TEST(JsonParserTest, TestIsPrefixFunction) {
- JSONDataParser<SimdJSONParser> parser;
- ParseConfig config;
-
- // Test is_prefix functionality through nested path parsing
- // This tests the is_prefix function used in checkAmbiguousStructure
-
- // Test case 1: Simple nested paths that should not be ambiguous
- std::string json1 = R"({"a": [{"b": 1}, {"b": 2}]})";
- auto result1 = parser.parse(json1.c_str(), json1.size(), config);
- ASSERT_TRUE(result1.has_value());
-
- // Test case 2: More complex nested paths
- std::string json2 = R"({"a": [{"b": {"c": 1}}, {"b": {"c": 2}}]})";
- auto result2 = parser.parse(json2.c_str(), json2.size(), config);
- ASSERT_TRUE(result2.has_value());
-
- // Test case 3: Deep nested structure
- std::string json3 = R"({"level1": {"level2": [{"level3": {"level4":
1}}]}})";
- auto result3 = parser.parse(json3.c_str(), json3.size(), config);
- ASSERT_TRUE(result3.has_value());
-}
-
-TEST(JsonParserTest, TestAmbiguousStructureDetection) {
- JSONDataParser<SimdJSONParser> parser;
- ParseConfig config;
- config.deprecated_enable_flatten_nested = true;
-
- // TODO: The following 3 cases no longer throw because
checkAmbiguousStructure requires
- // has_nested_in_flatten && is_top_array. "b" contains plain arrays (not
nested objects),
- // so has_nested=false → has_nested_in_flatten=false, and the ambiguity
check is skipped.
-
- // Test case 1: Arrays with different sizes in nested structure
- std::string json1 = R"([{"b": [1, 2]}, {"b": [1, 2, 3]}])";
- // EXPECT_ANY_THROW(parser.parse(json1.c_str(), json1.size(), config));
-
- // Test case 2: Arrays with same sizes should not throw
- std::string json2 = R"([{"b": [1, 2]}, {"b": [3, 4]}])";
- // EXPECT_ANY_THROW(parser.parse(json2.c_str(), json2.size(), config));
-
- // Test case 3: More complex nested array size mismatch (object-level,
is_top_array=false)
- std::string json3 = R"({"nested": [{"arr": [[1, 2], [3]]}, {"arr": [[1,
2], [3, 4]]}]})";
- // EXPECT_ANY_THROW(parser.parse(json3.c_str(), json3.size(), config));
-
- // Test case 4: Ambiguous structure with prefix paths
- // This should trigger the ambiguous structure exception
- std::string json4 = R"([{"a": {"c": 1}}, {"a": 2}])";
- EXPECT_ANY_THROW(parser.parse(json4.c_str(), json4.size(), config));
-}
-
-TEST(JsonParserTest, TestNestedArrayHandling) {
- JSONDataParser<SimdJSONParser> parser;
- ParseConfig config;
- config.deprecated_enable_flatten_nested = true;
-
- // Test case 1: Simple nested array handling
- std::string json1 = R"([{"b": 1}, {"c": 2}])";
- auto result1 = parser.parse(json1.c_str(), json1.size(), config);
- ASSERT_TRUE(result1.has_value());
- EXPECT_GT(result1->values.size(), 0);
-
- // Test case 2: Multi-level nested array
- std::string json2 = R"([{"a": {"b": 1}}, {"a": {"b": 2}}])";
- auto result2 = parser.parse(json2.c_str(), json2.size(), config);
- ASSERT_TRUE(result2.has_value());
- EXPECT_GT(result2->values.size(), 0);
-}
-
-TEST(JsonParserTest, TestNestedArrayWithDifferentConfigs) {
- JSONDataParser<SimdJSONParser> parser;
-
- // Test with flatten_nested = false
- ParseConfig config1;
- config1.deprecated_enable_flatten_nested = false;
-
- std::string json1 = R"([{"b": [1, 2]}, {"b": [3, 4]}])";
- auto result1 = parser.parse(json1.c_str(), json1.size(), config1);
- ASSERT_TRUE(result1.has_value());
- EXPECT_EQ(result1->values.size(), 1);
- EXPECT_EQ(result1->values[0].get_type(), doris::PrimitiveType::TYPE_JSONB);
-
- // Test with flatten_nested = true
- ParseConfig config2;
- config2.deprecated_enable_flatten_nested = true;
-
- // TODO: "b" contains plain arrays (no nested objects), so
has_nested=false,
- // has_nested_in_flatten=false, and checkAmbiguousStructure is not called.
- // EXPECT_ANY_THROW(parser.parse(json1.c_str(), json1.size(), config2));
-}
-
-// Test case for directly calling handleNewPath to cover the if
(!nested_key.empty()) branch
-TEST(JsonParserTest, TestHandleNewPathDirectCall) {
- JSONDataParser<SimdJSONParser> parser;
-
- // Create a ParseArrayContext
- JSONDataParser<SimdJSONParser>::ParseArrayContext ctx;
- ctx.current_size = 1;
- ctx.total_size = 2;
- ctx.has_nested_in_flatten = true;
- ctx.is_top_array = true;
-
- // Create a path with nested parts
- doris::PathInData::Parts path;
- // Create a nested part (is_nested = true)
- path.emplace_back("nested_key", true, 0); // is_nested = true
- path.emplace_back("inner_key", false, 0); // is_nested = false
-
- // Create a Field with array type (required for getNameOfNested to return
non-empty)
- doris::Array array_data;
- array_data.push_back(doris::Field::create_field<doris::TYPE_INT>(1));
- array_data.push_back(doris::Field::create_field<doris::TYPE_INT>(2));
- doris::Field value =
doris::Field::create_field<doris::TYPE_ARRAY>(std::move(array_data));
-
- // Create hash for the path
- UInt128 hash = doris::PathInData::get_parts_hash(path);
-
- // Call handleNewPath directly
- // This should trigger the if (!nested_key.empty()) branch
- parser.handleNewPath(hash, path, value, ctx);
-
- // Verify that the nested_sizes_by_key was populated
- EXPECT_EQ(ctx.nested_sizes_by_key.size(), 1);
-
- // Verify that the arrays_by_path was populated
- EXPECT_EQ(ctx.arrays_by_path.size(), 1);
- EXPECT_TRUE(ctx.arrays_by_path.find(hash) != ctx.arrays_by_path.end());
-}
-
-// Test case for testing the else branch in handleNewPath (when nested_sizes
is not empty)
-TEST(JsonParserTest, TestHandleNewPathElseBranch) {
- JSONDataParser<SimdJSONParser> parser;
-
- // Create a ParseArrayContext
- JSONDataParser<SimdJSONParser>::ParseArrayContext ctx;
- ctx.current_size = 2; // Start with size 2
- ctx.total_size = 3;
- ctx.has_nested_in_flatten = true;
- ctx.is_top_array = true;
-
- // Create a path with nested parts
- doris::PathInData::Parts path;
- path.emplace_back("nested_key", true, 0);
- path.emplace_back("inner_key", false, 0);
-
- // Create a Field with array type
- doris::Array array_data;
- array_data.push_back(doris::Field::create_field<doris::TYPE_INT>(1));
- array_data.push_back(doris::Field::create_field<doris::TYPE_INT>(2));
- doris::Field value =
doris::Field::create_field<doris::TYPE_ARRAY>(std::move(array_data));
-
- // Create hash for the path
- UInt128 hash = doris::PathInData::get_parts_hash(path);
-
- // First call to populate nested_sizes_by_key
- parser.handleNewPath(hash, path, value, ctx);
-
- // Verify nested_sizes_by_key was populated
-
EXPECT_EQ(ctx.nested_sizes_by_key.at(doris::StringRef("nested_key")).size(), 3);
- EXPECT_EQ(ctx.nested_sizes_by_key.at(doris::StringRef("nested_key"))[0],
0);
-
- // Create another array with same size
- doris::Array array_data2;
- array_data2.push_back(doris::Field::create_field<doris::TYPE_INT>(3));
- array_data2.push_back(doris::Field::create_field<doris::TYPE_INT>(4));
- doris::Field value2 =
doris::Field::create_field<doris::TYPE_ARRAY>(std::move(array_data2));
-
- // Second call should trigger the else branch (nested_sizes is not empty)
- ctx.is_top_array = false;
- ctx.has_nested_in_flatten = false;
- parser.handleNewPath(hash, path, value2, ctx);
-
- // Verify nested_sizes_by_key was updated
-
EXPECT_EQ(ctx.nested_sizes_by_key.at(doris::StringRef("nested_key")).size(), 3);
- EXPECT_EQ(ctx.nested_sizes_by_key.at(doris::StringRef("nested_key"))[1],
0);
-}
-
-TEST(JsonParserTest, ParseUInt64) {
- JSONDataParser<SimdJSONParser> parser;
- ParseConfig config;
-
- std::string json = R"({"a": 18446744073709551615})";
- auto result = parser.parse(json.c_str(), json.size(), config);
- ASSERT_TRUE(result.has_value());
- EXPECT_EQ(result->values.size(), 1);
- EXPECT_EQ(result->paths.size(), 1);
- EXPECT_EQ(result->paths[0].get_path(), "a");
- EXPECT_EQ(result->values[0].get_type(),
doris::PrimitiveType::TYPE_LARGEINT);
- EXPECT_EQ(result->values[0].get<doris::PrimitiveType::TYPE_LARGEINT>(),
- 18446744073709551615ULL);
-
- std::string array_json = R"({"a": [18446744073709551615]})";
- result = parser.parse(array_json.c_str(), array_json.size(), config);
- ASSERT_TRUE(result.has_value());
- EXPECT_EQ(result->values.size(), 1);
- EXPECT_EQ(result->paths.size(), 1);
- EXPECT_EQ(result->paths[0].get_path(), "a");
- EXPECT_EQ(result->values[0].get_type(), doris::PrimitiveType::TYPE_ARRAY);
- auto& array_field =
result->values[0].get<doris::PrimitiveType::TYPE_ARRAY>();
- EXPECT_EQ(array_field.size(), 1);
- EXPECT_EQ(array_field[0].get_type(), doris::PrimitiveType::TYPE_LARGEINT);
- EXPECT_EQ(array_field[0].get<doris::PrimitiveType::TYPE_LARGEINT>(),
18446744073709551615ULL);
-
- std::string nested_json = R"({"a": [{"b": 18446744073709551615}]})";
- config.deprecated_enable_flatten_nested = true;
- result = parser.parse(nested_json.c_str(), nested_json.size(), config);
- ASSERT_TRUE(result.has_value());
- EXPECT_EQ(result->values.size(), 1);
- EXPECT_EQ(result->paths.size(), 1);
- EXPECT_EQ(result->values[0].get_type(), doris::PrimitiveType::TYPE_ARRAY);
- auto& array_field_2 =
result->values[0].get<doris::PrimitiveType::TYPE_ARRAY>();
- EXPECT_EQ(array_field_2.size(), 1);
- EXPECT_EQ(array_field_2[0].get_type(),
doris::PrimitiveType::TYPE_LARGEINT);
- EXPECT_EQ(array_field_2[0].get<doris::PrimitiveType::TYPE_LARGEINT>(),
18446744073709551615ULL);
-}
-
-TEST(JsonParserTest, KeyLengthLimitByConfig) {
- struct ScopedMaxJsonKeyLength {
- int32_t old_value;
- explicit ScopedMaxJsonKeyLength(int32_t new_value)
- : old_value(doris::config::variant_max_json_key_length) {
- doris::config::variant_max_json_key_length = new_value;
- }
- ~ScopedMaxJsonKeyLength() { doris::config::variant_max_json_key_length
= old_value; }
- };
-
- JSONDataParser<SimdJSONParser> parser;
- ParseConfig config;
-
- {
- ScopedMaxJsonKeyLength guard(10);
- std::string key11(11, 'a');
-
- std::string obj_json = "{\"" + key11 + "\": 1}";
- EXPECT_ANY_THROW(parser.parse(obj_json.c_str(), obj_json.size(),
config));
-
- config.deprecated_enable_flatten_nested = false;
- std::string jsonb_json = "{\"a\": [{\"" + key11 + "\": 1}]}";
- EXPECT_ANY_THROW(parser.parse(jsonb_json.c_str(), jsonb_json.size(),
config));
- }
-
- {
- ScopedMaxJsonKeyLength guard(255);
- std::string key255(255, 'b');
-
- std::string obj_json = "{\"" + key255 + "\": 1}";
- auto result = parser.parse(obj_json.c_str(), obj_json.size(), config);
- ASSERT_TRUE(result.has_value());
-
- config.deprecated_enable_flatten_nested = false;
- std::string jsonb_json = "{\"a\": [{\"" + key255 + "\": 1}]}";
- result = parser.parse(jsonb_json.c_str(), jsonb_json.size(), config);
- ASSERT_TRUE(result.has_value());
- ASSERT_EQ(result->values.size(), 1);
- EXPECT_EQ(result->values[0].get_type(),
doris::PrimitiveType::TYPE_JSONB);
- }
-}
diff --git a/be/test/exprs/function/cast/cast_to_decimal128_perf.cpp
b/be/test/exprs/function/cast/cast_to_decimal128_perf.cpp
index b371de445d6..1ec43f8317a 100644
--- a/be/test/exprs/function/cast/cast_to_decimal128_perf.cpp
+++ b/be/test/exprs/function/cast/cast_to_decimal128_perf.cpp
@@ -26,7 +26,6 @@
#include "core/data_type/primitive_type.h"
#include "core/field.h"
#include "exprs/function/cast/cast_to_decimal_test.h"
-#include "util/json/json_parser.h"
#include "util/string_parser.hpp"
namespace doris {
diff --git a/be/test/storage/variant/index_storage_variant_debug_point_test.cpp
b/be/test/storage/variant/index_storage_variant_debug_point_test.cpp
index 9a5007b8eeb..475dee85190 100644
--- a/be/test/storage/variant/index_storage_variant_debug_point_test.cpp
+++ b/be/test/storage/variant/index_storage_variant_debug_point_test.cpp
@@ -210,7 +210,6 @@ TEST_F(IndexStorageVariantDebugPointTest,
ArrayPathIndexAcceptsMixedTypedElement
{R"({"c_arr": ["text"]})", R"({"c_arr": [1.1]})", R"({"c_arr":
[1.0]})",
R"({"c_arr": [90]})", R"({"c_arr": [90999999999999]})"},
0));
- rowset.batches.back().parse_to = ParseConfig::ParseTo::OnlyDocValueColumn;
auto rowset_result = write_rowset(rowset);
ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error();
diff --git a/be/test/testutil/index_storage_test_util.h
b/be/test/testutil/index_storage_test_util.h
index 31995566e17..873ce5db4fc 100644
--- a/be/test/testutil/index_storage_test_util.h
+++ b/be/test/testutil/index_storage_test_util.h
@@ -40,7 +40,6 @@
#include "storage/rowset/rowset.h"
#include "storage/tablet/tablet.h"
#include "storage/tablet/tablet_schema.h"
-#include "util/json/json_parser.h"
namespace doris {
@@ -117,7 +116,6 @@ struct IndexBatch {
std::vector<ColumnPtr> variant_columns_by_column;
bool deprecated_enable_flatten_nested = false;
bool check_duplicate_json_path = false;
- ParseConfig::ParseTo parse_to = ParseConfig::ParseTo::OnlySubcolumns;
static IndexBatch single_text(std::vector<std::string> values, int32_t
first_key = 0);
static IndexBatch single_variant(std::vector<std::string> jsons, int32_t
first_key = 0);
diff --git a/be/test/util/variant/variant_field_test.cpp
b/be/test/util/variant/variant_field_test.cpp
index 2bbec8cfb38..1c072fd80db 100644
--- a/be/test/util/variant/variant_field_test.cpp
+++ b/be/test/util/variant/variant_field_test.cpp
@@ -30,8 +30,6 @@
#include "common/exception.h"
#include "core/value/variant/variant_parquet_encoding.h"
#include "exprs/function/parse/variant_string_parse.h"
-#include "util/json/json_parser.h"
-#include "util/json/simd_json_parser.h"
namespace doris {
namespace {
@@ -541,29 +539,11 @@ TEST(VariantFieldTest, AllComparisonsThrow) {
EXPECT_THROW(static_cast<void>(left > right), Exception);
}
-// This is a narrow current-source comparison, not the deferred T0.2 semantics
baseline.
-TEST(VariantFieldTest, LegacyJsonDataParserStableSubsetDoesNotReplaceT02) {
- JSONDataParser<SimdJSONParser> legacy;
- ParseConfig config;
-
- auto old_scalar = legacy.parse("123", 3, config);
- ASSERT_TRUE(old_scalar.has_value());
- ASSERT_EQ(old_scalar->values.size(), 1);
- EXPECT_EQ(old_scalar->values[0].get<TYPE_BIGINT>(), 123);
+TEST(VariantFieldTest, EncodeJsonStableSubset) {
EXPECT_EQ(encode_json("123").ref().get_int(), 123);
-
- old_scalar = legacy.parse("\"text\"", 6, config);
- ASSERT_TRUE(old_scalar.has_value());
- EXPECT_EQ(old_scalar->values[0].get<TYPE_STRING>(), "text");
EXPECT_EQ(encode_json("\"text\"").ref().get_string(), StringRef("text"));
- const std::string object_json = R"({"a":1,"b":"x"})";
- auto old_object = legacy.parse(object_json.data(), object_json.size(),
config);
- ASSERT_TRUE(old_object.has_value());
- ASSERT_EQ(old_object->paths.size(), 2);
- EXPECT_EQ(old_object->paths[0].get_path(), "a");
- EXPECT_EQ(old_object->paths[1].get_path(), "b");
- VariantField new_object = encode_json(object_json);
+ VariantField new_object = encode_json(R"({"a":1,"b":"x"})");
VariantRef a;
ASSERT_TRUE(new_object.ref().object_find(StringRef("a"), &a));
EXPECT_EQ(a.get_int(), 1);
@@ -571,29 +551,14 @@ TEST(VariantFieldTest,
LegacyJsonDataParserStableSubsetDoesNotReplaceT02) {
ASSERT_TRUE(new_object.ref().object_find(StringRef("b"), &b));
EXPECT_EQ(b.get_string(), StringRef("x"));
- const std::string array_json = R"([1,null,"x"])";
- auto old_array = legacy.parse(array_json.data(), array_json.size(),
config);
- ASSERT_TRUE(old_array.has_value());
- const auto& old_elements = old_array->values[0].get<TYPE_ARRAY>();
- ASSERT_EQ(old_elements.size(), 3);
- EXPECT_EQ(old_elements[0].get<TYPE_BIGINT>(), 1);
- EXPECT_TRUE(old_elements[1].is_null());
- EXPECT_EQ(old_elements[2].get<TYPE_STRING>(), "x");
- VariantField new_array_field = encode_json(array_json);
+ VariantField new_array_field = encode_json(R"([1,null,"x"])");
VariantRef new_array = new_array_field.ref();
ASSERT_EQ(new_array.num_elements(), 3);
EXPECT_EQ(new_array.array_at(0).get_int(), 1);
EXPECT_TRUE(new_array.array_at(1).is_null());
EXPECT_EQ(new_array.array_at(2).get_string(), StringRef("x"));
- auto old_empty_object = legacy.parse("{}", 2, config);
- ASSERT_TRUE(old_empty_object.has_value());
- EXPECT_TRUE(old_empty_object->paths.empty());
EXPECT_EQ(encode_json("{}").ref().num_elements(), 0);
-
- auto old_empty_array = legacy.parse("[]", 2, config);
- ASSERT_TRUE(old_empty_array.has_value());
- EXPECT_TRUE(old_empty_array->values[0].get<TYPE_ARRAY>().empty());
EXPECT_EQ(encode_json("[]").ref().num_elements(), 0);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
