This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch branch-2.0-var
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0-var by this push:
new 361ae5466ed [performance](variant) support topn 2phase read for
variant column (#28381)
361ae5466ed is described below
commit 361ae5466ed4b5fefa497a6d75e865655318814e
Author: lihangyu <[email protected]>
AuthorDate: Fri Dec 15 19:56:47 2023 +0800
[performance](variant) support topn 2phase read for variant column (#28381)
* [performance](variant) support topn 2phase read for variant column
* fix segment use after free
* fix use after free
* add case
* fix case
* fix
---
.../rowset/segment_v2/hierarchical_data_reader.h | 4 +-
be/src/olap/rowset/segment_v2/segment.cpp | 75 ++++++++++++++++++
be/src/olap/rowset/segment_v2/segment.h | 11 +++
be/src/olap/tablet_schema.cpp | 14 ++++
be/src/olap/tablet_schema.h | 5 ++
be/src/runtime/descriptors.cpp | 6 +-
be/src/service/internal_service.cpp | 90 +++++++++++++++-------
be/src/vec/columns/column_object.cpp | 9 ++-
be/src/vec/exec/scan/new_olap_scanner.cpp | 40 ++++------
be/src/vec/json/path_in_data.cpp | 11 +++
be/src/vec/json/path_in_data.h | 1 +
.../java/org/apache/doris/analysis/SelectStmt.java | 3 +-
gensrc/proto/descriptors.proto | 1 +
regression-test/data/variant_p0/sql/gh_data.out | 86 +++++++++++----------
.../suites/variant_p0/column_size_limit.groovy | 18 ++++-
regression-test/suites/variant_p0/load.groovy | 6 +-
regression-test/suites/variant_p0/sql/gh_data.sql | 11 ++-
17 files changed, 276 insertions(+), 115 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h
b/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h
index a56e80aa464..99bbb36ab87 100644
--- a/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h
+++ b/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h
@@ -140,7 +140,7 @@ private:
// copy container variant to dst variant, todo avoid copy
variant.insert_range_from(container_variant, 0, nrows);
- variant.set_num_rows(nrows);
+ // variant.set_num_rows(nrows);
_rows_read += nrows;
variant.finalize();
#ifndef NDEBUG
@@ -180,7 +180,7 @@ public:
private:
Status extract_to(vectorized::MutableColumnPtr& dst, size_t nrows);
- const TabletColumn& _col;
+ TabletColumn _col;
// may shared among different column iterators
std::unique_ptr<StreamReader> _root_reader;
};
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp
b/be/src/olap/rowset/segment_v2/segment.cpp
index f1285977047..9c12fc0e074 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -60,10 +60,12 @@
#include "util/slice.h" // Slice
#include "vec/columns/column.h"
#include "vec/common/string_ref.h"
+#include "vec/core/field.h"
#include "vec/data_types/data_type.h"
#include "vec/data_types/data_type_factory.hpp"
#include "vec/data_types/data_type_nullable.h"
#include "vec/data_types/data_type_object.h"
+#include "vec/json/path_in_data.h"
#include "vec/olap/vgeneric_iterators.h"
namespace doris {
@@ -672,5 +674,78 @@ bool Segment::is_same_file_col_type_with_expected(int32_t
cid, const Schema& sch
return (!file_column_type) || (file_column_type &&
file_column_type->equals(*expected_type));
}
+ std::shared_ptr<const vectorized::IDataType>
+ Segment::get_data_type_of(vectorized::PathInData path,
+ bool ignore_children) const {
+ auto node = _sub_column_tree.find_leaf(path);
+ if (node) {
+ if (ignore_children || node->children.empty()) {
+ return node->data.file_column_type;
+ }
+ }
+ // it contains children or column missing in storage, so treat it as
variant
+ return std::make_shared<vectorized::DataTypeObject>();
+}
+
+Status Segment::seek_and_read_by_rowid(const TabletSchema& schema,
SlotDescriptor* slot,
+ uint32_t row_id,
vectorized::MutableColumnPtr& result,
+ OlapReaderStatistics& stats,
+ std::unique_ptr<ColumnIterator>&
iterator_hint) {
+ StorageReadOptions storage_read_opt;
+ storage_read_opt.io_ctx.reader_type = ReaderType::READER_QUERY;
+ io::IOContext io_ctx;
+ io_ctx.reader_type = ReaderType::READER_QUERY;
+ segment_v2::ColumnIteratorOptions opt {
+ .file_reader = file_reader().get(),
+ .stats = &stats,
+ .use_page_cache = !config::disable_storage_page_cache,
+ .io_ctx = io_ctx,
+ };
+ std::vector<segment_v2::rowid_t> single_row_loc {row_id};
+ if (!slot->column_paths().empty()) {
+ vectorized::PathInData
path(schema.column_by_uid(slot->col_unique_id()).name_lower_case(),
+ slot->column_paths());
+ auto storage_type = get_data_type_of(path, false);
+ vectorized::MutableColumnPtr file_storage_column =
storage_type->create_column();
+ DCHECK(storage_type != nullptr);
+ TabletColumn column = TabletColumn::create_materialized_variant_column(
+ schema.column_by_uid(slot->col_unique_id()).name_lower_case(),
slot->column_paths(),
+ slot->col_unique_id());
+ if (iterator_hint == nullptr) {
+ RETURN_IF_ERROR(
+ new_column_iterator(column, &iterator_hint,
&storage_read_opt));
+ RETURN_IF_ERROR(iterator_hint->init(opt));
+ }
+ RETURN_IF_ERROR(
+ iterator_hint->read_by_rowids(single_row_loc.data(), 1,
file_storage_column));
+ // iterator_hint.reset(nullptr);
+ // Get it's inner field, for JSONB case
+ vectorized::Field field = remove_nullable(storage_type)->get_default();
+ file_storage_column->get(0, field);
+ result->insert(field);
+ } else {
+ int index = -1;
+ if (slot->col_unique_id() >= 0) {
+ index = schema.field_index(slot->col_name());
+ } else {
+ index = schema.field_index(slot->col_name());
+ }
+ if (index < 0) {
+ std::stringstream ss;
+ ss << "field name is invalid. field=" << slot->col_name()
+ << ", field_name_to_index=" << schema.get_all_field_names();
+ return Status::InternalError(ss.str());
+ }
+ storage_read_opt.io_ctx.reader_type = ReaderType::READER_QUERY;
+ if (iterator_hint == nullptr) {
+ RETURN_IF_ERROR(
+ new_column_iterator(schema.column(index), &iterator_hint,
&storage_read_opt));
+ RETURN_IF_ERROR(iterator_hint->init(opt));
+ }
+ RETURN_IF_ERROR(iterator_hint->read_by_rowids(single_row_loc.data(),
1, result));
+ }
+ return Status::OK();
+}
+
} // namespace segment_v2
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/segment.h
b/be/src/olap/rowset/segment_v2/segment.h
index 7983f86b3a8..f504d624a96 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -39,9 +39,14 @@
#include "olap/rowset/segment_v2/page_handle.h"
#include "olap/schema.h"
#include "olap/tablet_schema.h"
+#include "runtime/descriptors.h"
#include "util/once.h"
#include "util/slice.h"
+#include "vec/columns/column.h"
#include "vec/columns/subcolumn_tree.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/json/path_in_data.h"
namespace doris {
namespace vectorized {
@@ -124,6 +129,10 @@ public:
Status read_key_by_rowid(uint32_t row_id, std::string* key);
+ Status seek_and_read_by_rowid(const TabletSchema& schema, SlotDescriptor*
slot, uint32_t row_id,
+ vectorized::MutableColumnPtr& result,
OlapReaderStatistics& stats,
+ std::unique_ptr<ColumnIterator>&
iterator_hint);
+
Status load_index();
Status load_pk_index_and_bf();
@@ -147,6 +156,8 @@ public:
std::shared_ptr<const vectorized::IDataType> get_data_type_of(const Field&
filed,
bool
ignore_children) const;
+ std::shared_ptr<const vectorized::IDataType>
get_data_type_of(vectorized::PathInData path,
+ bool
ignore_children) const;
// If column in segment is the same type in schema
bool is_same_file_col_type_with_expected(int32_t cid, const Schema& schema,
bool ignore_children) const;
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 0191241e5d3..c227b9cc06a 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -457,6 +457,20 @@ void TabletColumn::init_from_pb(const ColumnPB& column) {
}
}
+TabletColumn TabletColumn::create_materialized_variant_column(const
std::string& root,
+ const
std::vector<std::string>& paths,
+ int32_t
parent_unique_id) {
+ TabletColumn subcol;
+ subcol.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT);
+ subcol.set_is_nullable(true);
+ subcol.set_unique_id(-1);
+ subcol.set_parent_unique_id(parent_unique_id);
+ vectorized::PathInData path(root, paths);
+ subcol.set_path_info(path);
+ subcol.set_name(path.get_path());
+ return subcol;
+}
+
void TabletColumn::to_schema_pb(ColumnPB* column) const {
column->set_unique_id(_unique_id);
column->set_name(_col_name);
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 087fb437eca..79401d0d6b8 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -88,6 +88,11 @@ public:
_type == FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE ||
_type == FieldType::OLAP_FIELD_TYPE_AGG_STATE;
}
+ // Such columns are not exist in frontend schema info, so we need to
+ // add them into tablet_schema for later column indexing.
+ static TabletColumn create_materialized_variant_column(const std::string&
root,
+ const
std::vector<std::string>& paths,
+ int32_t
parent_unique_id);
bool has_default_value() const { return _has_default_value; }
std::string default_value() const { return _default_value; }
size_t length() const { return _length; }
diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp
index f8125588ae5..435df6648c6 100644
--- a/be/src/runtime/descriptors.cpp
+++ b/be/src/runtime/descriptors.cpp
@@ -83,7 +83,8 @@ SlotDescriptor::SlotDescriptor(const PSlotDescriptor& pdesc)
_field_idx(-1),
_is_materialized(pdesc.is_materialized()),
_is_key(pdesc.is_key()),
- _need_materialize(true) {}
+ _need_materialize(true),
+ _column_paths(pdesc.column_paths().begin(),
pdesc.column_paths().end()) {}
void SlotDescriptor::to_protobuf(PSlotDescriptor* pslot) const {
pslot->set_id(_id);
@@ -100,6 +101,9 @@ void SlotDescriptor::to_protobuf(PSlotDescriptor* pslot)
const {
pslot->set_col_unique_id(_col_unique_id);
pslot->set_is_key(_is_key);
pslot->set_col_type(_col_type);
+ for (const std::string& path : _column_paths) {
+ pslot->add_column_paths(path);
+ }
}
vectorized::MutableColumnPtr SlotDescriptor::get_empty_mutable_column() const {
diff --git a/be/src/service/internal_service.cpp
b/be/src/service/internal_service.cpp
index 597c8fdfef4..2a69400fe3f 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -25,6 +25,7 @@
#include <butil/errno.h>
#include <butil/iobuf.h>
#include <fcntl.h>
+#include <fmt/core.h>
#include <gen_cpp/MasterService_types.h>
#include <gen_cpp/PaloInternalService_types.h>
#include <gen_cpp/PlanNodes_types.h>
@@ -46,6 +47,7 @@
#include <set>
#include <sstream>
#include <string>
+#include <unordered_map>
#include <utility>
#include <vector>
@@ -69,6 +71,7 @@
#include "olap/rowset/segment_v2/common.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
#include "olap/rowset/segment_v2/segment.h"
+#include "olap/rowset/segment_v2/segment_iterator.h"
#include "olap/segment_loader.h"
#include "olap/storage_engine.h"
#include "olap/tablet.h"
@@ -1596,6 +1599,38 @@ auto scope_timer_run(Func fn, int64_t* cost) ->
decltype(fn()) {
return res;
}
+struct IteratorKey {
+ int64_t tablet_id;
+ RowsetId rowset_id;
+ uint64_t segment_id;
+ int slot_id;
+
+ // unordered map std::equal_to
+ bool operator==(const IteratorKey& rhs) const {
+ return tablet_id == rhs.tablet_id && rowset_id == rhs.rowset_id &&
+ segment_id == rhs.segment_id && slot_id == rhs.slot_id;
+ }
+};
+
+struct HashOfIteratorKey {
+ size_t operator()(const IteratorKey& key) const {
+ size_t seed = 0;
+ seed = HashUtil::hash64(&key.tablet_id, sizeof(key.tablet_id), seed);
+ seed = HashUtil::hash64(&key.rowset_id.hi, sizeof(key.rowset_id.hi),
seed);
+ seed = HashUtil::hash64(&key.rowset_id.mi, sizeof(key.rowset_id.mi),
seed);
+ seed = HashUtil::hash64(&key.rowset_id.lo, sizeof(key.rowset_id.lo),
seed);
+ seed = HashUtil::hash64(&key.segment_id, sizeof(key.segment_id), seed);
+ seed = HashUtil::hash64(&key.slot_id, sizeof(key.slot_id), seed);
+ return seed;
+ }
+};
+
+struct IteratorItem {
+ std::unique_ptr<ColumnIterator> iterator;
+ // for holding the reference of segment to avoid use after release
+ SegmentSharedPtr segment;
+};
+
Status PInternalServiceImpl::_multi_get(const PMultiGetRequest& request,
PMultiGetResponse* response) {
OlapReaderStatistics stats;
@@ -1620,6 +1655,7 @@ Status PInternalServiceImpl::_multi_get(const
PMultiGetRequest& request,
full_read_schema.append_column(TabletColumn(column_pb));
}
+ std::unordered_map<IteratorKey, IteratorItem, HashOfIteratorKey>
iterator_map;
// read row by row
for (size_t i = 0; i < request.row_locs_size(); ++i) {
const auto& row_loc = request.row_locs(i);
@@ -1658,8 +1694,8 @@ Status PInternalServiceImpl::_multi_get(const
PMultiGetRequest& request,
},
&acquire_segments_ms));
// find segment
- auto it = std::find_if(segment_cache.get_segments().begin(),
- segment_cache.get_segments().end(),
+ auto it = std::find_if(segment_cache.get_segments().cbegin(),
+ segment_cache.get_segments().cend(),
[&row_loc](const segment_v2::SegmentSharedPtr&
seg) {
return seg->id() == row_loc.segment_id();
});
@@ -1687,33 +1723,28 @@ Status PInternalServiceImpl::_multi_get(const
PMultiGetRequest& request,
if (result_block.is_empty_column()) {
result_block = vectorized::Block(desc.slots(),
request.row_locs().size());
}
+ VLOG_DEBUG << "Read row location "
+ << fmt::format("{}, {}, {}, {}", row_location.tablet_id,
+
row_location.row_location.rowset_id.to_string(),
+ row_location.row_location.segment_id,
+ row_location.row_location.row_id);
for (int x = 0; x < desc.slots().size(); ++x) {
- int index = -1;
- if (desc.slots()[x]->col_unique_id() >= 0) {
- // light sc enabled
- index =
full_read_schema.field_index(desc.slots()[x]->col_unique_id());
- } else {
- index =
full_read_schema.field_index(desc.slots()[x]->col_name());
- }
- if (index < 0) {
- std::stringstream ss;
- ss << "field name is invalid. field=" <<
desc.slots()[x]->col_name()
- << ", field_name_to_index=" <<
full_read_schema.get_all_field_names();
- return Status::InternalError(ss.str());
- }
- std::unique_ptr<segment_v2::ColumnIterator> column_iterator;
+ auto row_id =
static_cast<segment_v2::rowid_t>(row_loc.ordinal_id());
vectorized::MutableColumnPtr column =
result_block.get_by_position(x).column->assume_mutable();
- RETURN_IF_ERROR(
-
segment->new_column_iterator(full_read_schema.column(index), &column_iterator));
- segment_v2::ColumnIteratorOptions opt;
- opt.file_reader = segment->file_reader().get();
- opt.stats = &stats;
- opt.use_page_cache = !config::disable_storage_page_cache;
- column_iterator->init(opt);
- std::vector<segment_v2::rowid_t> single_row_loc {
- static_cast<segment_v2::rowid_t>(row_loc.ordinal_id())};
-
RETURN_IF_ERROR(column_iterator->read_by_rowids(single_row_loc.data(), 1,
column));
+ IteratorKey iterator_key {.tablet_id = tablet->tablet_id(),
+ .rowset_id = rowset_id,
+ .segment_id = row_loc.segment_id(),
+ .slot_id = desc.slots()[x]->id()};
+ IteratorItem& iterator_item = iterator_map[iterator_key];
+ if (iterator_item.segment == nullptr) {
+ // hold the reference
+ iterator_map[iterator_key].segment = segment;
+ }
+ segment = iterator_item.segment;
+ RETURN_IF_ERROR(segment->seek_and_read_by_rowid(full_read_schema,
desc.slots()[x],
+ row_id, column,
stats,
+
iterator_item.iterator));
}
}
// serialize block if not empty
@@ -1733,11 +1764,13 @@ Status PInternalServiceImpl::_multi_get(const
PMultiGetRequest& request,
"hit_cached_pages:{}, total_pages_read:{},
compressed_bytes_read:{}, "
"io_latency:{}ns, "
"uncompressed_bytes_read:{},"
+ "bytes_read:{},"
"acquire_tablet_ms:{}, acquire_rowsets_ms:{},
acquire_segments_ms:{}, "
"lookup_row_data_ms:{}",
stats.cached_pages_num, stats.total_pages_num,
stats.compressed_bytes_read,
- stats.io_ns, stats.uncompressed_bytes_read,
acquire_tablet_ms,
- acquire_rowsets_ms, acquire_segments_ms,
lookup_row_data_ms);
+ stats.io_ns, stats.uncompressed_bytes_read,
stats.bytes_read,
+ acquire_tablet_ms, acquire_rowsets_ms,
acquire_segments_ms,
+ lookup_row_data_ms);
return Status::OK();
}
@@ -1746,6 +1779,7 @@ void
PInternalServiceImpl::multiget_data(google::protobuf::RpcController* contro
PMultiGetResponse* response,
google::protobuf::Closure* done) {
bool ret = _light_work_pool.try_offer([request, response, done, this]() {
+ signal::set_signal_task_id(request->query_id());
// multi get data by rowid
MonotonicStopWatch watch;
watch.start();
diff --git a/be/src/vec/columns/column_object.cpp
b/be/src/vec/columns/column_object.cpp
index c8818505790..c889562d83b 100644
--- a/be/src/vec/columns/column_object.cpp
+++ b/be/src/vec/columns/column_object.cpp
@@ -642,11 +642,12 @@ void ColumnObject::for_each_subcolumn(ColumnCallback
callback) {
}
void ColumnObject::insert_from(const IColumn& src, size_t n) {
- const auto& src_v = assert_cast<const ColumnObject&>(src);
+ const auto* src_v = check_and_get_column<ColumnObject>(src);
// optimize when src and this column are scalar variant, since try_insert
is inefficiency
- if (src_v.is_scalar_variant() && is_scalar_variant() &&
- src_v.get_root_type()->equals(*get_root_type()) &&
src_v.is_finalized() && is_finalized()) {
-
assert_cast<ColumnNullable&>(*get_root()).insert_from(*src_v.get_root(), n);
+ if (src_v != nullptr && src_v->is_scalar_variant() && is_scalar_variant()
&&
+ src_v->get_root_type()->equals(*get_root_type()) &&
src_v->is_finalized() &&
+ is_finalized()) {
+
assert_cast<ColumnNullable&>(*get_root()).insert_from(*src_v->get_root(), n);
++num_rows;
return;
}
diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp
b/be/src/vec/exec/scan/new_olap_scanner.cpp
index d6c84b193af..8f319af81f1 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.cpp
+++ b/be/src/vec/exec/scan/new_olap_scanner.cpp
@@ -56,6 +56,7 @@
#include "vec/exec/scan/new_olap_scan_node.h"
#include "vec/exec/scan/vscan_node.h"
#include "vec/exprs/vexpr_context.h"
+#include "vec/json/path_in_data.h"
#include "vec/olap/block_reader.h"
namespace doris::vectorized {
@@ -404,20 +405,6 @@ Status NewOlapScanner::_init_tablet_reader_params(
return Status::OK();
}
-vectorized::PathInData NewOlapScanner::_build_path(SlotDescriptor* slot) {
- PathInDataBuilder path_builder;
- const std::string& col_name = slot->col_name_lower_case();
- auto delimeter_index = col_name.find(".");
- std::string_view root_name = delimeter_index == std::string::npos
- ? col_name
- : std::string_view(col_name.data(),
delimeter_index);
- path_builder = path_builder.append(root_name, false);
- for (const std::string& path : slot->column_paths()) {
- path_builder.append(path, false);
- }
- return path_builder.build();
-}
-
Status NewOlapScanner::_init_variant_columns() {
// Parent column has path info to distinction from each other
for (auto slot : _output_tuple_desc->slots()) {
@@ -430,21 +417,18 @@ Status NewOlapScanner::_init_variant_columns() {
if (slot->type().is_variant_type()) {
// Such columns are not exist in frontend schema info, so we need
to
// add them into tablet_schema for later column indexing.
- TabletColumn subcol;
- subcol.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT);
- subcol.set_name(slot->col_name());
- subcol.set_is_nullable(true);
- subcol.set_unique_id(slot->col_unique_id());
- PathInData path = _build_path(slot);
- subcol.set_path_info(path);
- if (_tablet_schema->field_index(path) < 0) {
+ TabletColumn subcol =
TabletColumn::create_materialized_variant_column(
+
_tablet_schema->column_by_uid(slot->col_unique_id()).name_lower_case(),
+ slot->column_paths(), slot->col_unique_id());
+ if (_tablet_schema->field_index(subcol.path_info()) < 0) {
_tablet_schema->append_column(subcol,
TabletSchema::ColumnType::VARIANT);
}
} else if (!slot->column_paths().empty()) {
- // Extracted materialized columns update it's path info
- PathInData path = _build_path(slot);
- int index = _tablet_schema->field_index(slot->col_unique_id());
- _tablet_schema->mutable_columns()[index].set_path_info(path);
+ CHECK(false) << "Not implemented";
+ // // Extracted materialized columns update it's path info
+ // PathInData path = _build_path(slot);
+ // int index = _tablet_schema->field_index(slot->col_unique_id());
+ // _tablet_schema->mutable_columns()[index].set_path_info(path);
}
}
return Status::OK();
@@ -462,7 +446,9 @@ Status NewOlapScanner::_init_return_columns() {
// variant column using path to index a column
int32_t index = 0;
if (slot->type().is_variant_type()) {
- index = _tablet_schema->field_index(_build_path(slot));
+ index = _tablet_schema->field_index(PathInData(
+
_tablet_schema->column_by_uid(slot->col_unique_id()).name_lower_case(),
+ slot->column_paths()));
} else {
index = slot->col_unique_id() >= 0 ?
_tablet_schema->field_index(slot->col_unique_id())
:
_tablet_schema->field_index(slot->col_name());
diff --git a/be/src/vec/json/path_in_data.cpp b/be/src/vec/json/path_in_data.cpp
index 0ed1975339f..c32f97d7d0a 100644
--- a/be/src/vec/json/path_in_data.cpp
+++ b/be/src/vec/json/path_in_data.cpp
@@ -46,11 +46,22 @@ PathInData::PathInData(const PathInData& other) :
path(other.path) {
build_parts(other.get_parts());
}
+PathInData::PathInData(const std::string& root, const
std::vector<std::string>& paths) {
+ PathInDataBuilder path_builder;
+ path_builder.append(root, false);
+ for (const std::string& path : paths) {
+ path_builder.append(path, false);
+ }
+ build_path(path_builder.get_parts());
+ build_parts(path_builder.get_parts());
+}
+
PathInData::PathInData(const std::vector<std::string>& paths) {
PathInDataBuilder path_builder;
for (size_t i = 0; i < paths.size(); ++i) {
path_builder.append(paths[i], false);
}
+ build_path(path_builder.get_parts());
build_parts(path_builder.get_parts());
}
diff --git a/be/src/vec/json/path_in_data.h b/be/src/vec/json/path_in_data.h
index 87278c1c925..ddb27b14cf5 100644
--- a/be/src/vec/json/path_in_data.h
+++ b/be/src/vec/json/path_in_data.h
@@ -61,6 +61,7 @@ public:
explicit PathInData(std::string_view path_);
explicit PathInData(const Parts& parts_);
explicit PathInData(const std::vector<std::string>& paths);
+ explicit PathInData(const std::string& root, const
std::vector<std::string>& pathsd);
PathInData(const PathInData& other);
PathInData& operator=(const PathInData& other);
static UInt128 get_parts_hash(const Parts& parts_);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
index 8f34046401c..fe7a804f64d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
@@ -798,8 +798,7 @@ public class SelectStmt extends QueryStmt {
LOG.debug("only support duplicate key or MOW model");
return false;
}
- if (!olapTable.getEnableLightSchemaChange() ||
!Strings.isNullOrEmpty(olapTable.getStoragePolicy())
- || olapTable.hasVariantColumns()) {
+ if (!olapTable.getEnableLightSchemaChange()) {
return false;
}
if (getOrderByElements() != null) {
diff --git a/gensrc/proto/descriptors.proto b/gensrc/proto/descriptors.proto
index 270199cc020..8762a78c0f4 100644
--- a/gensrc/proto/descriptors.proto
+++ b/gensrc/proto/descriptors.proto
@@ -38,6 +38,7 @@ message PSlotDescriptor {
optional bool is_key = 12;
optional bool is_auto_increment = 13;
optional int32 col_type = 14 [default = 0];
+ repeated string column_paths = 15;
};
message PTupleDescriptor {
diff --git a/regression-test/data/variant_p0/sql/gh_data.out
b/regression-test/data/variant_p0/sql/gh_data.out
index 232f4071d33..257f1fb0ce6 100644
--- a/regression-test/data/variant_p0/sql/gh_data.out
+++ b/regression-test/data/variant_p0/sql/gh_data.out
@@ -3,65 +3,47 @@
0
-- !gh_data_2 --
-5000
+0
-- !gh_data_3 --
+0
+
+-- !gh_data_4 --
+5000
+
+-- !gh_data_5 --
leonardomso/33-js-concepts 3
ytdl-org/youtube-dl 3
Bogdanp/neko 2
bminossi/AllVideoPocsFromHackerOne 2
disclose/diodata 2
--- !gh_data_4 --
-14690758274
-
--- !gh_data_5 --
-73453762334584
-
-- !gh_data_6 --
-457806339
+14690758274
-- !gh_data_7 --
-0
+73453762334584
-- !gh_data_8 --
-19829
+457806339
-- !gh_data_9 --
-\N
-\N
-\N
-\N
-\N
-\N
-\N
-\N
-\N
-\N
+0
-- !gh_data_10 --
-49390617
-64890096
-10696700
-33066637
-32271952
-2051941
-32271952
-57325392
-42386044
-73801003
+19829
-- !gh_data_11 --
-27
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:27Z","id":"14690746717","public":1,"actor":{"gravatar_id":"","display_login":"sergdudnik","url":"https://api.github.com/users/sergdudnik","id":16341546,"login":"sergdudnik","avatar_url":"https://avatars.githubusercontent.com/u/16341546?"},"repo":{"url":"https://api.github.com/repos/leonardomso/33-js-concepts","id":147350463,"name":"leonardomso/33-js-concepts"},"type":"WatchEvent"}
-36
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:27Z","id":"14690746732","public":1,"actor":{"gravatar_id":"","display_login":"juliusHuelsmann","url":"https://api.github.com/users/juliusHuelsmann","id":9212314,"login":"juliusHuelsmann","avatar_url":"https://avatars.githubusercontent.com/u/9212314?"},"repo":{"url":"https://api.github.com/repos/odeke-em/drive","id":26109545,"name":"odeke-em/drive"},"type":"WatchEvent"}
-46
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:27Z","id":"14690746749","org":{"gravatar_id":"","url":"https://api.github.com/orgs/GO-LiFE","id":38434522,"login":"GO-LiFE","avatar_url":"https://avatars.githubusercontent.com/u/38434522?"},"public":1,"actor":{"gravatar_id":"","display_login":"okbean","url":"https://api.github.com/users/okbean","id":75969386,"login":"okbean","avatar_url":"https://avatars.githubusercontent.com/u/75969386?"},"repo":{"url":"https://api.github
[...]
-56
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:28Z","id":"14690746773","public":1,"actor":{"gravatar_id":"","display_login":"PWDream","url":"https://api.github.com/users/PWDream","id":4903755,"login":"PWDream","avatar_url":"https://avatars.githubusercontent.com/u/4903755?"},"repo":{"url":"https://api.github.com/repos/MrXujiang/h5-Dooring","id":289417971,"name":"MrXujiang/h5-Dooring"},"type":"WatchEvent"}
-86
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:29Z","id":"14690746843","public":1,"actor":{"gravatar_id":"","display_login":"Gui-Yom","url":"https://api.github.com/users/Gui-Yom","id":25181283,"login":"Gui-Yom","avatar_url":"https://avatars.githubusercontent.com/u/25181283?"},"repo":{"url":"https://api.github.com/repos/redsaph/cleartext","id":106453399,"name":"redsaph/cleartext"},"type":"WatchEvent"}
-98
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:29Z","id":"14690746866","org":{"gravatar_id":"","url":"https://api.github.com/orgs/sherlock-project","id":48293496,"login":"sherlock-project","avatar_url":"https://avatars.githubusercontent.com/u/48293496?"},"public":1,"actor":{"gravatar_id":"","display_login":"humaidk2","url":"https://api.github.com/users/humaidk2","id":12982026,"login":"humaidk2","avatar_url":"https://avatars.githubusercontent.com/u/12982026?"},"repo":{";
[...]
-101
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:29Z","id":"14690746870","public":1,"actor":{"gravatar_id":"","display_login":"hasantezcan","url":"https://api.github.com/users/hasantezcan","id":32804505,"login":"hasantezcan","avatar_url":"https://avatars.githubusercontent.com/u/32804505?"},"repo":{"url":"https://api.github.com/repos/okandavut/react-spotify-nowplaying","id":326215605,"name":"okandavut/react-spotify-nowplaying"},"type":"WatchEvent"}
-112
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:30Z","id":"14690746899","public":1,"actor":{"gravatar_id":"","display_login":"nicholas-robertson","url":"https://api.github.com/users/nicholas-robertson","id":17681331,"login":"nicholas-robertson","avatar_url":"https://avatars.githubusercontent.com/u/17681331?"},"repo":{"url":"https://api.github.com/repos/sentriz/gonic","id":178435468,"name":"sentriz/gonic"},"type":"WatchEvent"}
-122
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:30Z","id":"14690746914","org":{"gravatar_id":"","url":"https://api.github.com/orgs/netlify-labs","id":47546088,"login":"netlify-labs","avatar_url":"https://avatars.githubusercontent.com/u/47546088?"},"public":1,"actor":{"gravatar_id":"","display_login":"javaniecampbell","url":"https://api.github.com/users/javaniecampbell","id":1676496,"login":"javaniecampbell","avatar_url":"https://avatars.githubusercontent.com/u/1676496?
[...]
-169
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:32Z","id":"14690747028","org":{"gravatar_id":"","url":"https://api.github.com/orgs/microsoft","id":6154722,"login":"microsoft","avatar_url":"https://avatars.githubusercontent.com/u/6154722?"},"public":1,"actor":{"gravatar_id":"","display_login":"Yxnt","url":"https://api.github.com/users/Yxnt","id":10323352,"login":"Yxnt","avatar_url":"https://avatars.githubusercontent.com/u/10323352?"},"repo":{"url":"https://api.github.co
[...]
+27
{"url":"https://api.github.com/repos/leonardomso/33-js-concepts","id":147350463,"name":"leonardomso/33-js-concepts"}
+36
{"url":"https://api.github.com/repos/odeke-em/drive","id":26109545,"name":"odeke-em/drive"}
+46
{"url":"https://api.github.com/repos/GO-LiFE/GoFIT_SDK_Android","id":141905736,"name":"GO-LiFE/GoFIT_SDK_Android"}
+56
{"url":"https://api.github.com/repos/MrXujiang/h5-Dooring","id":289417971,"name":"MrXujiang/h5-Dooring"}
+86
{"url":"https://api.github.com/repos/redsaph/cleartext","id":106453399,"name":"redsaph/cleartext"}
+98
{"url":"https://api.github.com/repos/sherlock-project/sherlock","id":162998479,"name":"sherlock-project/sherlock"}
+101
{"url":"https://api.github.com/repos/okandavut/react-spotify-nowplaying","id":326215605,"name":"okandavut/react-spotify-nowplaying"}
+112
{"url":"https://api.github.com/repos/sentriz/gonic","id":178435468,"name":"sentriz/gonic"}
+122
{"url":"https://api.github.com/repos/netlify-labs/react-netlify-identity-widget","id":182606378,"name":"netlify-labs/react-netlify-identity-widget"}
+169
{"url":"https://api.github.com/repos/microsoft/BotBuilder-Samples","id":68730444,"name":"microsoft/BotBuilder-Samples"}
-- !gh_data_12 --
2051941 1
@@ -75,3 +57,27 @@ disclose/diodata 2
64890096 1
73801003 1
+-- !gh_data_13 --
+27
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:27Z","id":"14690746717","repo":{"url":"https://api.github.com/repos/leonardomso/33-js-concepts","id":147350463,"name":"leonardomso/33-js-concepts"},"actor":{"gravatar_id":"","display_login":"sergdudnik","url":"https://api.github.com/users/sergdudnik","id":16341546,"login":"sergdudnik","avatar_url":"https://avatars.githubusercontent.com/u/16341546?"},"public":1,"type":"WatchEvent"}
{"payload":{"action":"started"},"created_a [...]
+36
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:27Z","id":"14690746732","repo":{"url":"https://api.github.com/repos/odeke-em/drive","id":26109545,"name":"odeke-em/drive"},"actor":{"gravatar_id":"","display_login":"juliusHuelsmann","url":"https://api.github.com/users/juliusHuelsmann","id":9212314,"login":"juliusHuelsmann","avatar_url":"https://avatars.githubusercontent.com/u/9212314?"},"public":1,"type":"WatchEvent"}
{"payload":{"action":"started"},"created_at":"2021-01- [...]
+46
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:27Z","org":{"gravatar_id":"","url":"https://api.github.com/orgs/GO-LiFE","id":38434522,"login":"GO-LiFE","avatar_url":"https://avatars.githubusercontent.com/u/38434522?"},"id":"14690746749","repo":{"url":"https://api.github.com/repos/GO-LiFE/GoFIT_SDK_Android","id":141905736,"name":"GO-LiFE/GoFIT_SDK_Android"},"actor":{"gravatar_id":"","display_login":"okbean","url":"https://api.github.com/users/okbean","id":75969386,"logi
[...]
+56
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:28Z","id":"14690746773","repo":{"url":"https://api.github.com/repos/MrXujiang/h5-Dooring","id":289417971,"name":"MrXujiang/h5-Dooring"},"actor":{"gravatar_id":"","display_login":"PWDream","url":"https://api.github.com/users/PWDream","id":4903755,"login":"PWDream","avatar_url":"https://avatars.githubusercontent.com/u/4903755?"},"public":1,"type":"WatchEvent"}
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:28 [...]
+86
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:29Z","id":"14690746843","repo":{"url":"https://api.github.com/repos/redsaph/cleartext","id":106453399,"name":"redsaph/cleartext"},"actor":{"gravatar_id":"","display_login":"Gui-Yom","url":"https://api.github.com/users/Gui-Yom","id":25181283,"login":"Gui-Yom","avatar_url":"https://avatars.githubusercontent.com/u/25181283?"},"public":1,"type":"WatchEvent"}
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:29Z"," [...]
+98
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:29Z","org":{"gravatar_id":"","url":"https://api.github.com/orgs/sherlock-project","id":48293496,"login":"sherlock-project","avatar_url":"https://avatars.githubusercontent.com/u/48293496?"},"id":"14690746866","repo":{"url":"https://api.github.com/repos/sherlock-project/sherlock","id":162998479,"name":"sherlock-project/sherlock"},"actor":{"gravatar_id":"","display_login":"humaidk2","url":"https://api.github.com/users/humaidk
[...]
+101
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:29Z","id":"14690746870","repo":{"url":"https://api.github.com/repos/okandavut/react-spotify-nowplaying","id":326215605,"name":"okandavut/react-spotify-nowplaying"},"actor":{"gravatar_id":"","display_login":"hasantezcan","url":"https://api.github.com/users/hasantezcan","id":32804505,"login":"hasantezcan","avatar_url":"https://avatars.githubusercontent.com/u/32804505?"},"public":1,"type":"WatchEvent"}
{"payload":{"action":" [...]
+112
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:30Z","id":"14690746899","repo":{"url":"https://api.github.com/repos/sentriz/gonic","id":178435468,"name":"sentriz/gonic"},"actor":{"gravatar_id":"","display_login":"nicholas-robertson","url":"https://api.github.com/users/nicholas-robertson","id":17681331,"login":"nicholas-robertson","avatar_url":"https://avatars.githubusercontent.com/u/17681331?"},"public":1,"type":"WatchEvent"}
{"payload":{"action":"started"},"created_at [...]
+122
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:30Z","org":{"gravatar_id":"","url":"https://api.github.com/orgs/netlify-labs","id":47546088,"login":"netlify-labs","avatar_url":"https://avatars.githubusercontent.com/u/47546088?"},"id":"14690746914","repo":{"url":"https://api.github.com/repos/netlify-labs/react-netlify-identity-widget","id":182606378,"name":"netlify-labs/react-netlify-identity-widget"},"actor":{"gravatar_id":"","display_login":"javaniecampbell","url":"ht
[...]
+169
{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:32Z","org":{"gravatar_id":"","url":"https://api.github.com/orgs/microsoft","id":6154722,"login":"microsoft","avatar_url":"https://avatars.githubusercontent.com/u/6154722?"},"id":"14690747028","repo":{"url":"https://api.github.com/repos/microsoft/BotBuilder-Samples","id":68730444,"name":"microsoft/BotBuilder-Samples"},"actor":{"gravatar_id":"","display_login":"Yxnt","url":"https://api.github.com/users/Yxnt","id":10323352,";
[...]
+
+-- !gh_data_14 --
+{"payload":{"ref_type":"branch","pusher_type":"user","ref":"dependabot/npm_and_yarn/lodash-4.17.19"},"created_at":"2021-01-02T16:37:26Z","id":"14690746680","repo":{"url":"https://api.github.com/repos/megan777/calculator","id":212350500,"name":"megan777/calculator"},"actor":{"gravatar_id":"","display_login":"dependabot","url":"https://api.github.com/users/dependabot[bot]","id":49699333,"login":"dependabot[bot]","avatar_url":"https://avatars.githubusercontent.com/u/49699333?"},"type":"Dele
[...]
+{"payload":{"ref_type":"branch","pusher_type":"user","ref":"dependabot/npm_and_yarn/acorn-7.4.0"},"created_at":"2021-01-02T16:37:26Z","id":"14690746695","repo":{"url":"https://api.github.com/repos/megan777/calculator","id":212350500,"name":"megan777/calculator"},"actor":{"gravatar_id":"","display_login":"dependabot","url":"https://api.github.com/users/dependabot[bot]","id":49699333,"login":"dependabot[bot]","avatar_url":"https://avatars.githubusercontent.com/u/49699333?"},"type":"DeleteE
[...]
+{"payload":{"ref_type":"repository","pusher_type":"user","master_branch":"master","description":"Pratique
Bootstrap
Openclassroom"},"created_at":"2021-01-02T16:37:27Z","id":"14690746705","repo":{"url":"https://api.github.com/repos/Sonfresh/pratique-bootstrap","id":326225102,"name":"Sonfresh/pratique-bootstrap"},"actor":{"gravatar_id":"","display_login":"Sonfresh","url":"https://api.github.com/users/Sonfresh","id":40009609,"login":"Sonfresh","avatar_url":"https://avatars.githubusercontent
[...]
+{"payload":{"ref_type":"branch","pusher_type":"user","ref":"dependabot/npm_and_yarn/websocket-extensions-0.1.4"},"created_at":"2021-01-02T16:37:27Z","id":"14690746708","repo":{"url":"https://api.github.com/repos/megan777/calculator","id":212350500,"name":"megan777/calculator"},"actor":{"gravatar_id":"","display_login":"dependabot","url":"https://api.github.com/users/dependabot[bot]","id":49699333,"login":"dependabot[bot]","avatar_url":"https://avatars.githubusercontent.com/u/49699333?"},
[...]
+{"payload":{"ref_type":"tag","pusher_type":"user","master_branch":"master","description":"A
library for Frege
indexers","ref":"v_0.1"},"created_at":"2021-01-02T16:37:27Z","org":{"gravatar_id":"","url":"https://api.github.com/orgs/Software-Engineering-Jagiellonian","id":74735360,"login":"Software-Engineering-Jagiellonian","avatar_url":"https://avatars.githubusercontent.com/u/74735360?"},"id":"14690746727","repo":{"url":"https://api.github.com/repos/Software-Engineering-Jagiellonian/frege-
[...]
+{"payload":{"ref_type":"repository","pusher_type":"user","master_branch":"master"},"created_at":"2021-01-02T16:37:27Z","id":"14690746737","repo":{"url":"https://api.github.com/repos/TOtto91/orbital-debris","id":326225103,"name":"TOtto91/orbital-debris"},"actor":{"gravatar_id":"","display_login":"TOtto91","url":"https://api.github.com/users/TOtto91","id":54342679,"login":"TOtto91","avatar_url":"https://avatars.githubusercontent.com/u/54342679?"},"type":"CreateEvent","public":1}
+{"payload":{"ref_type":"tag","pusher_type":"user","ref":"flux-write-check-1ccffefc6f"},"created_at":"2021-01-02T16:37:27Z","id":"14690746750","repo":{"url":"https://api.github.com/repos/jimangel/auto-gitops","id":290091833,"name":"jimangel/auto-gitops"},"actor":{"gravatar_id":"","display_login":"jimangel","url":"https://api.github.com/users/jimangel","id":4601051,"login":"jimangel","avatar_url":"https://avatars.githubusercontent.com/u/4601051?"},"type":"DeleteEvent","public":1}
+{"payload":{"ref_type":"branch","pusher_type":"user","master_branch":"master","description":"Open
Source is hard. Show your
gratitude","ref":"emoji-conversion"},"created_at":"2021-01-02T16:37:27Z","id":"14690746754","repo":{"url":"https://api.github.com/repos/yeraydiazdiaz/shoutouts.dev","id":323309255,"name":"yeraydiazdiaz/shoutouts.dev"},"actor":{"gravatar_id":"","display_login":"yeraydiazdiaz","url":"https://api.github.com/users/yeraydiazdiaz","id":6739793,"login":"yeraydiazdiaz","ava
[...]
+{"payload":{"ref_type":"repository","pusher_type":"user","master_branch":"master","description":"BIPN145
Winter2020"},"created_at":"2021-01-02T16:37:27Z","id":"14690746759","repo":{"url":"https://api.github.com/repos/mwchu/BIPN145","id":326225104,"name":"mwchu/BIPN145"},"actor":{"gravatar_id":"","display_login":"mwchu","url":"https://api.github.com/users/mwchu","id":76878377,"login":"mwchu","avatar_url":"https://avatars.githubusercontent.com/u/76878377?"},"type":"CreateEvent","public":1}
+{"payload":{"ref_type":"branch","pusher_type":"user","master_branch":"master","description":"Simple
Python 3 CLI script for retrieving Wrocław MPK trams and buses real-time
location","ref":"fill-readme"},"created_at":"2021-01-02T16:37:28Z","id":"14690746775","repo":{"url":"https://api.github.com/repos/emkor/mpyk","id":186781673,"name":"emkor/mpyk"},"actor":{"gravatar_id":"","display_login":"emkor","url":"https://api.github.com/users/emkor","id":4871763,"login":"emkor","avatar_url":"https
[...]
+
diff --git a/regression-test/suites/variant_p0/column_size_limit.groovy
b/regression-test/suites/variant_p0/column_size_limit.groovy
index 8c8c9051585..70567d89c07 100644
--- a/regression-test/suites/variant_p0/column_size_limit.groovy
+++ b/regression-test/suites/variant_p0/column_size_limit.groovy
@@ -14,8 +14,9 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+import groovy.json.JsonBuilder
-suite("regression_test_variant_column_limit"){
+suite("regression_test_variant_column_limit", "nonConcurrent"){
def set_be_config = { key, value ->
String backend_id;
def backendId_to_backendIP = [:]
@@ -27,7 +28,6 @@ suite("regression_test_variant_column_limit"){
logger.info("update config: code=" + code + ", out=" + out + ", err="
+ err)
}
def table_name = "var_column_limit"
- set_be_config.call("variant_max_merged_tablet_schema_size", "5")
sql "DROP TABLE IF EXISTS ${table_name}"
sql """
CREATE TABLE IF NOT EXISTS ${table_name} (
@@ -39,11 +39,21 @@ suite("regression_test_variant_column_limit"){
properties("replication_num" = "1", "disable_auto_compaction" =
"false");
"""
try {
- sql """insert into ${table_name} values (1, '{"a" : 1, "b" : 2, "c" :
3, "d" : 4, "e" : 5}')"""
+ def jsonBuilder = new JsonBuilder()
+ def root = jsonBuilder {
+ // Generate 2049 fields
+ (1..2049).each { fieldNumber ->
+ "field$fieldNumber" fieldNumber
+ }
+ }
+
+ String jsonString = jsonBuilder.toPrettyString()
+ sql """insert into ${table_name} values (1, '$jsonString')"""
} catch(Exception ex) {
logger.info("""INSERT INTO ${table_name} failed: """ + ex)
+ assertTrue(ex.toString().contains("Reached max column"));
+ } finally {
}
sql """insert into ${table_name} values (1, '{"a" : 1, "b" : 2, "c" :
3}')"""
- set_be_config.call("variant_max_merged_tablet_schema_size", "1024")
}
\ No newline at end of file
diff --git a/regression-test/suites/variant_p0/load.groovy
b/regression-test/suites/variant_p0/load.groovy
index 778da69de96..95d17cff833 100644
--- a/regression-test/suites/variant_p0/load.groovy
+++ b/regression-test/suites/variant_p0/load.groovy
@@ -287,7 +287,7 @@ suite("regression_test_variant", "variant_type"){
// 12. streamload remote file
table_name = "logdata"
create_table.call(table_name, "4")
- sql "set enable_two_phase_read_opt = false;"
+ // sql "set enable_two_phase_read_opt = false;"
// no sparse columns
set_be_config.call("ratio_of_defaults_as_sparse_column", "1")
load_json_data.call(table_name, """${getS3Url() +
'/load/logdata.json'}""")
@@ -389,13 +389,13 @@ suite("regression_test_variant", "variant_type"){
sql """insert into ${table_name} values (2, "abe", '{"c" : 1}')"""
sql """insert into ${table_name} values (3, "abd", '{"d" : 1}')"""
sql "delete from ${table_name} where k in (select k from variant_mow
where k in (1, 2))"
- qt_sql_38 "select * from ${table_name} order by k"
+ qt_sql_38 "select * from ${table_name} order by k limit 10"
// read text from sparse col
set_be_config.call("ratio_of_defaults_as_sparse_column", "0")
sql """insert into sparse_columns select 0, '{"a": 1123, "b" : [123,
{"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}, "zzz" : null, "oooo" :
{"akakaka" : null, "xxxx" : {"xxx" : 123}}}' as json_str
union all select 0, '{"a" : 1234, "xxxx" : "kaana", "ddd" :
{"aaa" : 123, "mxmxm" : [456, "789"]}}' as json_str from numbers("number" =
"4096") limit 4096 ;"""
- qt_sql_31 """select cast(v:xxxx as string) from sparse_columns where
cast(v:xxxx as string) != 'null' limit 1;"""
+ qt_sql_31 """select cast(v:xxxx as string) from sparse_columns where
cast(v:xxxx as string) != 'null' order by k limit 1;"""
sql "truncate table sparse_columns"
set_be_config.call("ratio_of_defaults_as_sparse_column", "0.95")
diff --git a/regression-test/suites/variant_p0/sql/gh_data.sql
b/regression-test/suites/variant_p0/sql/gh_data.sql
index 7eae73a9faa..41369190ca8 100644
--- a/regression-test/suites/variant_p0/sql/gh_data.sql
+++ b/regression-test/suites/variant_p0/sql/gh_data.sql
@@ -1,4 +1,6 @@
set exec_mem_limit=8G;
+set enable_two_phase_read_opt = true;
+set global topn_opt_limit_threshold = 1024;
SELECT count() from ghdata;
SELECT cast(v:repo.name as string), count() AS stars FROM ghdata WHERE
cast(v:type as string) = 'WatchEvent' GROUP BY cast(v:repo.name as string)
ORDER BY stars DESC, cast(v:repo.name as string) LIMIT 5;
SELECT max(cast(cast(v:`id` as string) as bigint)) FROM ghdata;
@@ -6,8 +8,9 @@ SELECT sum(cast(cast(v:`id` as string) as bigint)) FROM ghdata;
SELECT sum(cast(v:payload.member.id as bigint)) FROM ghdata;
SELECT sum(cast(v:payload.pull_request.milestone.creator.site_admin as
bigint)) FROM ghdata;
SELECT sum(length(v:payload.pull_request.base.repo.html_url)) FROM ghdata;
-SELECT v:payload.commits.author.name FROM ghdata ORDER BY k LIMIT 10;
-SELECT v:payload.member.id FROM ghdata where cast(v:payload.member.id as
string) is not null ORDER BY k LIMIT 10;
+-- SELECT v:payload.member.id FROM ghdata where cast(v:payload.member.id as
string) is not null ORDER BY k LIMIT 10;
-- select k, v:payload.commits.author.name AS name, e FROM ghdata as t lateral
view explode(cast(v:payload.commits.author.name as array<string>)) tm1 as e
order by k limit 5;
-select k, v from ghdata WHERE cast(v:type as string) = 'WatchEvent' order by
k limit 10;
-SELECT cast(v:payload.member.id as bigint), count() FROM ghdata where
cast(v:payload.member.id as bigint) is not null group by
cast(v:payload.member.id as bigint) order by 1, 2 desc LIMIT 10;
\ No newline at end of file
+select k, json_extract(v, '$.repo') from ghdata WHERE cast(v:type as string) =
'WatchEvent' order by k limit 10;
+SELECT cast(v:payload.member.id as bigint), count() FROM ghdata where
cast(v:payload.member.id as bigint) is not null group by
cast(v:payload.member.id as bigint) order by 1, 2 desc LIMIT 10;
+select k, cast(v:`id` as string), cast(v:type as string), cast(v:repo.name as
string) from ghdata WHERE cast(v:type as string) = 'WatchEvent' order by k
limit 10;
+SELECT cast(v:payload.pusher_type as text) FROM ghdata where
cast(v:payload.pusher_type as text) is not null ORDER BY k LIMIT 10;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
