zbtzbtzbt commented on code in PR #9123:
URL: https://github.com/apache/incubator-doris/pull/9123#discussion_r855124864
##########
be/src/olap/rowset/segment_v2/binary_plain_page.h:
##########
@@ -127,11 +125,16 @@ class BinaryPlainPageBuilder : public PageBuilder {
return Status::OK();
}
- void update_prepared_size(size_t added_size) {
- _prepared_size += added_size;
- _prepared_size += sizeof(uint32_t);
+ inline Slice operator[](std::size_t idx) const {
Review Comment:
size_t
##########
be/src/olap/rowset/segment_v2/bitshuffle_page.h:
##########
@@ -93,13 +93,52 @@ class BitshufflePageBuilder : public PageBuilder {
bool is_page_full() override { return _remain_element_capacity == 0; }
Status add(const uint8_t* vals, size_t* count) override {
+ return add_internal<false>(vals, count);
+ }
+
+ Status single_add(const uint8_t* vals, size_t* count) {
Review Comment:
can we delete this function？
##########
be/src/olap/rowset/segment_v2/bitshuffle_page.h:
##########
@@ -93,13 +93,52 @@ class BitshufflePageBuilder : public PageBuilder {
bool is_page_full() override { return _remain_element_capacity == 0; }
Status add(const uint8_t* vals, size_t* count) override {
+ return add_internal<false>(vals, count);
Review Comment:
can we delete this function?
##########
be/src/olap/memtable.cpp:
##########
@@ -65,44 +75,46 @@ int MemTable::RowCursorComparator::operator()(const char*
left, const char* righ
return compare_row(lhs_row, rhs_row);
}
-void MemTable::insert(const Tuple* tuple) {
+// For non-DUP models, for the data rows passed from the upper layer, when
copying the data,
+// we first allocate from _buffer_mem_pool, and then check whether it already
exists in
+// _skiplist. If it exists, we aggregate the new row into the row in skiplist.
+// otherwise, we need to copy it into _table_mem_pool before we can insert it.
+void MemTable::_insert_agg(const Tuple* tuple) {
Review Comment:
we may change a name, such as `_insert_no_dup`? and put `_insert_dup`
before `_insert_no_dup`.
becasue this kind of `insert` agg and unique data.
##########
be/src/olap/rowset/segment_v2/bitshuffle_page.h:
##########
@@ -93,13 +93,52 @@ class BitshufflePageBuilder : public PageBuilder {
bool is_page_full() override { return _remain_element_capacity == 0; }
Status add(const uint8_t* vals, size_t* count) override {
+ return add_internal<false>(vals, count);
+ }
+
+ Status single_add(const uint8_t* vals, size_t* count) {
+ return add_internal<true>(vals, count);
+ }
+
+ template <bool single>
+ inline Status add_internal(const uint8_t* vals, size_t* count) {
DCHECK(!_finished);
+ if (_remain_element_capacity <= 0) {
+ *count = 0;
+ return Status::RuntimeError("page is full.");
+ }
int to_add = std::min<int>(_remain_element_capacity, *count);
- _data.append(vals, to_add * SIZE_OF_TYPE);
+ int to_add_size = to_add * SIZE_OF_TYPE;
+ size_t orig_size = _data.size();
+ _data.resize(orig_size + to_add_size);
_count += to_add;
_remain_element_capacity -= to_add;
// return added number through count
*count = to_add;
+ if constexpr (single) {
+ if constexpr (SIZE_OF_TYPE == 1) {
+ *reinterpret_cast<uint8_t*>(&_data[orig_size]) = *vals;
+ return Status::OK();
+ }
+ if constexpr (SIZE_OF_TYPE == 2) {
+ *reinterpret_cast<uint16_t*>(&_data[orig_size]) =
+ *reinterpret_cast<const uint16_t*>(vals);
+ return Status::OK();
+ }
+ if constexpr (SIZE_OF_TYPE == 4) {
+ *reinterpret_cast<uint32_t*>(&_data[orig_size]) =
+ *reinterpret_cast<const uint32_t*>(vals);
+ return Status::OK();
+ }
+ if constexpr (SIZE_OF_TYPE == 8) {
+ *reinterpret_cast<uint64_t*>(&_data[orig_size]) =
+ *reinterpret_cast<const uint64_t*>(vals);
+ return Status::OK();
+ }
+ memcpy(&_data[orig_size], vals, to_add_size);
Review Comment:
else
##########
be/src/olap/rowset/segment_v2/binary_dict_page.cpp:
##########
@@ -90,13 +92,18 @@ Status BinaryDictPageBuilder::add(const uint8_t* vals,
size_t* count) {
dict_item.relocate(item_mem);
}
value_code = _dictionary.size();
+ size_t add_count = 1;
+ RETURN_IF_ERROR(_dict_builder->add(reinterpret_cast<const
uint8_t*>(&dict_item),
+ &add_count));
+ if (add_count == 0) {
+ // current dict page is full, stop processing remaining
inputs
+ break;
+ }
_dictionary.emplace(dict_item, value_code);
- _dict_items.push_back(dict_item);
- _dict_builder->update_prepared_size(dict_item.size);
}
size_t add_count = 1;
- RETURN_IF_ERROR(_data_page_builder->add(reinterpret_cast<const
uint8_t*>(&value_code),
- &add_count));
+ RETURN_IF_ERROR(actual_builder->single_add(
Review Comment:
use `add_internal<true>`
##########
be/src/olap/rowset/segment_v2/bitshuffle_page.h:
##########
@@ -93,13 +93,52 @@ class BitshufflePageBuilder : public PageBuilder {
bool is_page_full() override { return _remain_element_capacity == 0; }
Status add(const uint8_t* vals, size_t* count) override {
+ return add_internal<false>(vals, count);
+ }
+
+ Status single_add(const uint8_t* vals, size_t* count) {
+ return add_internal<true>(vals, count);
+ }
+
+ template <bool single>
+ inline Status add_internal(const uint8_t* vals, size_t* count) {
DCHECK(!_finished);
+ if (_remain_element_capacity <= 0) {
+ *count = 0;
+ return Status::RuntimeError("page is full.");
+ }
int to_add = std::min<int>(_remain_element_capacity, *count);
- _data.append(vals, to_add * SIZE_OF_TYPE);
+ int to_add_size = to_add * SIZE_OF_TYPE;
+ size_t orig_size = _data.size();
+ _data.resize(orig_size + to_add_size);
_count += to_add;
_remain_element_capacity -= to_add;
// return added number through count
*count = to_add;
+ if constexpr (single) {
+ if constexpr (SIZE_OF_TYPE == 1) {
+ *reinterpret_cast<uint8_t*>(&_data[orig_size]) = *vals;
+ return Status::OK();
+ }
+ if constexpr (SIZE_OF_TYPE == 2) {
Review Comment:
else if constexpr
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
