This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 86ba0ebf42 [fix](mow) revert 17147 and 18750 (#19583)
86ba0ebf42 is described below
commit 86ba0ebf423c6203b6cf561cd52e5c079074148c
Author: zhannngchen <[email protected]>
AuthorDate: Sat May 13 08:43:36 2023 +0800
[fix](mow) revert 17147 and 18750 (#19583)
---
be/src/olap/delta_writer.cpp | 3 +--
be/src/olap/rowset/beta_rowset_writer.cpp | 9 ---------
be/src/olap/rowset/beta_rowset_writer.h | 8 +-------
be/src/olap/rowset/segment_v2/segment_writer.cpp | 7 -------
be/src/olap/rowset/segment_v2/segment_writer.h | 6 ------
be/src/olap/tablet.cpp | 18 ------------------
be/src/olap/tablet_meta.cpp | 8 --------
be/src/olap/tablet_meta.h | 2 --
be/src/olap/txn_manager.cpp | 12 ++++++------
be/src/olap/txn_manager.h | 3 +--
10 files changed, 9 insertions(+), 67 deletions(-)
diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp
index 4fb5100078..cd72ade8aa 100644
--- a/be/src/olap/delta_writer.cpp
+++ b/be/src/olap/delta_writer.cpp
@@ -404,8 +404,7 @@ Status DeltaWriter::close_wait(const PSlaveTabletNodes&
slave_tablet_nodes,
_cur_max_version, true));
_storage_engine->txn_manager()->set_txn_related_delete_bitmap(
_req.partition_id, _req.txn_id, _tablet->tablet_id(),
_tablet->schema_hash(),
- _tablet->tablet_uid(), true, _delete_bitmap, _rowset_ids,
-
dynamic_cast<BetaRowsetWriter*>(_rowset_writer.get())->get_num_mow_keys());
+ _tablet->tablet_uid(), true, _delete_bitmap, _rowset_ids);
}
_delta_written_success = true;
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp
b/be/src/olap/rowset/beta_rowset_writer.cpp
index 425f882d76..91244b78a5 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -614,7 +614,6 @@ void
BetaRowsetWriter::_build_rowset_meta(std::shared_ptr<RowsetMeta> rowset_met
int64_t total_data_size = 0;
int64_t total_index_size = 0;
std::vector<KeyBoundsPB> segments_encoded_key_bounds;
- std::unordered_set<std::string> key_set;
{
std::lock_guard<std::mutex> lock(_segid_statistics_map_mutex);
for (const auto& itr : _segid_statistics_map) {
@@ -622,15 +621,8 @@ void
BetaRowsetWriter::_build_rowset_meta(std::shared_ptr<RowsetMeta> rowset_met
total_data_size += itr.second.data_size;
total_index_size += itr.second.index_size;
segments_encoded_key_bounds.push_back(itr.second.key_bounds);
-#ifndef NDEBUG
- if (_context.enable_unique_key_merge_on_write) {
- DCHECK(itr.second.key_set.get() != nullptr);
- key_set.insert(itr.second.key_set->begin(),
itr.second.key_set->end());
- }
-#endif
}
}
- _num_mow_keys = key_set.size();
for (auto itr = _segments_encoded_key_bounds.begin(); itr !=
_segments_encoded_key_bounds.end();
++itr) {
segments_encoded_key_bounds.push_back(*itr);
@@ -778,7 +770,6 @@ Status
BetaRowsetWriter::_flush_segment_writer(std::unique_ptr<segment_v2::Segme
segstat.data_size = segment_size;
segstat.index_size = index_size;
segstat.key_bounds = key_bounds;
- segstat.key_set = (*writer)->get_key_set();
{
std::lock_guard<std::mutex> lock(_segid_statistics_map_mutex);
CHECK_EQ(_segid_statistics_map.find(segid) ==
_segid_statistics_map.end(), true);
diff --git a/be/src/olap/rowset/beta_rowset_writer.h
b/be/src/olap/rowset/beta_rowset_writer.h
index 011e53e300..b7d5b10ee6 100644
--- a/be/src/olap/rowset/beta_rowset_writer.h
+++ b/be/src/olap/rowset/beta_rowset_writer.h
@@ -110,8 +110,6 @@ public:
return _context.schema_change_recorder.get();
}
- uint64_t get_num_mow_keys() { return _num_mow_keys; }
-
SegcompactionWorker& get_segcompaction_worker() { return
_segcompaction_worker; }
Status flush_segment_writer_for_segcompaction(
@@ -191,13 +189,9 @@ protected:
int64_t data_size;
int64_t index_size;
KeyBoundsPB key_bounds;
- std::shared_ptr<std::unordered_set<std::string>> key_set;
};
- std::mutex _segid_statistics_map_mutex;
std::map<uint32_t, Statistics> _segid_statistics_map;
-
- // used for check correctness of unique key mow keys.
- std::atomic<uint64_t> _num_mow_keys;
+ std::mutex _segid_statistics_map_mutex;
bool _is_pending = false;
bool _already_built = false;
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp
b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 466fe5ef53..bb6f47d759 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -253,9 +253,6 @@ Status SegmentWriter::init(const std::vector<uint32_t>&
col_ids, bool has_key,
_primary_key_index_builder.reset(
new PrimaryKeyIndexBuilder(_file_writer, seq_col_length));
RETURN_IF_ERROR(_primary_key_index_builder->init());
-#ifndef NDEBUG
- _key_set.reset(new std::unordered_set<std::string>());
-#endif
} else {
_short_key_index_builder.reset(
new ShortKeyIndexBuilder(_segment_id,
_opts.num_rows_per_block));
@@ -582,10 +579,6 @@ Status SegmentWriter::append_block(const
vectorized::Block* block, size_t row_po
// create primary indexes
for (size_t pos = 0; pos < num_rows; pos++) {
std::string key = _full_encode_keys(key_columns, pos);
-#ifndef NDEBUG
- DCHECK(_key_set.get() != nullptr);
- _key_set->insert(key);
-#endif
if (_tablet_schema->has_sequence_col()) {
_encode_seq_column(seq_column, pos, &key);
}
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.h
b/be/src/olap/rowset/segment_v2/segment_writer.h
index e66ae33d43..a5b5178b43 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.h
+++ b/be/src/olap/rowset/segment_v2/segment_writer.h
@@ -67,8 +67,6 @@ namespace segment_v2 {
extern const char* k_segment_magic;
extern const uint32_t k_segment_magic_length;
-using KeySetPtr = std::shared_ptr<std::unordered_set<std::string>>;
-
struct SegmentWriterOptions {
uint32_t num_rows_per_block = 1024;
bool enable_unique_key_merge_on_write = false;
@@ -124,8 +122,6 @@ public:
Slice min_encoded_key();
Slice max_encoded_key();
- KeySetPtr get_key_set() { return _key_set; }
-
DataDir* get_data_dir() { return _data_dir; }
bool is_unique_key() { return _tablet_schema->keys_type() == UNIQUE_KEYS; }
@@ -192,8 +188,6 @@ private:
const KeyCoder* _seq_coder = nullptr;
std::vector<uint16_t> _key_index_size;
size_t _short_key_row_pos = 0;
- // used to check if there's duplicate key in aggregate key and unique key
data model
- KeySetPtr _key_set;
std::vector<uint32_t> _column_ids;
bool _has_key = true;
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 9b4e94bae2..47fe3a1bb8 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -3001,24 +3001,6 @@ Status Tablet::update_delete_bitmap(const
RowsetSharedPtr& rowset, const TabletT
RETURN_IF_ERROR(calc_delete_bitmap(rowset, segments, &rowset_ids_to_add,
delete_bitmap,
cur_version - 1, false, rowset_writer));
- // Check the delete_bitmap correctness, now the check is only enabled in
DEBUG env.
- if (load_info->num_keys != 0) {
- DeleteBitmap rs_bm(tablet_id());
- delete_bitmap->subset({rowset->rowset_id(), 0, 0},
- {rowset->rowset_id(), UINT32_MAX, INT64_MAX},
&rs_bm);
- auto num_rows = rowset->num_rows();
- auto bitmap_cardinality = rs_bm.cardinality();
- std::string err_msg = fmt::format(
- "The delete bitmap of unique key table may not correct, expect
num unique keys:"
- "{}, "
- "now the num_rows: {}, delete bitmap cardinality: {}, num
sgements: {}",
- load_info->num_keys, num_rows, bitmap_cardinality,
rowset->num_segments());
- DCHECK_EQ(load_info->num_keys, num_rows - bitmap_cardinality) <<
err_msg;
- if (load_info->num_keys != num_rows - bitmap_cardinality) {
- return Status::InternalError(err_msg);
- }
- }
-
// update version without write lock, compaction and publish_txn
// will update delete bitmap, handle compaction with _rowset_update_lock
// and publish_txn runs sequential so no need to lock here
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 74a48b07c8..cdcc78b6e5 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -960,14 +960,6 @@ void DeleteBitmap::merge(const DeleteBitmap& other) {
}
}
-uint64_t DeleteBitmap::cardinality() {
- uint64_t cardinality = 0;
- for (auto entry : delete_bitmap) {
- cardinality += entry.second.cardinality();
- }
- return cardinality;
-}
-
// We cannot just copy the underlying memory to construct a string
// due to equivalent objects may have different padding bytes.
// Reading padding bytes is undefined behavior, neither copy nor
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 3ac1b148dc..f5b03b363b 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -389,8 +389,6 @@ public:
*/
void merge(const DeleteBitmap& other);
- uint64_t cardinality();
-
/**
* Checks if the given row is marked deleted in bitmap with the condition:
* all the bitmaps that
diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp
index ab6af8e293..b87b86076f 100644
--- a/be/src/olap/txn_manager.cpp
+++ b/be/src/olap/txn_manager.cpp
@@ -164,10 +164,12 @@ Status TxnManager::prepare_txn(TPartitionId partition_id,
TTransactionId transac
return Status::OK();
}
-void TxnManager::set_txn_related_delete_bitmap(
- TPartitionId partition_id, TTransactionId transaction_id, TTabletId
tablet_id,
- SchemaHash schema_hash, TabletUid tablet_uid, bool
unique_key_merge_on_write,
- DeleteBitmapPtr delete_bitmap, const RowsetIdUnorderedSet& rowset_ids,
uint64_t num_keys) {
+void TxnManager::set_txn_related_delete_bitmap(TPartitionId partition_id,
+ TTransactionId transaction_id,
TTabletId tablet_id,
+ SchemaHash schema_hash,
TabletUid tablet_uid,
+ bool unique_key_merge_on_write,
+ DeleteBitmapPtr delete_bitmap,
+ const RowsetIdUnorderedSet&
rowset_ids) {
pair<int64_t, int64_t> key(partition_id, transaction_id);
TabletInfo tablet_info(tablet_id, schema_hash, tablet_uid);
@@ -189,7 +191,6 @@ void TxnManager::set_txn_related_delete_bitmap(
load_info.unique_key_merge_on_write = unique_key_merge_on_write;
load_info.delete_bitmap = delete_bitmap;
load_info.rowset_ids = rowset_ids;
- load_info.num_keys = num_keys;
}
}
@@ -275,7 +276,6 @@ Status TxnManager::commit_txn(OlapMeta* meta, TPartitionId
partition_id,
if (tablet != nullptr &&
tablet->enable_unique_key_merge_on_write()) {
load_info.unique_key_merge_on_write = true;
load_info.delete_bitmap.reset(new
DeleteBitmap(tablet->tablet_id()));
- load_info.num_keys = 0;
}
}
txn_tablet_map_t& txn_tablet_map = _get_txn_tablet_map(transaction_id);
diff --git a/be/src/olap/txn_manager.h b/be/src/olap/txn_manager.h
index 40925bc61d..2a54c13179 100644
--- a/be/src/olap/txn_manager.h
+++ b/be/src/olap/txn_manager.h
@@ -57,7 +57,6 @@ struct TabletTxnInfo {
// records rowsets calc in commit txn
RowsetIdUnorderedSet rowset_ids;
int64_t creation_time;
- uint64_t num_keys;
TabletTxnInfo(PUniqueId load_id, RowsetSharedPtr rowset)
: load_id(load_id),
@@ -167,7 +166,7 @@ public:
TTabletId tablet_id, SchemaHash
schema_hash,
TabletUid tablet_uid, bool
unique_key_merge_on_write,
DeleteBitmapPtr delete_bitmap,
- const RowsetIdUnorderedSet& rowset_ids,
uint64_t num_keys);
+ const RowsetIdUnorderedSet& rowset_ids);
private:
using TxnKey = std::pair<int64_t, int64_t>; // partition_id,
transaction_id;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
