This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 5277a55791a (pick 34003) release fd for shutdown tablets (#34224)
5277a55791a is described below
commit 5277a55791af835d74e1d0f8bfda0deed9fd2464
Author: Yongqiang YANG <[email protected]>
AuthorDate: Mon Apr 29 10:51:19 2024 +0800
(pick 34003) release fd for shutdown tablets (#34224)
---
be/src/olap/merger.cpp | 4 ++++
be/src/olap/rowset/beta_rowset.cpp | 17 +++++++++++++++
be/src/olap/rowset/beta_rowset.h | 2 ++
be/src/olap/rowset/rowset.cpp | 6 ++++++
be/src/olap/rowset/rowset.h | 3 +++
be/src/olap/storage_engine.cpp | 1 +
be/src/olap/tablet.cpp | 13 ++++++++++++
be/src/olap/tablet.h | 4 ++++
be/src/olap/tablet_manager.cpp | 42 +++++++-------------------------------
9 files changed, 57 insertions(+), 35 deletions(-)
diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp
index 761535abecf..b73c5bda645 100644
--- a/be/src/olap/merger.cpp
+++ b/be/src/olap/merger.cpp
@@ -107,6 +107,10 @@ Status Merger::vmerge_rowsets(TabletSharedPtr tablet,
ReaderType reader_type,
size_t output_rows = 0;
bool eof = false;
while (!eof && !StorageEngine::instance()->stopped()) {
+ if (tablet->tablet_state() == TABLET_SHUTDOWN) {
+ return Status::Error<INTERNAL_ERROR>("tablet {} is not used any
more",
+ tablet->tablet_id());
+ }
// Read one block from block reader
RETURN_NOT_OK_STATUS_WITH_WARN(reader.next_block_with_aggregation(&block, &eof),
"failed to read next block when merging
rowsets of tablet " +
diff --git a/be/src/olap/rowset/beta_rowset.cpp
b/be/src/olap/rowset/beta_rowset.cpp
index 44bf843b792..ab41a092f07 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -26,6 +26,7 @@
#include <ostream>
#include <utility>
+#include "beta_rowset.h"
#include "common/config.h"
#include "common/logging.h"
#include "common/status.h"
@@ -124,6 +125,21 @@ Status BetaRowset::get_inverted_index_size(size_t*
index_size) {
return Status::OK();
}
+void BetaRowset::clear_inverted_index_cache() {
+ for (int i = 0; i < num_segments(); ++i) {
+ auto seg_path = segment_file_path(i);
+ for (auto& column : tablet_schema()->columns()) {
+ const TabletIndex* index_meta =
tablet_schema()->get_inverted_index(*column);
+ if (index_meta) {
+ std::string inverted_index_file =
InvertedIndexDescriptor::get_index_file_name(
+ seg_path, index_meta->index_id(),
index_meta->get_index_suffix());
+
(void)segment_v2::InvertedIndexSearcherCache::instance()->erase(
+ inverted_index_file);
+ }
+ }
+ }
+}
+
Status BetaRowset::get_segments_size(std::vector<size_t>* segments_size) {
auto fs = _rowset_meta->fs();
if (!fs || _schema == nullptr) {
@@ -614,4 +630,5 @@ Status BetaRowset::add_to_binlog() {
return Status::OK();
}
+
} // namespace doris
diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h
index 46f16f4d502..404a45a1be1 100644
--- a/be/src/olap/rowset/beta_rowset.h
+++ b/be/src/olap/rowset/beta_rowset.h
@@ -94,7 +94,9 @@ public:
Status load_segment(int64_t seg_id, segment_v2::SegmentSharedPtr* segment);
Status get_segments_size(std::vector<size_t>* segments_size);
+
Status get_inverted_index_size(size_t* index_size);
+ void clear_inverted_index_cache() override;
[[nodiscard]] virtual Status add_to_binlog() override;
diff --git a/be/src/olap/rowset/rowset.cpp b/be/src/olap/rowset/rowset.cpp
index b3e6b1ca9e0..c9fa1c6a3b9 100644
--- a/be/src/olap/rowset/rowset.cpp
+++ b/be/src/olap/rowset/rowset.cpp
@@ -20,6 +20,7 @@
#include <gen_cpp/olap_file.pb.h>
#include "olap/olap_define.h"
+#include "olap/segment_loader.h"
#include "olap/tablet_schema.h"
#include "util/time.h"
@@ -92,4 +93,9 @@ void Rowset::merge_rowset_meta(const RowsetMetaSharedPtr&
other) {
}
}
+void Rowset::clear_cache() {
+ SegmentLoader::instance()->erase_segments(rowset_id(), num_segments());
+ clear_inverted_index_cache();
+}
+
} // namespace doris
diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h
index a2a275f2eac..e7675c1916a 100644
--- a/be/src/olap/rowset/rowset.h
+++ b/be/src/olap/rowset/rowset.h
@@ -302,6 +302,9 @@ public:
// set skip index compaction next time
void set_skip_index_compaction(int32_t column_id) {
skip_index_compaction.insert(column_id); }
+ virtual void clear_inverted_index_cache() { LOG(INFO) << "should not reach
here"; }
+ void clear_cache();
+
protected:
friend class RowsetFactory;
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 5b8d4945ad1..1954e069059 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -1094,6 +1094,7 @@ void StorageEngine::start_delete_unused_rowset() {
if (rs->is_local()) {
unused_rowsets_copy.push_back(std::move(rs));
}
+ it->second->clear_cache();
it = _unused_rowsets.erase(it);
} else {
if (rs.use_count() != 1) {
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index b6d347e27e9..6fba2a43b09 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -108,6 +108,7 @@
#include "olap/utils.h"
#include "segment_loader.h"
#include "service/point_query_executor.h"
+#include "tablet.h"
#include "util/bvar_helper.h"
#include "util/debug_points.h"
#include "util/defer_op.h"
@@ -821,6 +822,7 @@ void Tablet::delete_expired_stale_rowset() {
auto it =
_stale_rs_version_map.find(timestampedVersion->version());
if (it != _stale_rs_version_map.end()) {
// delete rowset
+ it->second->clear_cache();
_engine.add_unused_rowset(it->second);
_stale_rs_version_map.erase(it);
VLOG_NOTICE << "delete stale rowset tablet=" <<
tablet_id() << " version["
@@ -3703,6 +3705,17 @@ Status Tablet::ingest_binlog_metas(RowsetBinlogMetasPB*
metas_pb) {
return RowsetMetaManager::ingest_binlog_metas(_data_dir->get_meta(),
tablet_uid(), metas_pb);
}
+void Tablet::clear_cache() {
+ std::shared_lock rlock(get_header_lock());
+ static auto recycle_segment_cache = [](const auto& rowset_map) {
+ for (auto& [_, rowset] : rowset_map) {
+ rowset->clear_cache();
+ }
+ };
+ recycle_segment_cache(rowset_map());
+ recycle_segment_cache(stale_rowset_map());
+}
+
Status Tablet::calc_delete_bitmap_between_segments(
RowsetSharedPtr rowset, const
std::vector<segment_v2::SegmentSharedPtr>& segments,
DeleteBitmapPtr delete_bitmap) {
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 0f4ac7d4d5e..699fa96137a 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -582,6 +582,8 @@ public:
void set_alter_failed(bool alter_failed) { _alter_failed = alter_failed; }
bool is_alter_failed() { return _alter_failed; }
+ void clear_cache();
+
private:
Status _init_once_action();
void _print_missed_versions(const std::vector<Version>& missed_versions)
const;
@@ -630,6 +632,8 @@ private:
void _remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr
delete_bitmap);
std::string _get_rowset_info_str(RowsetSharedPtr rowset, bool delete_flag);
+ void _clear_cache_by_rowset(const BetaRowsetSharedPtr& rowset);
+
public:
static const int64_t K_INVALID_CUMULATIVE_POINT = -1;
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index ea0c5bffcf3..cf3c94f2a3d 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -551,41 +551,9 @@ Status TabletManager::_drop_tablet_unlocked(TTabletId
tablet_id, TReplicaId repl
_remove_tablet_from_partition(to_drop_tablet);
tablet_map_t& tablet_map = _get_tablet_map(tablet_id);
tablet_map.erase(tablet_id);
- {
- std::shared_lock rlock(to_drop_tablet->get_header_lock());
- static auto recycle_segment_cache = [](const auto& rowset_map) {
- for (auto& [_, rowset] : rowset_map) {
- // If the tablet was deleted, it need to remove all rowsets
fds directly
- SegmentLoader::instance()->erase_segments(rowset->rowset_id(),
-
rowset->num_segments());
- }
- };
- recycle_segment_cache(to_drop_tablet->rowset_map());
- recycle_segment_cache(to_drop_tablet->stale_rowset_map());
- // recycle inverted index cache
- static auto recycle_inverted_index_cache = [](const auto& rowset_map) {
- for (auto& [_, rowset] : rowset_map) {
- auto rs = std::static_pointer_cast<BetaRowset>(rowset);
- for (int i = 0; i < rs->num_segments(); ++i) {
- auto seg_path = rs->segment_file_path(i);
- for (auto& column : rs->tablet_schema()->columns()) {
- const TabletIndex* index_meta =
-
rs->tablet_schema()->get_inverted_index(*column);
- if (index_meta) {
- std::string inverted_index_file =
-
InvertedIndexDescriptor::get_index_file_name(
- seg_path, index_meta->index_id(),
- index_meta->get_index_suffix());
-
(void)segment_v2::InvertedIndexSearcherCache::instance()->erase(
- inverted_index_file);
- }
- }
- }
- }
- };
- recycle_inverted_index_cache(to_drop_tablet->rowset_map());
- recycle_inverted_index_cache(to_drop_tablet->stale_rowset_map());
- }
+
+ to_drop_tablet->clear_cache();
+
if (!keep_files) {
// drop tablet will update tablet meta, should lock
std::lock_guard<std::shared_mutex>
wrlock(to_drop_tablet->get_header_lock());
@@ -1155,6 +1123,9 @@ bool TabletManager::_move_tablet_to_trash(const
TabletSharedPtr& tablet) {
<< " cur tablet_uid=" << tablet_meta->tablet_uid();
return true;
}
+
+ tablet->clear_cache();
+
// move data to trash
const auto& tablet_path = tablet->tablet_path();
bool exists = false;
@@ -1196,6 +1167,7 @@ bool TabletManager::_move_tablet_to_trash(const
TabletSharedPtr& tablet) {
<< ", schema_hash=" << tablet->schema_hash() << ",
tablet_path=" << tablet_path;
return true;
} else {
+ tablet->clear_cache();
// if could not find tablet info in meta store, then check if dir
existed
const auto& tablet_path = tablet->tablet_path();
bool exists = false;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
