This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-0.15 in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 0158442c8645dde8d4e79a6265b45ff1ad85750d Author: HappenLee <[email protected]> AuthorDate: Sat Oct 16 21:54:32 2021 +0800 [MemerySave] Change TabletSchema in tablet to reference to save mem (#6814) Change TabletSchema in tablet to reference to save memory --- be/src/olap/base_tablet.cpp | 2 -- be/src/olap/base_tablet.h | 2 +- be/src/olap/tablet.cpp | 6 +----- be/src/olap/tablet_meta.cpp | 31 +++++++++++++++++++++++++------ be/src/olap/tablet_meta.h | 11 ++++++++--- 5 files changed, 35 insertions(+), 17 deletions(-) diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp index 35b9d7e..0f634ba 100644 --- a/be/src/olap/base_tablet.cpp +++ b/be/src/olap/base_tablet.cpp @@ -31,8 +31,6 @@ extern MetricPrototype METRIC_query_scan_count; BaseTablet::BaseTablet(TabletMetaSharedPtr tablet_meta, DataDir* data_dir) : _state(tablet_meta->tablet_state()), _tablet_meta(tablet_meta), - // TODO: Think we really copy tablet schema here, which cost double mem - // cost in TabletManager _schema(tablet_meta->tablet_schema()), _data_dir(data_dir) { _gen_tablet_path(); diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h index 9d0e331..3e1386b 100644 --- a/be/src/olap/base_tablet.h +++ b/be/src/olap/base_tablet.h @@ -70,7 +70,7 @@ protected: protected: TabletState _state; TabletMetaSharedPtr _tablet_meta; - TabletSchema _schema; + const TabletSchema& _schema; DataDir* _data_dir; std::string _tablet_path; diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 3d22614..b6dc448 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -136,9 +136,6 @@ void Tablet::save_meta() { auto res = _tablet_meta->save_meta(_data_dir); CHECK_EQ(res, OLAP_SUCCESS) << "fail to save tablet_meta. res=" << res << ", root=" << _data_dir->path(); - // User could directly update tablet schema by _tablet_meta, - // So we need to refetch schema again - _schema = _tablet_meta->tablet_schema(); } OLAPStatus Tablet::revise_tablet_meta(const std::vector<RowsetMetaSharedPtr>& rowsets_to_clone, @@ -149,8 +146,7 @@ OLAPStatus Tablet::revise_tablet_meta(const std::vector<RowsetMetaSharedPtr>& ro OLAPStatus res = OLAP_SUCCESS; do { // load new local tablet_meta to operate on - TabletMetaSharedPtr new_tablet_meta(new (nothrow) TabletMeta()); - generate_tablet_meta_copy_unlocked(new_tablet_meta); + TabletMetaSharedPtr new_tablet_meta(new (nothrow) TabletMeta(*_tablet_meta)); // delete versions from new local tablet_meta for (const Version& version : versions_to_delete) { diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 824cdaa..564ced8 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -46,14 +46,14 @@ OLAPStatus TabletMeta::create(const TCreateTabletReq& request, const TabletUid& return OLAP_SUCCESS; } -TabletMeta::TabletMeta() : _tablet_uid(0, 0) {} +TabletMeta::TabletMeta() : _tablet_uid(0, 0), _schema(new TabletSchema) {} TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id, int32_t schema_hash, uint64_t shard_id, const TTabletSchema& tablet_schema, uint32_t next_unique_id, const std::unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id, TabletUid tablet_uid, TTabletType::type tabletType) - : _tablet_uid(0, 0), _preferred_rowset_type(ALPHA_ROWSET) { + : _tablet_uid(0, 0), _preferred_rowset_type(BETA_ROWSET), _schema(new TabletSchema) { TabletMetaPB tablet_meta_pb; tablet_meta_pb.set_table_id(table_id); tablet_meta_pb.set_partition_id(partition_id); @@ -139,6 +139,25 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id init_from_pb(tablet_meta_pb); } +TabletMeta::TabletMeta(const TabletMeta& b) + : _table_id(b._table_id), + _partition_id(b._partition_id), + _tablet_id(b._tablet_id), + _schema_hash(b._schema_hash), + _shard_id(b._shard_id), + _creation_time(b._creation_time), + _cumulative_layer_point(b._cumulative_layer_point), + _tablet_uid(b._tablet_uid), + _tablet_type(b._tablet_type), + _tablet_state(b._tablet_state), + _schema(b._schema), + _in_restore_mode(b._in_restore_mode), + _rs_metas(b._rs_metas), + _stale_rs_metas(b._stale_rs_metas), + _del_pred_array(b._del_pred_array), + _preferred_rowset_type(b._preferred_rowset_type) { +} + void TabletMeta::_init_column_from_tcolumn(uint32_t unique_id, const TColumn& tcolumn, ColumnPB* column) { column->set_unique_id(unique_id); @@ -350,7 +369,7 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) { } // init _schema - _schema.init_from_pb(tablet_meta_pb.schema()); + _schema->init_from_pb(tablet_meta_pb.schema()); // init _rs_metas for (auto& it : tablet_meta_pb.rs_metas()) { @@ -411,7 +430,7 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) { for (auto rs : _stale_rs_metas) { rs->to_rowset_pb(tablet_meta_pb->add_stale_rs_metas()); } - _schema.to_schema_pb(tablet_meta_pb->mutable_schema()); + _schema->to_schema_pb(tablet_meta_pb->mutable_schema()); tablet_meta_pb->set_in_restore_mode(in_restore_mode()); @@ -423,7 +442,7 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) { uint32_t TabletMeta::mem_size() const { auto size = sizeof(TabletMeta); - size += _schema.mem_size(); + size += _schema->mem_size(); return size; } @@ -625,7 +644,7 @@ bool operator==(const TabletMeta& a, const TabletMeta& b) { if (a._tablet_uid != b._tablet_uid) return false; if (a._tablet_type != b._tablet_type) return false; if (a._tablet_state != b._tablet_state) return false; - if (a._schema != b._schema) return false; + if (*a._schema != *b._schema) return false; if (a._rs_metas.size() != b._rs_metas.size()) return false; for (int i = 0; i < a._rs_metas.size(); ++i) { if (a._rs_metas[i] != b._rs_metas[i]) return false; diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 9f3ba46..8995e96 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -82,6 +82,9 @@ public: uint64_t shard_id, const TTabletSchema& tablet_schema, uint32_t next_unique_id, const std::unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id, TabletUid tablet_uid, TTabletType::type tabletType); + // If need add a filed in TableMeta, filed init copy in copy construct function + TabletMeta(const TabletMeta& tablet_meta); + TabletMeta(TabletMeta&& tablet_meta) = delete; // Function create_from_file is used to be compatible with previous tablet_meta. // Previous tablet_meta is a physical file in tablet dir, which is not stored in rocksdb. @@ -184,7 +187,9 @@ private: TabletTypePB _tablet_type = TabletTypePB::TABLET_TYPE_DISK; TabletState _tablet_state = TABLET_NOTREADY; - TabletSchema _schema; + // the reference of _schema may use in tablet, so here need keep + // the lifetime of tablemeta and _schema is same with tablet + std::shared_ptr<TabletSchema> _schema; std::vector<RowsetMetaSharedPtr> _rs_metas; // This variable _stale_rs_metas is used to record these rowsets‘ meta which are be compacted. @@ -282,11 +287,11 @@ inline void TabletMeta::set_in_restore_mode(bool in_restore_mode) { } inline const TabletSchema& TabletMeta::tablet_schema() const { - return _schema; + return *_schema; } inline TabletSchema* TabletMeta::mutable_tablet_schema() { - return &_schema; + return _schema.get(); } inline const std::vector<RowsetMetaSharedPtr>& TabletMeta::all_rs_metas() const { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
