This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new a0b3840 [MemerySave] Change TabletSchema in tablet to reference to
save mem (#6814)
a0b3840 is described below
commit a0b3840daa022c515ab056d4ef69d23be5dc32b8
Author: HappenLee <[email protected]>
AuthorDate: Sat Oct 16 21:54:32 2021 +0800
[MemerySave] Change TabletSchema in tablet to reference to save mem (#6814)
Change TabletSchema in tablet to reference to save memory
---
be/src/olap/base_tablet.cpp | 2 --
be/src/olap/base_tablet.h | 2 +-
be/src/olap/tablet.cpp | 6 +-----
be/src/olap/tablet_meta.cpp | 31 +++++++++++++++++++++++++------
be/src/olap/tablet_meta.h | 11 ++++++++---
5 files changed, 35 insertions(+), 17 deletions(-)
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 35b9d7e..0f634ba 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -31,8 +31,6 @@ extern MetricPrototype METRIC_query_scan_count;
BaseTablet::BaseTablet(TabletMetaSharedPtr tablet_meta, DataDir* data_dir)
: _state(tablet_meta->tablet_state()),
_tablet_meta(tablet_meta),
- // TODO: Think we really copy tablet schema here, which cost double
mem
- // cost in TabletManager
_schema(tablet_meta->tablet_schema()),
_data_dir(data_dir) {
_gen_tablet_path();
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index 9d0e331..3e1386b 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -70,7 +70,7 @@ protected:
protected:
TabletState _state;
TabletMetaSharedPtr _tablet_meta;
- TabletSchema _schema;
+ const TabletSchema& _schema;
DataDir* _data_dir;
std::string _tablet_path;
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 3d22614..b6dc448 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -136,9 +136,6 @@ void Tablet::save_meta() {
auto res = _tablet_meta->save_meta(_data_dir);
CHECK_EQ(res, OLAP_SUCCESS) << "fail to save tablet_meta. res=" << res
<< ", root=" << _data_dir->path();
- // User could directly update tablet schema by _tablet_meta,
- // So we need to refetch schema again
- _schema = _tablet_meta->tablet_schema();
}
OLAPStatus Tablet::revise_tablet_meta(const std::vector<RowsetMetaSharedPtr>&
rowsets_to_clone,
@@ -149,8 +146,7 @@ OLAPStatus Tablet::revise_tablet_meta(const
std::vector<RowsetMetaSharedPtr>& ro
OLAPStatus res = OLAP_SUCCESS;
do {
// load new local tablet_meta to operate on
- TabletMetaSharedPtr new_tablet_meta(new (nothrow) TabletMeta());
- generate_tablet_meta_copy_unlocked(new_tablet_meta);
+ TabletMetaSharedPtr new_tablet_meta(new (nothrow)
TabletMeta(*_tablet_meta));
// delete versions from new local tablet_meta
for (const Version& version : versions_to_delete) {
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 824cdaa..564ced8 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -46,14 +46,14 @@ OLAPStatus TabletMeta::create(const TCreateTabletReq&
request, const TabletUid&
return OLAP_SUCCESS;
}
-TabletMeta::TabletMeta() : _tablet_uid(0, 0) {}
+TabletMeta::TabletMeta() : _tablet_uid(0, 0), _schema(new TabletSchema) {}
TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t
tablet_id,
int32_t schema_hash, uint64_t shard_id, const
TTabletSchema& tablet_schema,
uint32_t next_unique_id,
const std::unordered_map<uint32_t, uint32_t>&
col_ordinal_to_unique_id,
TabletUid tablet_uid, TTabletType::type tabletType)
- : _tablet_uid(0, 0), _preferred_rowset_type(ALPHA_ROWSET) {
+ : _tablet_uid(0, 0), _preferred_rowset_type(BETA_ROWSET), _schema(new
TabletSchema) {
TabletMetaPB tablet_meta_pb;
tablet_meta_pb.set_table_id(table_id);
tablet_meta_pb.set_partition_id(partition_id);
@@ -139,6 +139,25 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t
partition_id, int64_t tablet_id
init_from_pb(tablet_meta_pb);
}
+TabletMeta::TabletMeta(const TabletMeta& b)
+ : _table_id(b._table_id),
+ _partition_id(b._partition_id),
+ _tablet_id(b._tablet_id),
+ _schema_hash(b._schema_hash),
+ _shard_id(b._shard_id),
+ _creation_time(b._creation_time),
+ _cumulative_layer_point(b._cumulative_layer_point),
+ _tablet_uid(b._tablet_uid),
+ _tablet_type(b._tablet_type),
+ _tablet_state(b._tablet_state),
+ _schema(b._schema),
+ _in_restore_mode(b._in_restore_mode),
+ _rs_metas(b._rs_metas),
+ _stale_rs_metas(b._stale_rs_metas),
+ _del_pred_array(b._del_pred_array),
+ _preferred_rowset_type(b._preferred_rowset_type) {
+}
+
void TabletMeta::_init_column_from_tcolumn(uint32_t unique_id, const TColumn&
tcolumn,
ColumnPB* column) {
column->set_unique_id(unique_id);
@@ -350,7 +369,7 @@ void TabletMeta::init_from_pb(const TabletMetaPB&
tablet_meta_pb) {
}
// init _schema
- _schema.init_from_pb(tablet_meta_pb.schema());
+ _schema->init_from_pb(tablet_meta_pb.schema());
// init _rs_metas
for (auto& it : tablet_meta_pb.rs_metas()) {
@@ -411,7 +430,7 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
for (auto rs : _stale_rs_metas) {
rs->to_rowset_pb(tablet_meta_pb->add_stale_rs_metas());
}
- _schema.to_schema_pb(tablet_meta_pb->mutable_schema());
+ _schema->to_schema_pb(tablet_meta_pb->mutable_schema());
tablet_meta_pb->set_in_restore_mode(in_restore_mode());
@@ -423,7 +442,7 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
uint32_t TabletMeta::mem_size() const {
auto size = sizeof(TabletMeta);
- size += _schema.mem_size();
+ size += _schema->mem_size();
return size;
}
@@ -625,7 +644,7 @@ bool operator==(const TabletMeta& a, const TabletMeta& b) {
if (a._tablet_uid != b._tablet_uid) return false;
if (a._tablet_type != b._tablet_type) return false;
if (a._tablet_state != b._tablet_state) return false;
- if (a._schema != b._schema) return false;
+ if (*a._schema != *b._schema) return false;
if (a._rs_metas.size() != b._rs_metas.size()) return false;
for (int i = 0; i < a._rs_metas.size(); ++i) {
if (a._rs_metas[i] != b._rs_metas[i]) return false;
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 9f3ba46..8995e96 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -82,6 +82,9 @@ public:
uint64_t shard_id, const TTabletSchema& tablet_schema, uint32_t
next_unique_id,
const std::unordered_map<uint32_t, uint32_t>&
col_ordinal_to_unique_id,
TabletUid tablet_uid, TTabletType::type tabletType);
+ // If need add a filed in TableMeta, filed init copy in copy construct
function
+ TabletMeta(const TabletMeta& tablet_meta);
+ TabletMeta(TabletMeta&& tablet_meta) = delete;
// Function create_from_file is used to be compatible with previous
tablet_meta.
// Previous tablet_meta is a physical file in tablet dir, which is not
stored in rocksdb.
@@ -184,7 +187,9 @@ private:
TabletTypePB _tablet_type = TabletTypePB::TABLET_TYPE_DISK;
TabletState _tablet_state = TABLET_NOTREADY;
- TabletSchema _schema;
+ // the reference of _schema may use in tablet, so here need keep
+ // the lifetime of tablemeta and _schema is same with tablet
+ std::shared_ptr<TabletSchema> _schema;
std::vector<RowsetMetaSharedPtr> _rs_metas;
// This variable _stale_rs_metas is used to record these rowsets‘ meta
which are be compacted.
@@ -282,11 +287,11 @@ inline void TabletMeta::set_in_restore_mode(bool
in_restore_mode) {
}
inline const TabletSchema& TabletMeta::tablet_schema() const {
- return _schema;
+ return *_schema;
}
inline TabletSchema* TabletMeta::mutable_tablet_schema() {
- return &_schema;
+ return _schema.get();
}
inline const std::vector<RowsetMetaSharedPtr>& TabletMeta::all_rs_metas()
const {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
