This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4737aff [Memory Engine] Make Tablet extensible (#3431)
4737aff is described below
commit 4737aff8fced7e5fa5c9f03d83f72188acf49834
Author: Binglin Chang <[email protected]>
AuthorDate: Fri May 1 21:21:09 2020 +0800
[Memory Engine] Make Tablet extensible (#3431)
Adding a new storage engine, we need to make an extensible tablet
interface, so olap/StorageEngine can support and manage new tablet types.
To start, this commit creates a class BaseTablet and make Tablet and new
MemTablet inherit
this base class, some common fields & methods are moved to BaseTablet
class, which fields
and methods belong to base/old class is not finalized yet, it will change
as the project evolves.
Fix #3384
---
be/CMakeLists.txt | 1 +
be/src/olap/CMakeLists.txt | 2 +
be/src/olap/base_tablet.cpp | 43 ++++++++++++
be/src/olap/base_tablet.h | 137 ++++++++++++++++++++++++++++++++++++++
be/src/olap/memory/CMakeLists.txt | 26 ++++++++
be/src/olap/memory/mem_tablet.cpp | 22 ++++++
be/src/olap/memory/mem_tablet.h | 39 +++++++++++
be/src/olap/tablet.cpp | 15 +----
be/src/olap/tablet.h | 87 +-----------------------
9 files changed, 273 insertions(+), 99 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 8a56448..fa0b966 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -363,6 +363,7 @@ set(DORIS_LINK_LIBS
Exec
Exprs
Gutil
+ Memory
Olap
Rowset
OlapFs
diff --git a/be/src/olap/CMakeLists.txt b/be/src/olap/CMakeLists.txt
index cb1e7ec..3ade4c0 100644
--- a/be/src/olap/CMakeLists.txt
+++ b/be/src/olap/CMakeLists.txt
@@ -23,10 +23,12 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/olap")
add_subdirectory(rowset)
add_subdirectory(fs)
+add_subdirectory(memory)
add_library(Olap STATIC
aggregate_func.cpp
base_compaction.cpp
+ base_tablet.cpp
bloom_filter.hpp
bloom_filter_reader.cpp
bloom_filter_writer.cpp
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
new file mode 100644
index 0000000..41aa93a
--- /dev/null
+++ b/be/src/olap/base_tablet.cpp
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "base_tablet.h"
+
+namespace doris {
+
+BaseTablet::BaseTablet(TabletMetaSharedPtr tablet_meta, DataDir* data_dir) :
+ _state(tablet_meta->tablet_state()),
+ _tablet_meta(tablet_meta),
+ _schema(tablet_meta->tablet_schema()),
+ _data_dir(data_dir) {
+}
+
+BaseTablet::~BaseTablet() {
+}
+
+
+OLAPStatus BaseTablet::set_tablet_state(TabletState state) {
+ if (_tablet_meta->tablet_state() == TABLET_SHUTDOWN && state !=
TABLET_SHUTDOWN) {
+ LOG(WARNING) << "could not change tablet state from shutdown to " <<
state;
+ return OLAP_ERR_META_INVALID_ARGUMENT;
+ }
+ _tablet_meta->set_tablet_state(state);
+ _state = state;
+ return OLAP_SUCCESS;
+}
+
+} /* namespace doris */
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
new file mode 100644
index 0000000..e680e07
--- /dev/null
+++ b/be/src/olap/base_tablet.h
@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef DORIS_BE_SRC_OLAP_BASE_TABLET_H
+#define DORIS_BE_SRC_OLAP_BASE_TABLET_H
+
+#include <memory>
+#include "olap/olap_define.h"
+#include "olap/tablet_meta.h"
+#include "olap/utils.h"
+
+namespace doris {
+
+// Base class for all tablet classes, currently only olap/Tablet and
+// olap/memory/MemTablet.
+// The fields and methods in this class is not final, it will change as memory
+// storage engine evolves.
+class BaseTablet : public std::enable_shared_from_this<BaseTablet> {
+public:
+ BaseTablet(TabletMetaSharedPtr tablet_meta, DataDir* data_dir);
+ virtual ~BaseTablet();
+
+ inline DataDir* data_dir() const;
+ std::string tablet_path() const;
+
+ TabletState tablet_state() const { return _state; }
+ OLAPStatus set_tablet_state(TabletState state);
+
+ // Property encapsulated in TabletMeta
+ inline const TabletMetaSharedPtr tablet_meta();
+
+ inline TabletUid tablet_uid() const;
+ inline int64_t table_id() const;
+ // Returns a string can be used to uniquely identify a tablet.
+ // The result string will often be printed to the log.
+ inline const std::string full_name() const;
+ inline int64_t partition_id() const;
+ inline int64_t tablet_id() const;
+ inline int32_t schema_hash() const;
+ inline int16_t shard_id();
+ inline const int64_t creation_time() const;
+ inline void set_creation_time(int64_t creation_time);
+ inline bool equal(int64_t tablet_id, int32_t schema_hash);
+
+ // propreties encapsulated in TabletSchema
+ inline const TabletSchema& tablet_schema() const;
+
+protected:
+ TabletState _state;
+ TabletMetaSharedPtr _tablet_meta;
+ TabletSchema _schema;
+
+ DataDir* _data_dir;
+ std::string _tablet_path;
+
+private:
+ DISALLOW_COPY_AND_ASSIGN(BaseTablet);
+};
+
+
+inline DataDir* BaseTablet::data_dir() const {
+ return _data_dir;
+}
+
+inline string BaseTablet::tablet_path() const {
+ return _tablet_path;
+}
+
+inline const TabletMetaSharedPtr BaseTablet::tablet_meta() {
+ return _tablet_meta;
+}
+
+inline TabletUid BaseTablet::tablet_uid() const {
+ return _tablet_meta->tablet_uid();
+}
+
+inline int64_t BaseTablet::table_id() const {
+ return _tablet_meta->table_id();
+}
+
+inline const std::string BaseTablet::full_name() const {
+ std::stringstream ss;
+ ss << _tablet_meta->tablet_id()
+ << "." << _tablet_meta->schema_hash()
+ << "." << _tablet_meta->tablet_uid().to_string();
+ return ss.str();
+}
+
+inline int64_t BaseTablet::partition_id() const {
+ return _tablet_meta->partition_id();
+}
+
+inline int64_t BaseTablet::tablet_id() const {
+ return _tablet_meta->tablet_id();
+}
+
+inline int32_t BaseTablet::schema_hash() const {
+ return _tablet_meta->schema_hash();
+}
+
+inline int16_t BaseTablet::shard_id() {
+ return _tablet_meta->shard_id();
+}
+
+inline const int64_t BaseTablet::creation_time() const {
+ return _tablet_meta->creation_time();
+}
+
+inline void BaseTablet::set_creation_time(int64_t creation_time) {
+ _tablet_meta->set_creation_time(creation_time);
+}
+
+inline bool BaseTablet::equal(int64_t id, int32_t hash) {
+ return (tablet_id() == id) && (schema_hash() == hash);
+}
+
+inline const TabletSchema& BaseTablet::tablet_schema() const {
+ return _schema;
+}
+
+} /* namespace doris */
+
+#endif /* DORIS_BE_SRC_OLAP_BASE_TABLET_H */
diff --git a/be/src/olap/memory/CMakeLists.txt
b/be/src/olap/memory/CMakeLists.txt
new file mode 100644
index 0000000..7b98444
--- /dev/null
+++ b/be/src/olap/memory/CMakeLists.txt
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# where to put generated libraries
+set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/olap/memory")
+
+# where to put generated binaries
+set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/olap/memory")
+
+add_library(Memory STATIC
+ mem_tablet.cpp
+)
diff --git a/be/src/olap/memory/mem_tablet.cpp
b/be/src/olap/memory/mem_tablet.cpp
new file mode 100644
index 0000000..9137c15
--- /dev/null
+++ b/be/src/olap/memory/mem_tablet.cpp
@@ -0,0 +1,22 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/memory/mem_tablet.h"
+
+namespace doris {
+
+} /* namespace doris */
diff --git a/be/src/olap/memory/mem_tablet.h b/be/src/olap/memory/mem_tablet.h
new file mode 100644
index 0000000..9fd1312
--- /dev/null
+++ b/be/src/olap/memory/mem_tablet.h
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef DORIS_BE_SRC_MEMORY_MEM_TABLET_H_
+#define DORIS_BE_SRC_MEMORY_MEM_TABLET_H_
+
+#include "olap/base_tablet.h"
+
+namespace doris {
+
+// Tablet class for memory-optimized storage engine.
+//
+// It stores all its data in-memory, and is designed for tables with
+// frequent updates.
+//
+// TODO: This is just a skeleton, will add implementation in the future.
+class MemTablet : public BaseTablet {
+
+private:
+ DISALLOW_COPY_AND_ASSIGN(MemTablet);
+};
+
+} /* namespace doris */
+
+#endif /* DORIS_BE_SRC_MEMORY_MEM_TABLET_H_ */
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 5119be0..04ce8db 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -62,10 +62,7 @@ void Tablet::_gen_tablet_path() {
}
Tablet::Tablet(TabletMetaSharedPtr tablet_meta, DataDir* data_dir) :
- _state(tablet_meta->tablet_state()),
- _tablet_meta(tablet_meta),
- _schema(tablet_meta->tablet_schema()),
- _data_dir(data_dir),
+ BaseTablet(tablet_meta, data_dir),
_is_bad(false),
_last_cumu_compaction_failure_millis(0),
_last_base_compaction_failure_millis(0),
@@ -118,16 +115,6 @@ OLAPStatus Tablet::init() {
return _init_once.call([this] { return _init_once_action(); });
}
-OLAPStatus Tablet::set_tablet_state(TabletState state) {
- if (_tablet_meta->tablet_state() == TABLET_SHUTDOWN && state !=
TABLET_SHUTDOWN) {
- LOG(WARNING) << "could not change tablet state from shutdown to " <<
state;
- return OLAP_ERR_META_INVALID_ARGUMENT;
- }
- _tablet_meta->set_tablet_state(state);
- _state = state;
- return OLAP_SUCCESS;
-}
-
// should save tablet meta to remote meta store
// if it's a primary replica
void Tablet::save_meta() {
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 8cb6565..f8746f8 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -35,6 +35,7 @@
#include "olap/rowset/rowset_reader.h"
#include "olap/tablet_meta.h"
#include "olap/utils.h"
+#include "olap/base_tablet.h"
#include "util/once.h"
namespace doris {
@@ -45,7 +46,7 @@ class TabletMeta;
using TabletSharedPtr = std::shared_ptr<Tablet>;
-class Tablet : public std::enable_shared_from_this<Tablet> {
+class Tablet : public BaseTablet {
public:
static TabletSharedPtr create_tablet_from_meta(TabletMetaSharedPtr
tablet_meta,
DataDir* data_dir =
nullptr);
@@ -57,44 +58,23 @@ public:
bool is_used();
- inline DataDir* data_dir() const;
void register_tablet_into_dir();
void deregister_tablet_from_dir();
- std::string tablet_path() const;
-
- TabletState tablet_state() const { return _state; }
- OLAPStatus set_tablet_state(TabletState state);
-
- // Property encapsulated in TabletMeta
- inline const TabletMetaSharedPtr tablet_meta();
void save_meta();
// Used in clone task, to update local meta when finishing a clone job
OLAPStatus revise_tablet_meta(const std::vector<RowsetMetaSharedPtr>&
rowsets_to_clone,
const std::vector<Version>&
versions_to_delete);
- inline TabletUid tablet_uid() const;
- inline int64_t table_id() const;
- // Returns a string can be used to uniquely identify a tablet.
- // The result string will often be printed to the log.
- inline const std::string full_name() const;
- inline int64_t partition_id() const;
- inline int64_t tablet_id() const;
- inline int32_t schema_hash() const;
- inline int16_t shard_id();
- inline const int64_t creation_time() const;
- inline void set_creation_time(int64_t creation_time);
inline const int64_t cumulative_layer_point() const;
inline void set_cumulative_layer_point(int64_t new_point);
- inline bool equal(int64_t tablet_id, int32_t schema_hash);
inline size_t tablet_footprint(); // disk space occupied by tablet
inline size_t num_rows();
inline int version_count() const;
inline Version max_version() const;
// propreties encapsulated in TabletSchema
- inline const TabletSchema& tablet_schema() const;
inline KeysType keys_type() const;
inline size_t num_columns() const;
inline size_t num_null_columns() const;
@@ -261,12 +241,7 @@ private:
private:
static const int64_t kInvalidCumulativePoint = -1;
- TabletState _state;
- TabletMetaSharedPtr _tablet_meta;
- TabletSchema _schema;
- DataDir* _data_dir;
- std::string _tablet_path;
RowsetGraph _rs_graph;
DorisCallOnce<OLAPStatus> _init_once;
@@ -319,10 +294,6 @@ inline bool Tablet::is_used() {
return !_is_bad && _data_dir->is_used();
}
-inline DataDir* Tablet::data_dir() const {
- return _data_dir;
-}
-
inline void Tablet::register_tablet_into_dir() {
_data_dir->register_tablet(this);
}
@@ -331,53 +302,6 @@ inline void Tablet::deregister_tablet_from_dir() {
_data_dir->deregister_tablet(this);
}
-inline string Tablet::tablet_path() const {
- return _tablet_path;
-}
-
-inline const TabletMetaSharedPtr Tablet::tablet_meta() {
- return _tablet_meta;
-}
-
-inline TabletUid Tablet::tablet_uid() const {
- return _tablet_meta->tablet_uid();
-}
-
-inline int64_t Tablet::table_id() const {
- return _tablet_meta->table_id();
-}
-
-inline const std::string Tablet::full_name() const {
- std::stringstream ss;
- ss << _tablet_meta->tablet_id()
- << "." << _tablet_meta->schema_hash()
- << "." << _tablet_meta->tablet_uid().to_string();
- return ss.str();
-}
-
-inline int64_t Tablet::partition_id() const {
- return _tablet_meta->partition_id();
-}
-
-inline int64_t Tablet::tablet_id() const {
- return _tablet_meta->tablet_id();
-}
-
-inline int32_t Tablet::schema_hash() const {
- return _tablet_meta->schema_hash();
-}
-
-inline int16_t Tablet::shard_id() {
- return _tablet_meta->shard_id();
-}
-
-inline const int64_t Tablet::creation_time() const {
- return _tablet_meta->creation_time();
-}
-
-inline void Tablet::set_creation_time(int64_t creation_time) {
- _tablet_meta->set_creation_time(creation_time);
-}
inline const int64_t Tablet::cumulative_layer_point() const {
return _cumulative_point;
@@ -387,9 +311,6 @@ inline void Tablet::set_cumulative_layer_point(int64_t
new_point) {
_cumulative_point = new_point;
}
-inline bool Tablet::equal(int64_t id, int32_t hash) {
- return (tablet_id() == id) && (schema_hash() == hash);
-}
// TODO(lingbin): Why other methods that need to get information from
_tablet_meta
// are not locked, here needs a comment to explain.
@@ -413,10 +334,6 @@ inline Version Tablet::max_version() const {
return _tablet_meta->max_version();
}
-inline const TabletSchema& Tablet::tablet_schema() const {
- return _schema;
-}
-
inline KeysType Tablet::keys_type() const {
return _schema.keys_type();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
