chaoyli closed pull request #429: add rowset meta manager
URL: https://github.com/apache/incubator-doris/pull/429
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/be/src/olap/CMakeLists.txt b/be/src/olap/CMakeLists.txt
index a61b7f19..c9aafa7a 100644
--- a/be/src/olap/CMakeLists.txt
+++ b/be/src/olap/CMakeLists.txt
@@ -84,4 +84,5 @@ add_library(Olap STATIC
types.cpp
utils.cpp
wrapper_field.cpp
+ rowset/rowset_meta_manager.cpp
)
diff --git a/be/src/olap/rowset.h b/be/src/olap/rowset/rowset.h
similarity index 67%
rename from be/src/olap/rowset.h
rename to be/src/olap/rowset/rowset.h
index cb8dd07f..d3b81c6d 100644
--- a/be/src/olap/rowset.h
+++ b/be/src/olap/rowset/rowset.h
@@ -1,45 +1,47 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef DORIS_BE_SRC_OLAP_ROWSET_H
-#define DORIS_BE_SRC_OLAP_ROWSET_H
-
-#include "olap/new_status.h"
-#include "olap/rowset_reader.h"
-#include "olap/rowset_builder.h"
-
-#include <memory>
-
-namespace doris {
-
-class Rowset {
-public:
- NewStatus init(const RowsetMeta& rowset_meta, const DataDir* dir) = 0;
-
- std::unique_ptr<RowsetReader> create_reader() = 0;
-
- NewStatus copy(RowsetBuilder* dest_rowset_builder) = 0;
-
- NewStatus delete() = 0;
-
-private:
- RowsetMeta _rowset_meta;
-};
-
-}
-
-#endif // DORIS_BE_SRC_OLAP_ROWSET_H
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_H
+#define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_H
+
+#include "olap/new_status.h"
+#include "olap/rowset/rowset_reader.h"
+#include "olap/rowset/rowset_builder.h"
+
+#include <memory>
+
+namespace doris {
+
+class Rowset {
+public:
+ virtual ~Rowset() { }
+
+ virtual NewStatus init(const RowsetMeta& rowset_meta) = 0;
+
+ virtual std::unique_ptr<RowsetReader> create_reader() = 0;
+
+ virtual NewStatus copy(RowsetBuilder* dest_rowset_builder) = 0;
+
+ virtual NewStatus delete() = 0;
+
+private:
+ RowsetMeta _rowset_meta;
+};
+
+}
+
+#endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_H
diff --git a/be/src/olap/rowset_builder.h b/be/src/olap/rowset/rowset_builder.h
similarity index 68%
rename from be/src/olap/rowset_builder.h
rename to be/src/olap/rowset/rowset_builder.h
index 9434efaa..d8697f89 100644
--- a/be/src/olap/rowset_builder.h
+++ b/be/src/olap/rowset/rowset_builder.h
@@ -1,45 +1,47 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef DORIS_BE_SRC_OLAP_ROWSET_BUILDER_H
-#define DORIS_BE_SRC_OLAP_ROWSET_BUILDER_H
-
-#include "rowset/rowset.h"
-#include "olap/new_status.h"
-#include "olap/schema.h"
-#include "olap/row_block.h"
-
-namespace doris {
-
-class RowsetBuilder {
-public:
- NewStatus init(std::string rowset_id, const std::string&
rowset_path_prefix, Schema* schema) = 0;
-
- // add a row block to rowset
- NewStatus add_row_block(RowBlock* row_block) = 0;
-
- // this is a temp api
- // it is used to get rewritten path for writing rowset data
- NewStatus generate_written_path(const std::string& src_path, std::string*
dest_path) = 0;
-
- // get a rowset
- NewStatus build(Rowset* rowset) = 0;
-};
-
-}
-
-#endif // DORIS_BE_SRC_OLAP_ROWSET_BUILDER_H
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_BUILDER_H
+#define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_BUILDER_H
+
+#include "rowset/rowset.h"
+#include "olap/new_status.h"
+#include "olap/schema.h"
+#include "olap/row_block.h"
+
+namespace doris {
+
+class RowsetBuilder {
+public:
+ virtual ~RowsetBuilder() { }
+
+ virtual NewStatus init(std::string rowset_id, const std::string&
rowset_path_prefix, Schema* schema) = 0;
+
+ // add a row block to rowset
+ virtual NewStatus add_row_block(RowBlock* row_block) = 0;
+
+ // this is a temp api
+ // it is used to get rewritten path for writing rowset data
+ virtual NewStatus generate_written_path(const std::string& src_path,
std::string* dest_path) = 0;
+
+ // get a rowset
+ virtual NewStatus build(Rowset* rowset) = 0;
+};
+
+}
+
+#endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_BUILDER_H
diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h
new file mode 100644
index 00000000..abfc2030
--- /dev/null
+++ b/be/src/olap/rowset/rowset_meta.h
@@ -0,0 +1,214 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H
+#define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H
+
+#include "gen_cpp/olap_file.pb.h"
+
+#include <memory>
+#include <vector>
+
+#include "olap/new_status.h"
+#include "olap/olap_common.h"
+#include "json2pb/json_to_pb.h"
+#include "json2pb/pb_to_json.h"
+
+namespace doris {
+
+class RowsetMeta {
+public:
+ virtual ~RowsetMeta() { }
+
+ virtual bool init(const std::string& pb_rowset_meta) {
+ return _deserialize_from_pb(pb_rowset_meta);
+ }
+
+ virtual bool init_from_json(const std::string& json_rowset_meta) {
+ bool ret = json2pb::JsonToProtoMessage(json_rowset_meta,
&_rowset_meta_pb);
+ return ret;
+ }
+
+ virtual bool deserialize_extra_properties() {
+ return true;
+ }
+
+ virtual bool serialize(std::string* value) {
+ return _serialize_to_pb(value);
+ }
+
+ virtual bool get_json_rowset_meta(std::string* json_rowset_meta) {
+ json2pb::Pb2JsonOptions json_options;
+ json_options.pretty_json = true;
+ bool ret = json2pb::ProtoMessageToJson(_rowset_meta_pb,
json_rowset_meta, json_options);
+ return ret;
+ }
+
+ virtual int64_t get_rowset_id() {
+ return _rowset_meta_pb.rowset_id();
+ }
+
+ virtual void set_rowset_id(int64_t rowset_id) {
+ _rowset_meta_pb.set_rowset_id(rowset_id);
+ }
+
+ virtual int64_t get_tablet_id() {
+ return _rowset_meta_pb.tablet_id();
+ }
+
+ virtual void set_tablet_id(int64_t tablet_id) {
+ _rowset_meta_pb.set_tablet_id(tablet_id);
+ }
+
+ virtual int32_t get_tablet_schema_hash() {
+ return _rowset_meta_pb.tablet_schema_hash();
+ }
+
+ virtual void set_tablet_schema_hash(int64_t tablet_schema_hash) {
+ _rowset_meta_pb.set_tablet_schema_hash(tablet_schema_hash);
+ }
+
+ virtual RowsetType get_rowset_type() {
+ return _rowset_meta_pb.rowset_type();
+ }
+
+ virtual void set_rowset_type(RowsetType rowset_type) {
+ _rowset_meta_pb.set_rowset_type(rowset_type);
+ }
+
+ virtual RowsetState get_rowset_state() {
+ return _rowset_meta_pb.rowset_state();
+ }
+
+ virtual void set_rowset_state(RowsetState rowset_state) {
+ _rowset_meta_pb.set_rowset_state(rowset_state);
+ }
+
+ virtual Version get_version() {
+ Version version;
+ version.first = _rowset_meta_pb.start_version();
+ version.second = _rowset_meta_pb.end_version();
+ return version;
+ }
+
+ virtual void set_version(Version version) {
+ _rowset_meta_pb.set_start_version(version.first);
+ _rowset_meta_pb.set_end_version(version.second);
+ }
+
+ virtual int get_start_version() {
+ return _rowset_meta_pb.start_version();
+ }
+
+ virtual void set_start_version(int start_version) {
+ _rowset_meta_pb.set_start_version(start_version);
+ }
+
+ virtual int get_end_version() {
+ return _rowset_meta_pb.end_version();
+ }
+
+ virtual void set_end_version(int end_version) {
+ _rowset_meta_pb.set_end_version(end_version);
+ }
+
+ virtual int get_row_number() {
+ return _rowset_meta_pb.row_number();
+ }
+
+ virtual void set_row_number(int row_number) {
+ _rowset_meta_pb.set_row_number(row_number);
+ }
+
+ virtual int get_total_disk_size() {
+ return _rowset_meta_pb.total_disk_size();
+ }
+
+ virtual void set_total_disk_size(int total_disk_size) {
+ _rowset_meta_pb.set_total_disk_size(total_disk_size);
+ }
+
+ virtual int get_data_disk_size() {
+ return _rowset_meta_pb.data_disk_size();
+ }
+
+ virtual void set_data_disk_size(int data_disk_size) {
+ _rowset_meta_pb.set_data_disk_size(data_disk_size);
+ }
+
+ virtual int get_index_disk_size() {
+ return _rowset_meta_pb.index_disk_size();
+ }
+
+ virtual void set_index_disk_size(int index_disk_size) {
+ _rowset_meta_pb.set_index_disk_size(index_disk_size);
+ }
+
+ virtual void get_column_statistics(std::vector<ColumnPruning>*
column_statistics) {
+ for (const ColumnPruning& column_statistic :
_rowset_meta_pb.column_statistics()) {
+ column_statistics->push_back(column_statistic);
+ }
+ }
+
+ virtual void set_column_statistics(const std::vector<ColumnPruning>&
column_statistics) {
+ for (const ColumnPruning& column_statistic : column_statistics) {
+ ColumnPruning* new_column_statistic =
_rowset_meta_pb.add_column_statistics();
+ *new_column_statistic = column_statistic;
+ }
+ }
+
+ virtual void add_column_statistic(const ColumnPruning& column_statistic) {
+ ColumnPruning* new_column_statistic =
_rowset_meta_pb.add_column_statistics();
+ *new_column_statistic = column_statistic;
+ }
+
+ virtual const DeleteConditionMessage& get_delete_predicate() {
+ return _rowset_meta_pb.delete_predicate();
+ }
+
+ virtual void set_delete_predicate(DeleteConditionMessage&
delete_predicate) {
+ DeleteConditionMessage* new_delete_condition =
_rowset_meta_pb.mutable_delete_predicate();
+ *new_delete_condition = delete_predicate;
+ }
+
+ virtual int64_t get_txn_id() {
+ return _rowset_meta_pb.txn_id();
+ }
+
+ virtual void set_txn_id(int64_t txn_id) {
+ _rowset_meta_pb.set_txn_id(txn_id);
+ }
+
+private:
+ bool _deserialize_from_pb(const std::string& value) {
+ return _rowset_meta_pb.ParseFromString(value);
+ }
+
+ bool _serialize_to_pb(std::string* value) {
+ if (value == nullptr) {
+ return false;
+ }
+ return _rowset_meta_pb.SerializeToString(value);
+ }
+
+private:
+ RowsetMetaPb _rowset_meta_pb;
+};
+
+}
+
+#endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H
diff --git a/be/src/olap/rowset/rowset_meta_manager.cpp
b/be/src/olap/rowset/rowset_meta_manager.cpp
new file mode 100644
index 00000000..15e0542c
--- /dev/null
+++ b/be/src/olap/rowset/rowset_meta_manager.cpp
@@ -0,0 +1,140 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/rowset_meta_manager.h"
+
+#include <vector>
+#include <sstream>
+#include <string>
+#include <fstream>
+#include <boost/algorithm/string/trim.hpp>
+
+#include "olap/olap_define.h"
+#include "olap/storage_engine.h"
+#include "common/logging.h"
+#include "json2pb/json_to_pb.h"
+#include "json2pb/pb_to_json.h"
+
+namespace doris {
+
+const std::string ROWSET_PREFIX = "rst_";
+
+NewStatus convert_meta_status(OLAPStatus status) {
+ if (status == OLAP_SUCCESS) {
+ return NewStatus::OK();
+ } else {
+ std::string error_msg = "meta operation failed";
+ LOG(WARNING) << error_msg;
+ return NewStatus::IOError(error_msg);
+ }
+}
+
+NewStatus RowsetMetaManager::get_rowset_meta(OlapMeta* meta, int64_t
rowset_id, RowsetMeta* rowset_meta) {
+ std::string key = ROWSET_PREFIX + std::to_string(rowset_id);
+ std::string value;
+ OLAPStatus s = meta->get(META_COLUMN_FAMILY_INDEX, key, value);
+ if (s == OLAP_ERR_META_KEY_NOT_FOUND) {
+ std::string error_msg = "rowset id:" + std::to_string(rowset_id) + "
not found.";
+ LOG(WARNING) << error_msg;
+ return NewStatus::NotFound(error_msg);
+ } else if (s != OLAP_SUCCESS) {
+ std::string error_msg = "load rowset id:" + std::to_string(rowset_id)
+ " failed.";
+ LOG(WARNING) << error_msg;
+ return NewStatus::IOError(error_msg);
+ }
+ bool ret = rowset_meta->init(value);
+ if (!ret) {
+ std::string error_msg = "parse rowset meta failed. rowset id:" +
std::to_string(rowset_id);
+ return NewStatus::Corruption(error_msg);
+ }
+ return NewStatus::OK();
+}
+
+NewStatus RowsetMetaManager::get_json_rowset_meta(OlapMeta* meta, int64_t
rowset_id, std::string* json_rowset_meta) {
+ RowsetMeta rowset_meta;
+ NewStatus s = get_rowset_meta(meta, rowset_id, &rowset_meta);
+ if (!s.ok()) {
+ return s;
+ }
+ bool ret = rowset_meta.get_json_rowset_meta(json_rowset_meta);
+ if (!ret) {
+ std::string error_msg = "get json rowset meta failed. rowset id:" +
std::to_string(rowset_id);
+ return NewStatus::Corruption(error_msg);
+ }
+ return NewStatus::OK();
+}
+
+NewStatus RowsetMetaManager::save(OlapMeta* meta, int64_t rowset_id,
RowsetMeta* rowset_meta) {
+ std::string key = ROWSET_PREFIX + std::to_string(rowset_id);
+ std::string value;
+ bool ret = rowset_meta->serialize(&value);
+ if (!ret) {
+ std::string error_msg = "serialize rowset pb failed. rowset id:" +
std::to_string(rowset_id);
+ LOG(WARNING) << error_msg;
+ return NewStatus::Corruption(error_msg);
+ }
+ OLAPStatus status = meta->put(META_COLUMN_FAMILY_INDEX, key, value);
+ return convert_meta_status(status);
+}
+
+NewStatus RowsetMetaManager::remove(OlapMeta* meta, int64_t rowset_id) {
+ std::string key = ROWSET_PREFIX + std::to_string(rowset_id);
+ LOG(INFO) << "start to remove rowset, key:" << key;
+ OLAPStatus status = meta->remove(META_COLUMN_FAMILY_INDEX, key);
+ LOG(INFO) << "remove rowset key:" << key << " finished";
+ return convert_meta_status(status);
+}
+
+NewStatus RowsetMetaManager::traverse_rowset_metas(OlapMeta* meta,
+ std::function<bool(uint64_t, const std::string&)> const& func) {
+ auto traverse_rowset_meta_func = [&func](const std::string& key, const
std::string& value) -> bool {
+ std::vector<std::string> parts;
+ // key format: "rst_" + rowset_id
+ split_string<char>(key, '_', &parts);
+ if (parts.size() != 2) {
+ LOG(WARNING) << "invalid rowset key:" << key << ", splitted size:"
<< parts.size();
+ return true;
+ }
+ uint64_t rowset_id = std::stol(parts[1].c_str(), NULL, 10);
+ return func(rowset_id, value);
+ };
+ OLAPStatus status = meta->iterate(META_COLUMN_FAMILY_INDEX, ROWSET_PREFIX,
traverse_rowset_meta_func);
+ return convert_meta_status(status);
+}
+
+NewStatus RowsetMetaManager::load_json_rowset_meta(OlapMeta* meta, const
std::string& rowset_meta_path) {
+ std::ifstream infile(rowset_meta_path);
+ char buffer[1024];
+ std::string json_rowset_meta;
+ while (!infile.eof()) {
+ infile.getline(buffer, 1024);
+ json_rowset_meta = json_rowset_meta + buffer;
+ }
+ boost::algorithm::trim(json_rowset_meta);
+ RowsetMeta rowset_meta;
+ bool ret = rowset_meta.init_from_json(json_rowset_meta);
+ if (!ret) {
+ std::string error_msg = "parse json rowset meta failed.";
+ LOG(WARNING) << error_msg;
+ return NewStatus::Corruption(error_msg);
+ }
+ uint64_t rowset_id = rowset_meta.get_rowset_id();
+ NewStatus status = save(meta, rowset_id, &rowset_meta);
+ return status;
+}
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/rowset/rowset_meta_manager.h
b/be/src/olap/rowset/rowset_meta_manager.h
new file mode 100644
index 00000000..d5f161b9
--- /dev/null
+++ b/be/src/olap/rowset/rowset_meta_manager.h
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_MANAGER_H
+#define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_MANAGER_H
+
+#include <string>
+
+#include "olap/rowset/rowset_meta.h"
+#include "olap/olap_meta.h"
+#include "olap/new_status.h"
+
+namespace doris {
+
+// Helper class for managing rowset meta of one root path.
+class RowsetMetaManager {
+public:
+ static NewStatus get_rowset_meta(OlapMeta* meta, int64_t rowset_id,
RowsetMeta* rowset_meta);
+
+ static NewStatus get_json_rowset_meta(OlapMeta* meta, int64_t rowset_id,
std::string* json_rowset_meta);
+
+ static NewStatus save(OlapMeta* meta, int64_t rowset_id, RowsetMeta*
rowset_meta);
+
+ static NewStatus remove(OlapMeta* meta, int64_t rowset_id);
+
+ static NewStatus traverse_rowset_metas(OlapMeta* meta,
+ std::function<bool(uint64_t, const std::string&)> const& func);
+
+ static NewStatus load_json_rowset_meta(OlapMeta* meta, const std::string&
rowset_meta_path);
+};
+
+}
+
+#endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_MANAGER_H
diff --git a/be/src/olap/rowset_reader.h b/be/src/olap/rowset/rowset_reader.h
similarity index 79%
rename from be/src/olap/rowset_reader.h
rename to be/src/olap/rowset/rowset_reader.h
index 4478851d..b168cea3 100644
--- a/be/src/olap/rowset_reader.h
+++ b/be/src/olap/rowset/rowset_reader.h
@@ -1,57 +1,59 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef DORIS_BE_SRC_OLAP_ROWSET_READER_H
-#define DORIS_BE_SRC_OLAP_ROWSET_READER_H
-
-#include "olap/new_status.h"
-#include "olap/schema.h"
-#include "olap/column_predicate.h"
-#include "olap/row_cursor.h"
-#include "olap/row_block.h"
-
-#include <memory>
-#include <unordered_map>
-
-namespace doris {
-
-struct ReadContext {
- const Schema* projection;
- std::unordered_map<std::string, ColumnPredicate> predicates; // column
name -> column predicate
- const RowCursor* lower_bound_key;
- const RowCursor* exclusive_upper_bound_key;
-};
-
-class RowsetReader {
-public:
- // reader init
- // check whether this rowset can be filtered
- NewStatus init(ReadContext* read_context) = 0;
-
- // check whether rowset has more data
- bool has_next() = 0;
-
- // read next block data
- NewStatus next_block(RowBlock* row_block) = 0;
-
- // close reader
- void close() = 0;
-};
-
-}
-
-#endif // DORIS_BE_SRC_OLAP_ROWSET_READER_H
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_H
+#define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_H
+
+#include "olap/new_status.h"
+#include "olap/schema.h"
+#include "olap/column_predicate.h"
+#include "olap/row_cursor.h"
+#include "olap/row_block.h"
+
+#include <memory>
+#include <unordered_map>
+
+namespace doris {
+
+struct ReadContext {
+ const Schema* projection;
+ std::unordered_map<std::string, ColumnPredicate> predicates; // column
name -> column predicate
+ const RowCursor* lower_bound_key;
+ const RowCursor* exclusive_upper_bound_key;
+};
+
+class RowsetReader {
+public:
+ virtual ~RowsetReader() { }
+
+ // reader init
+ // check whether this rowset can be filtered
+ virtual NewStatus init(ReadContext* read_context) = 0;
+
+ // check whether rowset has more data
+ virtual bool has_next() = 0;
+
+ // read next block data
+ virtual NewStatus next_block(RowBlock* row_block) = 0;
+
+ // close reader
+ virtual void close() = 0;
+};
+
+}
+
+#endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_H
diff --git a/be/src/olap/rowset_meta.h b/be/src/olap/rowset_meta.h
deleted file mode 100644
index 022ea2a1..00000000
--- a/be/src/olap/rowset_meta.h
+++ /dev/null
@@ -1,158 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef DORIS_BE_SRC_OLAP_ROWSET_META_H
-#define DORIS_BE_SRC_OLAP_ROWSET_META_H
-
-#include "gen_cpp/olap_file.pb.h"
-
-#include <memory>
-#include <vector>
-
-namespace doris {
-
-class RowsetMeta {
-public:
- virtual void init(const RowsetMetaPb& rowset_meta) {
- _rowset_meta = rowset_meta;
- }
-
- virtual bool deserialize_extra_properties() = 0;
-
- virtual void get_rowset_meta_pb(RowsetMetaPb* rowset_meta) {
- rowset_meta = &_rowset_meta;
- }
- virtual int64_t get_rowset_id() {
- return _rowset_meta.rowset_id();
- }
-
- virtual void set_rowset_id(int64_t rowset_id) {
- _rowset_meta.set_rowset_id(rowset_id);
- }
-
- virtual int64_t get_version() {
- return _rowset_meta->version();
- }
-
- virtual void set_version(int64_t version) {
- _rowset_meta->set_version(version);
- }
-
- virtual int64_t get_tablet_id() {
- return _rowset_meta->tablet_id();
- }
-
- virtual void set_tablet_id(int64_t tablet_id) {
- _rowset_meta->set_tablet_id(tablet_id);
- }
-
- virtual int32_t get_tablet_schema_hash() {
- return _rowset_meta->tablet_schema_hash();
- }
-
- virtual void set_tablet_schema_hash(int64_t tablet_schema_hash) {
- _rowset_meta->set_tablet_schema_hash(tablet_schema_hash);
- }
-
- virtual RowsetType get_rowset_type() {
- return _rowset_meta->rowset_type();
- }
-
- virtual void set_rowset_type(RowsetType rowset_type) {
- _rowset_meta->set_rowset_type(rowset_type);
- }
-
- virtual RowsetState get_rowset_state() {
- return _rowset_meta->rowset_state();
- }
-
- virtual void set_rowset_state(RowsetState rowset_state) {
- _rowset_meta->set_rowset_state(rowset_state);
- }
-
- virtual int get_start_version() {
- return _rowset_meta->start_version();
- }
-
- virtual void set_start_version(int start_version) {
- _rowset_meta->set_start_version(start_version);
- }
-
- virtual int get_end_version() {
- return _rowset_meta->end_version();
- }
-
- virtual void set_end_version(int end_version) {
- _rowset_meta->set_end_version(end_version);
- }
-
- virtual int get_row_number() {
- return _rowset_meta->row_number();
- }
-
- virtual void set_row_number(int row_number) {
- _rowset_meta->set_row_number(row_number);
- }
-
- virtual int get_total_disk_size() {
- return _rowset_meta->total_disk_size();
- }
-
- virtual void set_total_disk_size(int total_disk_size) {
- _rowset_meta->set_total_disk_size(total_disk_size);
- }
-
- virtual int get_data_disk_size() {
- return _rowset_meta->data_disk_size();
- }
-
- virtual void set_data_disk_size(int data_disk_size) {
- _rowset_meta->set_data_disk_size(data_disk_size);
- }
-
- virtual int get_index_disk_size() {
- return _rowset_meta->index_disk_size();
- }
-
- virtual void set_index_disk_size(int index_disk_size) {
- _rowset_meta->set_index_disk_size(index_disk_size);
- }
-
- virtual void get_column_statistics(std::vector<ColumnPruning>*
column_statistics) {
- *column_statistics = _rowset_meta->column_statistics();
- }
-
- virtual void set_column_statistics(std::vector<ColumnPruning>
column_statistics) {
- std::vector<ColumnPruning>* new_column_statistics =
_rowset_meta.mutable_column_pruning();
- *new_column_statistics = column_statistics;
- }
-
- virtual DeleteConditionMessage get_delete_predicate() {
- return _rowset_meta->delete_predicate();
- }
-
- virtual void set_delete_predicate(DeleteConditionMessage delete_predicate)
{
- _rowset_meta->set_delete_predicate(delete_predicate);
- }
-
-private:
- RowsetMetaPb _rowset_meta;
-};
-
-}
-
-#endif // DORIS_BE_SRC_OLAP_ROWSET_META_H
diff --git a/be/test/olap/CMakeLists.txt b/be/test/olap/CMakeLists.txt
index 3be8aea7..295c7e8a 100644
--- a/be/test/olap/CMakeLists.txt
+++ b/be/test/olap/CMakeLists.txt
@@ -43,3 +43,4 @@ ADD_BE_TEST(delta_writer_test)
ADD_BE_TEST(serialize_test)
ADD_BE_TEST(olap_meta_test)
ADD_BE_TEST(tablet_meta_manager_test)
+ADD_BE_TEST(rowset/rowset_meta_manager_test)
diff --git a/be/test/olap/rowset/rowset_meta_manager_test.cpp
b/be/test/olap/rowset/rowset_meta_manager_test.cpp
new file mode 100644
index 00000000..0d9d3345
--- /dev/null
+++ b/be/test/olap/rowset/rowset_meta_manager_test.cpp
@@ -0,0 +1,107 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <string>
+#include <sstream>
+#include <fstream>
+
+#include "gtest/gtest.h"
+#include "gmock/gmock.h"
+#include "olap/olap_meta.h"
+#include "olap/rowset/rowset_meta_manager.h"
+#include "olap/new_status.h"
+#include "boost/filesystem.hpp"
+#include "json2pb/json_to_pb.h"
+
+#ifndef BE_TEST
+#define BE_TEST
+#endif
+
+using ::testing::_;
+using ::testing::Return;
+using ::testing::SetArgPointee;
+using std::string;
+
+namespace doris {
+
+const std::string rowset_meta_path =
"./be/test/olap/test_data/rowset_meta.json";
+
+class RowsetMetaManagerTest : public testing::Test {
+public:
+ virtual void SetUp() {
+ std::string meta_path = "./meta";
+ ASSERT_TRUE(boost::filesystem::create_directory(meta_path));
+ _meta = new(std::nothrow) OlapMeta(meta_path);
+ ASSERT_NE(nullptr, _meta);
+ OLAPStatus st = _meta->init();
+ ASSERT_TRUE(st == OLAP_SUCCESS);
+ ASSERT_TRUE(boost::filesystem::exists("./meta"));
+
+ std::ifstream infile(rowset_meta_path);
+ char buffer[1024];
+ while (!infile.eof()) {
+ infile.getline(buffer, 1024);
+ _json_rowset_meta = _json_rowset_meta + buffer + "\n";
+ }
+ _json_rowset_meta = _json_rowset_meta.substr(0,
_json_rowset_meta.size() - 1);
+ _json_rowset_meta = _json_rowset_meta.substr(0,
_json_rowset_meta.size() - 1);
+ }
+
+ virtual void TearDown() {
+ delete _meta;
+ ASSERT_TRUE(boost::filesystem::remove_all("./meta"));
+ }
+
+private:
+ OlapMeta* _meta;
+ std::string _json_rowset_meta;
+};
+
+TEST_F(RowsetMetaManagerTest, TestSaveAndGetAndRemove) {
+ uint64_t rowset_id = 10000;
+ RowsetMeta rowset_meta;
+ rowset_meta.init_from_json(_json_rowset_meta);
+ ASSERT_EQ(rowset_meta.get_rowset_id(), rowset_id);
+ NewStatus status = RowsetMetaManager::save(_meta, rowset_id, &rowset_meta);
+ ASSERT_TRUE(status.ok());
+ std::string json_rowset_meta_read;
+ status = RowsetMetaManager::get_json_rowset_meta(_meta, rowset_id,
&json_rowset_meta_read);
+ ASSERT_TRUE(status.ok());
+ ASSERT_EQ(_json_rowset_meta, json_rowset_meta_read);
+ status = RowsetMetaManager::remove(_meta, rowset_id);
+ ASSERT_TRUE(status.ok());
+ RowsetMeta rowset_meta_read;
+ status = RowsetMetaManager::get_rowset_meta(_meta, rowset_id,
&rowset_meta_read);
+ ASSERT_TRUE(!status.ok());
+}
+
+TEST_F(RowsetMetaManagerTest, TestLoad) {
+ uint64_t rowset_id = 10000;
+ NewStatus status = RowsetMetaManager::load_json_rowset_meta(_meta,
rowset_meta_path);
+ ASSERT_TRUE(status.ok());
+ std::string json_rowset_meta_read;
+ status = RowsetMetaManager::get_json_rowset_meta(_meta, rowset_id,
&json_rowset_meta_read);
+ ASSERT_TRUE(status.ok());
+ ASSERT_EQ(_json_rowset_meta, json_rowset_meta_read);
+}
+
+} // namespace doris
+
+int main(int argc, char **argv) {
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
diff --git a/be/test/olap/test_data/rowset_meta.json
b/be/test/olap/test_data/rowset_meta.json
new file mode 100644
index 00000000..db962f23
--- /dev/null
+++ b/be/test/olap/test_data/rowset_meta.json
@@ -0,0 +1,12 @@
+{
+ "rowset_id": 10000,
+ "tablet_id": 20487,
+ "tablet_schema_hash": 1520686811,
+ "rowset_type": "ALPHA_ROWSET",
+ "rowset_state": "VISIBLE",
+ "start_version": 0,
+ "end_version": 1,
+ "row_number": 123456,
+ "total_disk_size": 100000,
+ "data_disk_size": 95000
+}
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index be1bc05b..3a45ed0b 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -79,12 +79,14 @@ message RowsetMetaPb {
optional int32 start_version = 6;
optional int32 end_version = 7;
optional int32 row_number = 8;
+ // unit in byte
optional int64 total_disk_size = 9;
optional int64 data_disk_size = 10;
optional int64 index_disk_size = 11;
// column min/max/null flag statistic info
repeated ColumnPruning column_statistics = 12;
optional DeleteConditionMessage delete_predicate = 13;
+ optional int64 txn_id = 14;
// spare field id 15-49 for future use
optional bytes extra_properties = 50;
}
diff --git a/run-ut.sh b/run-ut.sh
index bfe6d9bd..b787bf27 100755
--- a/run-ut.sh
+++ b/run-ut.sh
@@ -203,9 +203,10 @@ ${DORIS_TEST_BINARY_DIR}/olap/column_reader_test
${DORIS_TEST_BINARY_DIR}/olap/row_cursor_test
${DORIS_TEST_BINARY_DIR}/olap/skiplist_test
${DORIS_TEST_BINARY_DIR}/olap/serialize_test
-${DORIS_TEST_BINARY_DIR}/olap/olap_header_manager_test
+${DORIS_TEST_BINARY_DIR}/olap/tablet_meta_manager_test
${DORIS_TEST_BINARY_DIR}/olap/olap_meta_test
${DORIS_TEST_BINARY_DIR}/olap/delta_writer_test
+${DORIS_TEST_BINARY_DIR}/olap/olap/rowset/rowset_meta_manager_test
## Running agent unittest
# Prepare agent testdata
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
