chaoyli closed pull request #399: Add Rowset Interface URL: https://github.com/apache/incubator-doris/pull/399
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/be/src/olap/rowset.h b/be/src/olap/rowset.h new file mode 100644 index 00000000..cb8dd07f --- /dev/null +++ b/be/src/olap/rowset.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef DORIS_BE_SRC_OLAP_ROWSET_H +#define DORIS_BE_SRC_OLAP_ROWSET_H + +#include "olap/new_status.h" +#include "olap/rowset_reader.h" +#include "olap/rowset_builder.h" + +#include <memory> + +namespace doris { + +class Rowset { +public: + NewStatus init(const RowsetMeta& rowset_meta, const DataDir* dir) = 0; + + std::unique_ptr<RowsetReader> create_reader() = 0; + + NewStatus copy(RowsetBuilder* dest_rowset_builder) = 0; + + NewStatus delete() = 0; + +private: + RowsetMeta _rowset_meta; +}; + +} + +#endif // DORIS_BE_SRC_OLAP_ROWSET_H diff --git a/be/src/olap/rowset_builder.h b/be/src/olap/rowset_builder.h new file mode 100644 index 00000000..9434efaa --- /dev/null +++ b/be/src/olap/rowset_builder.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef DORIS_BE_SRC_OLAP_ROWSET_BUILDER_H +#define DORIS_BE_SRC_OLAP_ROWSET_BUILDER_H + +#include "rowset/rowset.h" +#include "olap/new_status.h" +#include "olap/schema.h" +#include "olap/row_block.h" + +namespace doris { + +class RowsetBuilder { +public: + NewStatus init(std::string rowset_id, const std::string& rowset_path_prefix, Schema* schema) = 0; + + // add a row block to rowset + NewStatus add_row_block(RowBlock* row_block) = 0; + + // this is a temp api + // it is used to get rewritten path for writing rowset data + NewStatus generate_written_path(const std::string& src_path, std::string* dest_path) = 0; + + // get a rowset + NewStatus build(Rowset* rowset) = 0; +}; + +} + +#endif // DORIS_BE_SRC_OLAP_ROWSET_BUILDER_H diff --git a/be/src/olap/rowset_meta.h b/be/src/olap/rowset_meta.h new file mode 100644 index 00000000..022ea2a1 --- /dev/null +++ b/be/src/olap/rowset_meta.h @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef DORIS_BE_SRC_OLAP_ROWSET_META_H +#define DORIS_BE_SRC_OLAP_ROWSET_META_H + +#include "gen_cpp/olap_file.pb.h" + +#include <memory> +#include <vector> + +namespace doris { + +class RowsetMeta { +public: + virtual void init(const RowsetMetaPb& rowset_meta) { + _rowset_meta = rowset_meta; + } + + virtual bool deserialize_extra_properties() = 0; + + virtual void get_rowset_meta_pb(RowsetMetaPb* rowset_meta) { + rowset_meta = &_rowset_meta; + } + virtual int64_t get_rowset_id() { + return _rowset_meta.rowset_id(); + } + + virtual void set_rowset_id(int64_t rowset_id) { + _rowset_meta.set_rowset_id(rowset_id); + } + + virtual int64_t get_version() { + return _rowset_meta->version(); + } + + virtual void set_version(int64_t version) { + _rowset_meta->set_version(version); + } + + virtual int64_t get_tablet_id() { + return _rowset_meta->tablet_id(); + } + + virtual void set_tablet_id(int64_t tablet_id) { + _rowset_meta->set_tablet_id(tablet_id); + } + + virtual int32_t get_tablet_schema_hash() { + return _rowset_meta->tablet_schema_hash(); + } + + virtual void set_tablet_schema_hash(int64_t tablet_schema_hash) { + _rowset_meta->set_tablet_schema_hash(tablet_schema_hash); + } + + virtual RowsetType get_rowset_type() { + return _rowset_meta->rowset_type(); + } + + virtual void set_rowset_type(RowsetType rowset_type) { + _rowset_meta->set_rowset_type(rowset_type); + } + + virtual RowsetState get_rowset_state() { + return _rowset_meta->rowset_state(); + } + + virtual void set_rowset_state(RowsetState rowset_state) { + _rowset_meta->set_rowset_state(rowset_state); + } + + virtual int get_start_version() { + return _rowset_meta->start_version(); + } + + virtual void set_start_version(int start_version) { + _rowset_meta->set_start_version(start_version); + } + + virtual int get_end_version() { + return _rowset_meta->end_version(); + } + + virtual void set_end_version(int end_version) { + _rowset_meta->set_end_version(end_version); + } + + virtual int get_row_number() { + return _rowset_meta->row_number(); + } + + virtual void set_row_number(int row_number) { + _rowset_meta->set_row_number(row_number); + } + + virtual int get_total_disk_size() { + return _rowset_meta->total_disk_size(); + } + + virtual void set_total_disk_size(int total_disk_size) { + _rowset_meta->set_total_disk_size(total_disk_size); + } + + virtual int get_data_disk_size() { + return _rowset_meta->data_disk_size(); + } + + virtual void set_data_disk_size(int data_disk_size) { + _rowset_meta->set_data_disk_size(data_disk_size); + } + + virtual int get_index_disk_size() { + return _rowset_meta->index_disk_size(); + } + + virtual void set_index_disk_size(int index_disk_size) { + _rowset_meta->set_index_disk_size(index_disk_size); + } + + virtual void get_column_statistics(std::vector<ColumnPruning>* column_statistics) { + *column_statistics = _rowset_meta->column_statistics(); + } + + virtual void set_column_statistics(std::vector<ColumnPruning> column_statistics) { + std::vector<ColumnPruning>* new_column_statistics = _rowset_meta.mutable_column_pruning(); + *new_column_statistics = column_statistics; + } + + virtual DeleteConditionMessage get_delete_predicate() { + return _rowset_meta->delete_predicate(); + } + + virtual void set_delete_predicate(DeleteConditionMessage delete_predicate) { + _rowset_meta->set_delete_predicate(delete_predicate); + } + +private: + RowsetMetaPb _rowset_meta; +}; + +} + +#endif // DORIS_BE_SRC_OLAP_ROWSET_META_H diff --git a/be/src/olap/rowset_reader.h b/be/src/olap/rowset_reader.h new file mode 100644 index 00000000..4478851d --- /dev/null +++ b/be/src/olap/rowset_reader.h @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef DORIS_BE_SRC_OLAP_ROWSET_READER_H +#define DORIS_BE_SRC_OLAP_ROWSET_READER_H + +#include "olap/new_status.h" +#include "olap/schema.h" +#include "olap/column_predicate.h" +#include "olap/row_cursor.h" +#include "olap/row_block.h" + +#include <memory> +#include <unordered_map> + +namespace doris { + +struct ReadContext { + const Schema* projection; + std::unordered_map<std::string, ColumnPredicate> predicates; // column name -> column predicate + const RowCursor* lower_bound_key; + const RowCursor* exclusive_upper_bound_key; +}; + +class RowsetReader { +public: + // reader init + // check whether this rowset can be filtered + NewStatus init(ReadContext* read_context) = 0; + + // check whether rowset has more data + bool has_next() = 0; + + // read next block data + NewStatus next_block(RowBlock* row_block) = 0; + + // close reader + void close() = 0; +}; + +} + +#endif // DORIS_BE_SRC_OLAP_ROWSET_READER_H diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index f58c00ff..be1bc05b 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -58,6 +58,37 @@ message PDelta { optional DeleteConditionMessage delete_condition = 6; } +enum RowsetType { + ALPHA_ROWSET = 0; // doris原有的列存格式 + BETA_ROWSET = 1; // 新列存 +} + +enum RowsetState { + PREPARING = 0; // 表示正在写入Rowset + COMMITTED = 1; // 表示rowset 写入完成，但是用户还不可见；这个状态下的rowset，BE不能自行判断是否删除，必须由FE的指令 + VISIBLE = 2; // 表示rowset 已经对用户可见 +} + +message RowsetMetaPb { + required int64 rowset_id = 1; + optional int64 tablet_id = 2; + optional int32 tablet_schema_hash = 3; + optional RowsetType rowset_type = 4; + optional RowsetState rowset_state = 5; + // Rowset data version range + optional int32 start_version = 6; + optional int32 end_version = 7; + optional int32 row_number = 8; + optional int64 total_disk_size = 9; + optional int64 data_disk_size = 10; + optional int64 index_disk_size = 11; + // column min/max/null flag statistic info + repeated ColumnPruning column_statistics = 12; + optional DeleteConditionMessage delete_predicate = 13; + // spare field id 15-49 for future use + optional bytes extra_properties = 50; +} + message PSegmentGroup { required int32 segment_group_id = 1; required int32 num_segments = 2; ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
