yiguolei commented on code in PR #66472:
URL: https://github.com/apache/doris/pull/66472#discussion_r3763177929
##########
be/src/storage/schema.h:
##########
@@ -37,110 +39,181 @@
namespace doris {
-// The class is used to represent row's format in memory. Each row contains
-// multiple columns, some of which are key-columns (the rest are
value-columns).
-// NOTE: If both key-columns and value-columns exist, then the key-columns
-// must be placed before value-columns.
-//
-// To compare two rows whose schemas are different, but they are from the same
origin
-// we store all column schema maybe accessed here. And default access through
column id
-class Schema;
-using SchemaSPtr = std::shared_ptr<const Schema>;
-class Schema {
+class ReadSchema;
+class Block;
+using ReadSchemaSPtr = std::shared_ptr<ReadSchema>;
+class ReadSchema {
public:
- // All the columns of one table may exist in the columns param, but
col_ids is only a subset.
- Schema(const std::vector<TabletColumnPtr>& columns, const
std::vector<ColumnId>& col_ids) {
- size_t num_key_columns = 0;
- for (int i = 0; i < columns.size(); ++i) {
- if (columns[i]->is_key()) {
- ++num_key_columns;
- }
- if (columns[i]->name() == DELETE_SIGN) {
- _delete_sign_idx = i;
- }
- if (columns[i]->name() == BeConsts::ROWID_COL ||
- columns[i]->name().starts_with(BeConsts::GLOBAL_ROWID_COL)) {
- _rowid_col_idx = i;
- }
- if (columns[i]->name() == VERSION_COL) {
- _version_col_idx = i;
- }
- if (columns[i]->name() == BINLOG_TSO_COL) {
- _tso_col_idx = i;
- }
- if (columns[i]->name() == BINLOG_LSN_COL) {
- _lsn_col_idx = i;
- }
- if (columns[i]->name() == BINLOG_OP_COL) {
- _op_col_idx = i;
- }
- if (columns[i]->name() == COMMIT_TSO_COL) {
- _commit_tso_col_idx = i;
- }
- }
- _init(columns, col_ids, num_key_columns);
- }
+ using SequenceMap = std::unordered_map<ColumnId, std::vector<ColumnId>>;
- Schema(const Schema&);
- Schema& operator=(const Schema& other);
+ explicit ReadSchema(std::vector<TabletColumnPtr> columns);
- ~Schema();
+ explicit ReadSchema(const std::vector<TabletColumnPtr>& columns,
+ const std::vector<ColumnId>& cids);
- static DataTypePtr get_data_type_ptr(const TabletColumn& column);
+ // Initially every column is a caller-visible FE slot. Storage-only columns
+ // may be appended later without changing `num_block_columns()`.
+ explicit ReadSchema(std::vector<TabletColumnPtr> columns,
std::vector<DataTypePtr> read_types);
static IColumn::MutablePtr get_predicate_column_ptr(const DataTypePtr&
data_type,
const ReaderType
reader_type);
- const std::vector<TabletColumnPtr>& columns() const { return _cols; }
+ const std::vector<TabletColumnPtr>& columns() const { return
_read_columns; }
- const TabletColumn* column(ColumnId cid) const { return _cols[cid].get(); }
+ DataTypePtr data_type(size_t ordinal) const {
+ DCHECK_LT(ordinal, _read_types.size());
+ return _read_types[ordinal];
+ }
+
+ // Append a storage-only column without extending the caller-visible slot
prefix.
+ ColumnId append_column(TabletColumnPtr column) {
+ auto data_type = column->get_vec_type();
+ auto ordinal = cast_set<ColumnId>(_read_columns.size());
+ if (column->unique_id() >= 0) {
+ _uid_to_ordinal.emplace(column->unique_id(), ordinal);
+ }
+ _read_columns.emplace_back(std::move(column));
+ _read_types.emplace_back(std::move(data_type));
+ return ordinal;
+ }
+
+ // Create caller-visible Blocks from the FE-slot prefix.
+ Block create_read_block() const;
+
+ std::string read_columns_to_string() const;
+
+ Status init_sequence_map(const TabletSchema& tablet_schema);
+
+ const SequenceMap& sequence_map() const { return _sequence_map; }
+
+ ColumnId before_column_ordinal(ColumnId ordinal) const {
+ DCHECK_LT(ordinal, _before_column_ordinals.size());
+ return _before_column_ordinals[ordinal];
+ }
+
+ const TabletColumn* column(size_t ordinal) const { return
_read_columns[ordinal].get(); }
+
+ // Resolve by unique id when one exists. Name identity is only for legacy
+ // columns without a unique id.
+ int32_t ordinal_by_column(const TabletColumn& column) const {
+ if (column.unique_id() >= 0) {
+ return ordinal_by_uid(column.unique_id());
+ }
+ for (uint32_t ordinal = 0; ordinal < _read_columns.size(); ++ordinal) {
+ if (_read_columns[ordinal]->name() == column.name()) {
+ return static_cast<int32_t>(ordinal);
+ }
+ }
+ return -1;
+ }
+
+ // Total columns used inside storage, including appended storage-only
columns.
+ // Use this for per-column state and iteration over the complete
ReadSchema.
+ size_t num_read_columns() const { return _read_columns.size(); }
+
+ // Columns materialized in caller Blocks. They are the ReadSchema prefix
before
+ // appended storage-only columns; use this for Block layout and position
bounds.
+ size_t num_block_columns() const { return _num_block_columns; }
size_t num_key_columns() const { return _num_key_columns; }
- size_t num_columns() const { return _cols.size(); }
- size_t num_column_ids() const { return _col_ids.size(); }
- const std::vector<ColumnId>& column_ids() const { return _col_ids; }
- ColumnId column_id(size_t index) const { return _col_ids[index]; }
- int column_index(ColumnId cid) const { return _column_id_to_index[cid]; }
- const std::vector<int>& column_id_to_index() const { return
_column_id_to_index; }
- int32_t delete_sign_idx() const { return _delete_sign_idx; }
- bool has_sequence_col() const { return _has_sequence_col; }
- int32_t rowid_col_idx() const { return _rowid_col_idx; }
- int32_t version_col_idx() const { return _version_col_idx; }
- int32_t commit_tso_col_idx() const { return _commit_tso_col_idx; }
- int32_t tso_col_idx() const { return _tso_col_idx; }
- int32_t lsn_col_idx() const { return _lsn_col_idx; }
- int32_t op_col_idx() const { return _op_col_idx; }
- // Don't use.
- // TODO: memory size of Schema cannot be accurately tracked.
- // In some places, temporarily use num_columns() as Schema size.
- int64_t mem_size() const { return _mem_size; }
+ int32_t delete_sign_ordinal() const { return _delete_sign_ordinal; }
+ int32_t sequence_ordinal() const { return _sequence_ordinal; }
+ int32_t rowid_ordinal() const { return _rowid_ordinal; }
+ int32_t version_ordinal() const { return _version_ordinal; }
+ int32_t tso_ordinal() const { return _tso_ordinal; }
+ int32_t lsn_ordinal() const { return _lsn_ordinal; }
+ int32_t op_ordinal() const { return _op_ordinal; }
+ int32_t commit_tso_ordinal() const { return _commit_tso_ordinal; }
+
+ // -1 if no column with this unique id is present. Columns without a valid
+ // unique id (e.g. variant extracted subcolumns) are not in this map;
+ // resolve those with ordinal_by_column().
+ int32_t ordinal_by_uid(int32_t unique_id) const {
+ auto it = _uid_to_ordinal.find(unique_id);
+ return it == _uid_to_ordinal.end() ? -1 : it->second;
+ }
private:
- void _init(const std::vector<TabletColumnPtr>& cols, const
std::vector<ColumnId>& col_ids,
- size_t num_key_columns);
-
- void _copy_from(const Schema& other);
-
- // NOTE: The ColumnId here represents the sequential index number
(starting from 0) of
- // a column in current row, not the unique id-identifier of each column
- std::vector<ColumnId> _col_ids;
- // NOTE: _cols[cid] can only be accessed when the cid is
- // contained in _col_ids
- std::vector<TabletColumnPtr> _cols;
- // Tablet column id -> slot index in this Schema.
- std::vector<int> _column_id_to_index;
-
- size_t _num_key_columns;
- int32_t _delete_sign_idx = -1;
- bool _has_sequence_col = false;
- int32_t _rowid_col_idx = -1;
- int32_t _version_col_idx = -1;
- int32_t _commit_tso_col_idx = -1;
- int32_t _tso_col_idx = -1;
- int32_t _lsn_col_idx = -1;
- int32_t _op_col_idx = -1;
- int64_t _mem_size = 0;
+ void _init_read_types();
+ void _init_before_column_ordinals();
+
+ void _init_descriptors() {
+ DORIS_CHECK_LE(_num_block_columns, _read_columns.size());
+ DORIS_CHECK_EQ(_read_columns.size(), _read_types.size());
+ _num_key_columns = 0;
+ _delete_sign_ordinal = -1;
+ _sequence_ordinal = -1;
+ _rowid_ordinal = -1;
+ _version_ordinal = -1;
+ _tso_ordinal = -1;
+ _lsn_ordinal = -1;
+ _op_ordinal = -1;
+ _commit_tso_ordinal = -1;
+ _before_column_ordinals.clear();
+ _uid_to_ordinal.clear();
+ for (uint32_t i = 0; i < _read_columns.size(); ++i) {
+ const auto& col = *_read_columns[i];
+ if (col.unique_id() >= 0) {
+ _uid_to_ordinal.emplace(col.unique_id(), i);
+ }
+ }
+ for (uint32_t i = 0; i < _num_block_columns; ++i) {
+ const auto& col = *_read_columns[i];
Review Comment:
这里其实有一个问题。
我们过去的schema.h里他是输入的tablet schema的all columns，然后去计算的num key columns，并不是从read
columns 中计算的key columns
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
