This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 340a37116f2 [cleanup](load) remove unused dynamic schema code (#25932)
340a37116f2 is described below
commit 340a37116f268a7dd444a87b077f6ad205bfcac2
Author: Kaijie Chen <[email protected]>
AuthorDate: Fri Oct 27 16:21:40 2023 +0800
[cleanup](load) remove unused dynamic schema code (#25932)
---
be/src/olap/rowset/beta_rowset_writer.cpp | 73 ----------------------------
be/src/olap/rowset/beta_rowset_writer.h | 10 ----
be/src/olap/rowset/beta_rowset_writer_v2.cpp | 6 ---
be/src/olap/rowset/beta_rowset_writer_v2.h | 4 --
4 files changed, 93 deletions(-)
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp
b/be/src/olap/rowset/beta_rowset_writer.cpp
index 7a3eb1d9bd8..c4a8dbb568a 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -748,77 +748,4 @@ Status
BetaRowsetWriter::flush_segment_writer_for_segcompaction(
return Status::OK();
}
-Status BetaRowsetWriter::_unfold_variant_column(vectorized::Block& block,
- TabletSchemaSPtr&
flush_schema) {
- if (block.rows() == 0) {
- return Status::OK();
- }
-
- // Sanitize block to match exactly from the same type of frontend meta
- vectorized::schema_util::FullBaseSchemaView schema_view;
- schema_view.table_id = _context.tablet_schema->table_id();
- vectorized::ColumnWithTypeAndName* variant_column =
- block.try_get_by_name(BeConsts::DYNAMIC_COLUMN_NAME);
- if (!variant_column) {
- return Status::OK();
- }
- auto base_column = variant_column->column;
- vectorized::ColumnObject& object_column =
-
assert_cast<vectorized::ColumnObject&>(base_column->assume_mutable_ref());
- if (object_column.empty()) {
- block.erase(BeConsts::DYNAMIC_COLUMN_NAME);
- return Status::OK();
- }
- object_column.finalize();
- // Has extended columns
-
RETURN_IF_ERROR(vectorized::schema_util::send_fetch_full_base_schema_view_rpc(&schema_view));
- // Dynamic Block consists of two parts, dynamic part of columns and static
part of columns
- // static dynamic
- // | ----- | ------- |
- // The static ones are original _tablet_schame columns
- flush_schema = std::make_shared<TabletSchema>(*_context.tablet_schema);
- vectorized::Block flush_block(std::move(block));
- // The dynamic ones are auto generated and extended, append them the the
orig_block
- for (auto& entry : object_column.get_subcolumns()) {
- const std::string& column_name = entry->path.get_path();
- auto column_iter = schema_view.column_name_to_column.find(column_name);
- if (UNLIKELY(column_iter == schema_view.column_name_to_column.end())) {
- // Column maybe dropped by light weight schema change DDL
- continue;
- }
- TabletColumn column(column_iter->second);
- auto data_type =
vectorized::DataTypeFactory::instance().create_data_type(
- column, column.is_nullable());
- // Dynamic generated columns does not appear in original tablet schema
- if (_context.tablet_schema->field_index(column.name()) < 0) {
- flush_schema->append_column(column);
- flush_block.insert({data_type->create_column(), data_type,
column.name()});
- }
- }
-
- // Ensure column are all present at this schema version.Otherwise there
will be some senario:
- // Load1 -> version(10) with schema [a, b, c, d, e], d & e is new added
columns and schema version became 10
- // Load2 -> version(10) with schema [a, b, c] and has no extended columns
and fetched the schema at version 10
- // Load2 will persist meta with [a, b, c] but Load1 will persist meta
with [a, b, c, d, e]
- // So we should make sure that rowset at the same schema version alawys
contain the same size of columns.
- // so that all columns at schema_version is in either
_context.tablet_schema or schema_change_recorder
- for (const auto& [name, column] : schema_view.column_name_to_column) {
- if (_context.tablet_schema->field_index(name) == -1) {
- const auto& tcolumn = schema_view.column_name_to_column[name];
- TabletColumn new_column(tcolumn);
- _context.schema_change_recorder->add_extended_columns(column,
-
schema_view.schema_version);
- }
- }
-
- // Last schema alignment before flush to disk, due to the schema maybe
variant before this procedure
- // Eg. add columnA(INT) -> drop ColumnA -> add ColumnA(Double), then
columnA could be type of `Double`,
- // unfold will cast to Double type
- RETURN_IF_ERROR(vectorized::schema_util::unfold_object(
- flush_block.get_position_by_name(BeConsts::DYNAMIC_COLUMN_NAME),
flush_block, true));
- flush_block.erase(BeConsts::DYNAMIC_COLUMN_NAME);
- block.swap(flush_block);
- return Status::OK();
-}
-
} // namespace doris
diff --git a/be/src/olap/rowset/beta_rowset_writer.h
b/be/src/olap/rowset/beta_rowset_writer.h
index 5d1b3075933..9c615048a7f 100644
--- a/be/src/olap/rowset/beta_rowset_writer.h
+++ b/be/src/olap/rowset/beta_rowset_writer.h
@@ -58,9 +58,6 @@ class SegmentWriter;
using SegCompactionCandidates = std::vector<segment_v2::SegmentSharedPtr>;
using SegCompactionCandidatesSharedPtr =
std::shared_ptr<SegCompactionCandidates>;
-namespace vectorized::schema_util {
-class LocalSchemaChangeRecorder;
-}
class BetaRowsetWriter : public RowsetWriter {
friend class SegcompactionWorker;
@@ -166,13 +163,6 @@ private:
Status _rename_compacted_segment_plain(uint64_t seg_id);
Status _rename_compacted_indices(int64_t begin, int64_t end, uint64_t
seg_id);
- // Unfold variant column to Block
- // Eg. [A | B | C | (D, E, F)]
- // After unfold block structure changed to -> [A | B | C | D | E | F]
- // The expanded D, E, F is dynamic part of the block
- // The flushed Block columns should match exactly from the same type of
frontend meta
- Status _unfold_variant_column(vectorized::Block& block, TabletSchemaSPtr&
flush_schema);
-
// build a tmp rowset for load segment to calc delete_bitmap
// for this segment
RowsetSharedPtr _build_tmp();
diff --git a/be/src/olap/rowset/beta_rowset_writer_v2.cpp
b/be/src/olap/rowset/beta_rowset_writer_v2.cpp
index 6d030308805..abd7af2aaba 100644
--- a/be/src/olap/rowset/beta_rowset_writer_v2.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer_v2.cpp
@@ -103,12 +103,6 @@ Status
BetaRowsetWriterV2::flush_memtable(vectorized::Block* block, int32_t segm
}
TabletSchemaSPtr flush_schema;
- /* TODO: support dynamic schema
- if (_context.tablet_schema->is_dynamic_schema()) {
- // Unfold variant column
- RETURN_IF_ERROR(_unfold_variant_column(*block, flush_schema));
- }
- */
{
SCOPED_RAW_TIMER(&_segment_writer_ns);
RETURN_IF_ERROR(
diff --git a/be/src/olap/rowset/beta_rowset_writer_v2.h
b/be/src/olap/rowset/beta_rowset_writer_v2.h
index 01f9cef3f1c..d35f4b2486f 100644
--- a/be/src/olap/rowset/beta_rowset_writer_v2.h
+++ b/be/src/olap/rowset/beta_rowset_writer_v2.h
@@ -56,10 +56,6 @@ namespace segment_v2 {
class SegmentWriter;
} // namespace segment_v2
-namespace vectorized::schema_util {
-class LocalSchemaChangeRecorder;
-}
-
class LoadStreamStub;
class BetaRowsetWriterV2 : public RowsetWriter {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
