This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
from e159e748df [chore](dependency) fix opentelemetry-cpp enable o3
optimization will core. (#10675)
add 331fa50501 [feature](cold-data) move cold data to object storage
without losing any feature(BE) (#10280)
No new revisions were added by this update.
Summary of changes:
be/CMakeLists.txt | 3 +-
be/src/agent/agent_server.cpp | 9 +-
be/src/agent/agent_server.h | 4 +-
be/src/agent/task_worker_pool.cpp | 259 ++++++++++---------
be/src/agent/task_worker_pool.h | 16 +-
be/src/agent/utils.cpp | 40 +++
be/src/agent/utils.h | 9 +
be/src/common/config.h | 10 +
be/src/env/env.h | 6 +
be/src/http/action/tablet_migration_action.cpp | 4 +-
be/src/io/CMakeLists.txt | 30 ++-
be/src/io/{file_writer.h => fs/file_reader.h} | 25 +-
be/src/io/fs/file_system.h | 84 ++++++
.../fs/file_system_map.cpp} | 31 ++-
.../brpc_service.h => io/fs/file_system_map.h} | 32 +--
.../bitmap_index_writer.h => io/fs/file_writer.h} | 44 ++--
be/src/io/fs/local_file_reader.cpp | 84 ++++++
.../fs/local_file_reader.h} | 40 ++-
be/src/io/fs/local_file_system.cpp | 171 +++++++++++++
be/src/io/fs/local_file_system.h | 58 +++++
be/src/io/fs/local_file_writer.cpp | 197 ++++++++++++++
.../fs/local_file_writer.h} | 43 +++-
be/src/{common/resource_tls.h => io/fs/path.h} | 16 +-
.../fs/remote_file_system.h} | 31 +--
be/src/io/fs/s3_common.h | 43 ++++
be/src/io/fs/s3_file_reader.cpp | 79 ++++++
.../array_utils.h => src/io/fs/s3_file_reader.h} | 40 +--
be/src/io/fs/s3_file_system.cpp | 221 ++++++++++++++++
be/src/io/fs/s3_file_system.h | 95 +++++++
be/src/olap/CMakeLists.txt | 3 +-
be/src/olap/base_compaction.cpp | 11 +-
be/src/olap/base_tablet.cpp | 22 +-
be/src/olap/base_tablet.h | 14 +-
be/src/olap/compaction.cpp | 4 +
be/src/olap/compaction.h | 3 +
be/src/olap/cumulative_compaction_policy.cpp | 2 +-
be/src/olap/data_dir.cpp | 282 ++++++++-------------
be/src/olap/data_dir.h | 31 ++-
be/src/olap/olap_common.h | 6 +-
be/src/olap/olap_define.h | 24 +-
be/src/olap/olap_server.cpp | 81 +++++-
be/src/olap/primary_key_index.cpp | 6 +-
be/src/olap/primary_key_index.h | 14 +-
be/src/olap/rowset/beta_rowset.cpp | 193 +++++++-------
be/src/olap/rowset/beta_rowset.h | 18 +-
be/src/olap/rowset/beta_rowset_reader.h | 3 +
be/src/olap/rowset/beta_rowset_writer.cpp | 113 +++------
be/src/olap/rowset/beta_rowset_writer.h | 15 +-
be/src/olap/rowset/rowset.cpp | 7 +-
be/src/olap/rowset/rowset.h | 18 +-
be/src/olap/rowset/rowset_factory.cpp | 5 +-
be/src/olap/rowset/rowset_factory.h | 2 +-
be/src/olap/rowset/rowset_meta.h | 41 ++-
be/src/olap/rowset/rowset_reader.h | 3 +
be/src/olap/rowset/rowset_writer.h | 2 -
be/src/olap/rowset/rowset_writer_context.h | 7 +-
.../olap/rowset/segment_v2/bitmap_index_reader.cpp | 4 +-
.../olap/rowset/segment_v2/bitmap_index_reader.h | 9 +-
.../olap/rowset/segment_v2/bitmap_index_writer.cpp | 6 +-
.../olap/rowset/segment_v2/bitmap_index_writer.h | 6 +-
.../segment_v2/bloom_filter_index_reader.cpp | 2 +-
.../rowset/segment_v2/bloom_filter_index_reader.h | 9 +-
.../segment_v2/bloom_filter_index_writer.cpp | 6 +-
.../rowset/segment_v2/bloom_filter_index_writer.h | 6 +-
be/src/olap/rowset/segment_v2/column_reader.cpp | 35 +--
be/src/olap/rowset/segment_v2/column_reader.h | 17 +-
be/src/olap/rowset/segment_v2/column_writer.cpp | 34 +--
be/src/olap/rowset/segment_v2/column_writer.h | 12 +-
.../rowset/segment_v2/indexed_column_reader.cpp | 26 +-
.../olap/rowset/segment_v2/indexed_column_reader.h | 24 +-
.../rowset/segment_v2/indexed_column_writer.cpp | 13 +-
.../olap/rowset/segment_v2/indexed_column_writer.h | 8 +-
.../olap/rowset/segment_v2/ordinal_page_index.cpp | 16 +-
be/src/olap/rowset/segment_v2/ordinal_page_index.h | 16 +-
be/src/olap/rowset/segment_v2/page_io.cpp | 18 +-
be/src/olap/rowset/segment_v2/page_io.h | 17 +-
be/src/olap/rowset/segment_v2/segment.cpp | 55 ++--
be/src/olap/rowset/segment_v2/segment.h | 10 +-
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 9 +-
be/src/olap/rowset/segment_v2/segment_iterator.h | 3 +-
be/src/olap/rowset/segment_v2/segment_writer.cpp | 22 +-
be/src/olap/rowset/segment_v2/segment_writer.h | 12 +-
be/src/olap/rowset/segment_v2/zone_map_index.cpp | 7 +-
be/src/olap/rowset/segment_v2/zone_map_index.h | 21 +-
be/src/olap/schema_change.cpp | 40 ++-
be/src/olap/schema_change.h | 6 +-
be/src/olap/snapshot_manager.cpp | 103 ++++----
be/src/olap/snapshot_manager.h | 12 +-
be/src/olap/storage_engine.cpp | 103 +++-----
be/src/olap/storage_engine.h | 14 +-
be/src/olap/storage_migration_v2.cpp | 2 +
be/src/olap/storage_policy_mgr.cpp | 91 +++++++
be/src/olap/storage_policy_mgr.h | 79 ++++++
be/src/olap/tablet.cpp | 210 +++++++++++++--
be/src/olap/tablet.h | 32 ++-
be/src/olap/tablet_manager.cpp | 179 ++++++-------
be/src/olap/tablet_manager.h | 1 +
be/src/olap/tablet_meta.cpp | 25 +-
be/src/olap/tablet_meta.h | 51 +++-
be/src/olap/task/engine_batch_load_task.cpp | 2 +-
be/src/olap/task/engine_clone_task.cpp | 37 ++-
be/src/olap/task/engine_clone_task.h | 2 +-
be/src/olap/task/engine_storage_migration_task.cpp | 9 +-
be/src/runtime/exec_env.h | 3 +
be/src/runtime/exec_env_init.cpp | 2 +
be/src/runtime/load_path_mgr.cpp | 7 +-
be/src/runtime/snapshot_loader.cpp | 22 +-
be/src/service/backend_service.cpp | 8 +-
be/src/service/http_service.cpp | 3 -
be/src/util/doris_metrics.cpp | 31 +++
be/src/util/doris_metrics.h | 17 ++
be/src/util/file_cache.cpp | 8 +-
be/src/util/file_cache.h | 3 +-
be/src/util/metrics.h | 3 +-
be/src/util/s3_uri.cpp | 2 +-
be/src/util/s3_util.cpp | 1 +
be/test/CMakeLists.txt | 2 +
be/test/olap/delete_handler_test.cpp | 12 +-
be/test/olap/delta_writer_test.cpp | 2 +-
.../olap/engine_storage_migration_task_test.cpp | 2 +-
be/test/olap/memtable_flush_executor_test.cpp | 2 +-
be/test/olap/primary_key_index_test.cpp | 17 +-
be/test/olap/rowset/beta_rowset_test.cpp | 156 ++++++++++--
.../olap/rowset/segment_v2/bitmap_index_test.cpp | 37 +--
.../bloom_filter_index_reader_writer_test.cpp | 39 +--
.../segment_v2/column_reader_writer_test.cpp | 62 ++---
.../rowset/segment_v2/ordinal_page_index_test.cpp | 21 +-
be/test/olap/rowset/segment_v2/segment_test.cpp | 123 ++++-----
.../olap/rowset/segment_v2/zone_map_index_test.cpp | 42 +--
be/test/olap/tablet_clone_test.cpp | 221 ++++++++++++++++
...tion_task_test.cpp => tablet_cooldown_test.cpp} | 217 ++++++----------
be/test/olap/tablet_mgr_test.cpp | 8 +-
be/test/olap/tablet_test.cpp | 140 ++++++++++
be/test/olap/test_data/header_without_inc_rs.txt | 3 +-
be/test/olap/txn_manager_test.cpp | 10 +-
be/test/runtime/array_test.cpp | 54 ++--
be/test/tools/benchmark_tool.cpp | 25 +-
.../apache/doris/service/FrontendServiceImpl.java | 5 +
gensrc/proto/olap_file.proto | 7 +
gensrc/thrift/AgentService.thrift | 18 ++
gensrc/thrift/BackendService.thrift | 2 +
gensrc/thrift/FrontendService.thrift | 2 +
gensrc/thrift/MasterService.thrift | 5 +
gensrc/thrift/Types.thrift | 3 +-
144 files changed, 3750 insertions(+), 1672 deletions(-)
copy be/src/io/{file_writer.h => fs/file_reader.h} (69%)
create mode 100644 be/src/io/fs/file_system.h
copy be/src/{runtime/record_batch_queue.cpp => io/fs/file_system_map.cpp} (63%)
copy be/src/{service/brpc_service.h => io/fs/file_system_map.h} (64%)
copy be/src/{olap/rowset/segment_v2/bitmap_index_writer.h =>
io/fs/file_writer.h} (51%)
create mode 100644 be/src/io/fs/local_file_reader.cpp
copy be/src/{util/ref_count_closure.h => io/fs/local_file_reader.h} (56%)
create mode 100644 be/src/io/fs/local_file_system.cpp
create mode 100644 be/src/io/fs/local_file_system.h
create mode 100644 be/src/io/fs/local_file_writer.cpp
copy be/src/{util/dummy_runtime_profile.h => io/fs/local_file_writer.h} (57%)
copy be/src/{common/resource_tls.h => io/fs/path.h} (82%)
copy be/src/{exec/schema_scanner/schema_charsets_scanner.h =>
io/fs/remote_file_system.h} (57%)
create mode 100644 be/src/io/fs/s3_common.h
create mode 100644 be/src/io/fs/s3_file_reader.cpp
copy be/{test/testutil/array_utils.h => src/io/fs/s3_file_reader.h} (58%)
create mode 100644 be/src/io/fs/s3_file_system.cpp
create mode 100644 be/src/io/fs/s3_file_system.h
create mode 100644 be/src/olap/storage_policy_mgr.cpp
create mode 100644 be/src/olap/storage_policy_mgr.h
create mode 100644 be/test/olap/tablet_clone_test.cpp
copy be/test/olap/{engine_storage_migration_task_test.cpp =>
tablet_cooldown_test.cpp} (51%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
