This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
from afe6e656f43 [cast](number) add line processing function for casting to
numbers (#53869)
add 0314f519825 [feature](restore) Add cloud restore job (#47300)
No new revisions were added by this update.
Summary of changes:
be/src/agent/agent_server.cpp | 12 +
be/src/agent/task_worker_pool.cpp | 108 ++++
be/src/agent/task_worker_pool.h | 6 +
be/src/cloud/cloud_meta_mgr.cpp | 34 ++
be/src/cloud/cloud_meta_mgr.h | 16 +
be/src/cloud/cloud_snapshot_loader.cpp | 244 ++++++++
be/src/cloud/cloud_snapshot_loader.h | 54 ++
be/src/cloud/cloud_snapshot_mgr.cpp | 270 +++++++++
be/src/cloud/cloud_snapshot_mgr.h | 94 ++++
be/src/cloud/cloud_storage_engine.cpp | 3 +
be/src/cloud/cloud_storage_engine.h | 4 +
be/src/olap/file_header.h | 107 ++++
be/src/olap/tablet_meta.cpp | 25 +-
be/src/olap/tablet_meta.h | 2 +
be/src/runtime/snapshot_loader.cpp | 70 ++-
be/src/runtime/snapshot_loader.h | 73 ++-
be/test/cloud/cloud_snapshot_mgr_test.cpp | 108 ++++
be/test/olap/tablet_meta_test.cpp | 118 ++++
cloud/src/common/bvars.cpp | 4 +
cloud/src/common/bvars.h | 4 +
cloud/src/common/config.h | 1 +
cloud/src/meta-service/meta_service.cpp | 619 +++++++++++++++++++++
cloud/src/meta-service/meta_service.h | 30 +
cloud/src/meta-service/meta_service_partition.cpp | 42 +-
cloud/src/meta-store/keys.cpp | 20 +-
cloud/src/meta-store/keys.h | 13 +
cloud/src/recycler/recycler.cpp | 276 ++++++++-
cloud/src/recycler/recycler.h | 6 +
cloud/test/keys_test.cpp | 72 +++
cloud/test/meta_service_test.cpp | 381 +++++++++++++
cloud/test/recycler_test.cpp | 144 +++++
.../main/java/org/apache/doris/common/Config.java | 8 +
.../org/apache/doris/alter/CloudRollupJobV2.java | 2 +-
.../apache/doris/alter/CloudSchemaChangeJobV2.java | 2 +-
.../java/org/apache/doris/backup/AbstractJob.java | 4 +
.../org/apache/doris/backup/BackupHandler.java | 34 +-
.../apache/doris/backup/RestoreFileMapping.java | 2 +-
.../java/org/apache/doris/backup/RestoreJob.java | 367 ++++++------
.../java/org/apache/doris/backup/SnapshotInfo.java | 19 +
.../main/java/org/apache/doris/catalog/Env.java | 4 +-
.../java/org/apache/doris/catalog/OlapTable.java | 20 +-
.../org/apache/doris/catalog/TableProperty.java | 8 +
.../apache/doris/cloud/backup/CloudRestoreJob.java | 539 ++++++++++++++++++
.../apache/doris/cloud/common/util/CopyUtil.java | 54 ++
.../cloud/datasource/CloudInternalCatalog.java | 27 +-
.../plans/commands/DropRepositoryCommand.java | 6 -
.../trees/plans/commands/RestoreCommand.java | 34 +-
.../org/apache/doris/persist/gson/GsonUtils.java | 4 +-
.../main/java/org/apache/doris/qe/DdlExecutor.java | 5 -
.../java/org/apache/doris/task/DownloadTask.java | 7 +-
.../org/apache/doris/task/ReleaseSnapshotTask.java | 1 +
.../org/apache/doris/backup/BackupHandlerTest.java | 2 +-
.../doris/cloud/backup/CloudRestoreJobTest.java | 260 +++++++++
.../doris/cloud/common/util/CopyUtilTest.java | 187 +++++++
gensrc/proto/cloud.proto | 34 ++
gensrc/thrift/AgentService.thrift | 2 +
.../restore_p0/test_unique_validate_restore.out | Bin 0 -> 149 bytes
.../restore_p0/test_validate_restore_alias.out | Bin 0 -> 607 bytes
.../test_validate_restore_bloom_filter.out | Bin 0 -> 245 bytes
...t_validate_restore_dup_without_default_keys.out | Bin 0 -> 221 bytes
.../restore_p0/test_validate_restore_exclude.out | Bin 0 -> 187 bytes
.../restore_p0/test_validate_restore_partition.out | Bin 0 -> 239 bytes
.../test_validate_restore_with_view.out} | Bin
.../org/apache/doris/regression/Config.groovy | 11 +
.../apache/doris/regression/ConfigOptions.groovy | 9 +
.../apache/doris/regression/suite/Syncer.groovy | 18 +-
.../doris/regression/util/RepoValidateUtils.groovy | 116 ++++
.../test_duplicate_validate_restore.groovy | 88 +++
.../restore_p0/test_unique_validate_restore.groovy | 107 ++++
.../suites/restore_p0/test_validate_restore.groovy | 60 ++
.../restore_p0/test_validate_restore_alias.groovy | 82 +++
...t_validate_restore_backup_temp_partition.groovy | 63 +++
.../test_validate_restore_bloom_filter.groovy | 69 +++
.../restore_p0/test_validate_restore_db.groovy | 69 +++
...validate_restore_diff_repo_same_snapshot.groovy | 87 +++
...alidate_restore_dup_without_default_keys.groovy | 70 +++
...e_restore_dynamic_partition_reserve_true.groovy | 70 +++
.../test_validate_restore_exclude.groovy | 95 ++++
.../test_validate_restore_inverted_idx.groovy | 164 ++++++
.../test_validate_restore_inverted_index.groovy | 76 +++
...alidate_restore_mix_exists_and_new_table.groovy | 96 ++++
.../restore_p0/test_validate_restore_mtmv.groovy | 71 +++
.../test_validate_restore_multi_tables.groovy | 75 +++
.../restore_p0/test_validate_restore_mv.groovy | 81 +++
...test_validate_restore_ngram_bloom_filter.groovy | 154 +++++
.../test_validate_restore_partition.groovy | 80 +++
..._restore_reserve_dynamic_partition_false.groovy | 66 +++
.../test_validate_restore_table_auto_bucket.groovy | 66 +++
.../test_validate_restore_to_new_table.groovy | 83 +++
.../test_validate_restore_version_revert.groovy | 63 +++
...alidate_restore_with_two_phase_fetch_opt.groovy | 63 +++
.../test_validate_restore_with_view.groovy | 112 ++++
92 files changed, 6558 insertions(+), 298 deletions(-)
create mode 100644 be/src/cloud/cloud_snapshot_loader.cpp
create mode 100644 be/src/cloud/cloud_snapshot_loader.h
create mode 100644 be/src/cloud/cloud_snapshot_mgr.cpp
create mode 100644 be/src/cloud/cloud_snapshot_mgr.h
create mode 100644 be/test/cloud/cloud_snapshot_mgr_test.cpp
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CopyUtil.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/cloud/backup/CloudRestoreJobTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/cloud/common/util/CopyUtilTest.java
create mode 100644
regression-test/data/restore_p0/test_unique_validate_restore.out
create mode 100644
regression-test/data/restore_p0/test_validate_restore_alias.out
create mode 100644
regression-test/data/restore_p0/test_validate_restore_bloom_filter.out
create mode 100644
regression-test/data/restore_p0/test_validate_restore_dup_without_default_keys.out
create mode 100644
regression-test/data/restore_p0/test_validate_restore_exclude.out
create mode 100644
regression-test/data/restore_p0/test_validate_restore_partition.out
copy regression-test/data/{backup_restore/test_backup_restore_with_view.out =>
restore_p0/test_validate_restore_with_view.out} (100%)
create mode 100644
regression-test/framework/src/main/groovy/org/apache/doris/regression/util/RepoValidateUtils.groovy
create mode 100644
regression-test/suites/restore_p0/test_duplicate_validate_restore.groovy
create mode 100644
regression-test/suites/restore_p0/test_unique_validate_restore.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_alias.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_backup_temp_partition.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_bloom_filter.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_db.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_diff_repo_same_snapshot.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_dup_without_default_keys.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_dynamic_partition_reserve_true.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_exclude.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_inverted_idx.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_inverted_index.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_mix_exists_and_new_table.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_mtmv.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_multi_tables.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_mv.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_ngram_bloom_filter.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_partition.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_reserve_dynamic_partition_false.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_table_auto_bucket.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_to_new_table.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_version_revert.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_with_two_phase_fetch_opt.groovy
create mode 100644
regression-test/suites/restore_p0/test_validate_restore_with_view.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
