This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 63c89df4741 [enhencement](RowsetWriter) Don't delete files when beta
rowset writer destructed (#25578)
63c89df4741 is described below
commit 63c89df474192df90d12fbb1f682146ade8fdb67
Author: AlexYue <[email protected]>
AuthorDate: Thu Oct 19 09:37:04 2023 +0800
[enhencement](RowsetWriter) Don't delete files when beta rowset writer
destructed (#25578)
---
be/src/olap/rowset/beta_rowset_writer.cpp | 8 ++++----
be/src/olap/rowset/vertical_beta_rowset_writer.cpp | 12 ++++++------
2 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp
b/be/src/olap/rowset/beta_rowset_writer.cpp
index 3346356096e..7a3eb1d9bd8 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -20,6 +20,7 @@
#include <assert.h>
// IWYU pragma: no_include <bthread/errno.h>
#include <errno.h> // IWYU pragma: keep
+#include <fmt/format.h>
#include <stdio.h>
#include <ctime> // time
@@ -33,7 +34,6 @@
#include "common/config.h"
#include "common/logging.h"
#include "common/status.h"
-#include "gutil/strings/substitute.h"
#include "io/fs/file_reader.h"
#include "io/fs/file_system.h"
#include "io/fs/file_writer.h"
@@ -85,8 +85,8 @@ BetaRowsetWriter::~BetaRowsetWriter() {
if (!_already_built) { // abnormal exit, remove all files generated
WARN_IF_ERROR(_segment_creator.close(),
"close segment creator failed"); // ensure all files are
closed
- auto fs = _rowset_meta->fs();
- if (fs->type() != io::FileSystemType::LOCAL) { // Remote fs will
delete them asynchronously
+ const auto& fs = _rowset_meta->fs();
+ if (!fs || !_rowset_meta->is_local()) { // Remote fs will delete them
asynchronously
return;
}
for (int i = _segment_start_id; i <
_segment_creator.next_segment_id(); ++i) {
@@ -96,7 +96,7 @@ BetaRowsetWriter::~BetaRowsetWriter() {
// will be cleaned up by the GC background. So here we only print
the error
// message when we encounter an error.
WARN_IF_ERROR(fs->delete_file(seg_path),
- strings::Substitute("Failed to delete file=$0",
seg_path));
+ fmt::format("Failed to delete file={}", seg_path));
}
}
}
diff --git a/be/src/olap/rowset/vertical_beta_rowset_writer.cpp
b/be/src/olap/rowset/vertical_beta_rowset_writer.cpp
index 53e20934c59..59c1afc9307 100644
--- a/be/src/olap/rowset/vertical_beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/vertical_beta_rowset_writer.cpp
@@ -17,6 +17,7 @@
#include "olap/rowset/vertical_beta_rowset_writer.h"
+#include <fmt/format.h>
#include <gen_cpp/olap_file.pb.h>
#include <algorithm>
@@ -29,7 +30,6 @@
// IWYU pragma: no_include <opentelemetry/common/threadlocal.h>
#include "common/compiler_util.h" // IWYU pragma: keep
#include "common/logging.h"
-#include "gutil/strings/substitute.h"
#include "io/fs/file_system.h"
#include "io/fs/file_writer.h"
#include "olap/rowset/beta_rowset.h"
@@ -44,8 +44,8 @@ using namespace ErrorCode;
VerticalBetaRowsetWriter::~VerticalBetaRowsetWriter() {
if (!_already_built) {
- auto fs = _rowset_meta->fs();
- if (!fs) {
+ const auto& fs = _rowset_meta->fs();
+ if (!fs || !_rowset_meta->is_local()) { // Remote fs will delete them
asynchronously
return;
}
for (auto& segment_writer : _segment_writers) {
@@ -56,8 +56,7 @@ VerticalBetaRowsetWriter::~VerticalBetaRowsetWriter() {
// Even if an error is encountered, these files that have not been
cleaned up
// will be cleaned up by the GC background. So here we only print
the error
// message when we encounter an error.
- WARN_IF_ERROR(fs->delete_file(path),
- strings::Substitute("Failed to delete file=$0",
path));
+ WARN_IF_ERROR(fs->delete_file(path), fmt::format("Failed to delete
file={}", path));
}
}
}
@@ -105,7 +104,8 @@ Status VerticalBetaRowsetWriter::add_columns(const
vectorized::Block* block,
RETURN_IF_ERROR(_segment_writers[_cur_writer_idx]->init(col_ids,
is_key));
}
// when splitting segment, need to make rows align between key columns
and value columns
- size_t start_offset = 0, limit = num_rows;
+ size_t start_offset = 0;
+ size_t limit = num_rows;
if (num_rows_written + num_rows >= num_rows_key_group &&
_cur_writer_idx < _segment_writers.size() - 1) {
RETURN_IF_ERROR(_segment_writers[_cur_writer_idx]->append_block(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
