This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 3f3028fc184 branch-3.1: [opt](inverted index) Optimize priv_getFN
method with buffer safety checks #52923 (#53981)
3f3028fc184 is described below
commit 3f3028fc184ad773638e72ff7363693d7f7b5812
Author: zzzxl <[email protected]>
AuthorDate: Tue Jul 29 15:09:17 2025 +0800
branch-3.1: [opt](inverted index) Optimize priv_getFN method with buffer
safety checks #52923 (#53981)
picked from #52923
---
.../segment_v2/inverted_index_fs_directory.cpp | 45 ++++++++--------------
.../segment_v2/inverted_index_fs_directory.h | 4 +-
.../inverted_index_fs_directory_test.cpp | 21 ++++++++++
3 files changed, 41 insertions(+), 29 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp
index 7bdca1941fa..430c946f8b6 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp
@@ -501,11 +501,10 @@ void DorisFSDirectory::init(const io::FileSystemSPtr& fs,
const char* path,
lucene::store::Directory::setLockFactory(lock_factory);
}
-void DorisFSDirectory::priv_getFN(char* buffer, const char* name) const {
- buffer[0] = 0;
- strcpy(buffer, directory.c_str());
- strcat(buffer, PATH_DELIMITERA);
- strcat(buffer, name);
+std::string DorisFSDirectory::priv_getFN(const std::string& name) const {
+ doris::io::Path path(directory);
+ path /= name;
+ return path.native();
}
DorisFSDirectory::~DorisFSDirectory() = default;
@@ -519,8 +518,7 @@ const char* DorisFSDirectory::getObjectName() const {
bool DorisFSDirectory::list(std::vector<std::string>* names) const {
CND_PRECONDITION(!directory.empty(), "directory is not open");
- char fl[CL_MAX_DIR];
- priv_getFN(fl, "");
+ std::string fl = priv_getFN("");
std::vector<io::FileInfo> files;
bool exists;
auto st = _fs->list(fl, true, &files, &exists);
@@ -541,8 +539,7 @@ bool DorisFSDirectory::list(std::vector<std::string>*
names) const {
bool DorisFSDirectory::fileExists(const char* name) const {
CND_PRECONDITION(directory[0] != 0, "directory is not open");
- char fl[CL_MAX_DIR];
- priv_getFN(fl, name);
+ std::string fl = priv_getFN(name);
bool exists = false;
auto st = _fs->exists(fl, &exists);
DBUG_EXECUTE_IF("DorisFSDirectory::fileExists_status_is_not_ok", {
@@ -560,9 +557,8 @@ const std::string& DorisFSDirectory::getDirName() const {
int64_t DorisFSDirectory::fileModified(const char* name) const {
CND_PRECONDITION(directory[0] != 0, "directory is not open");
struct stat buf;
- char buffer[CL_MAX_DIR];
- priv_getFN(buffer, name);
- if (stat(buffer, &buf) == -1) {
+ std::string buffer = priv_getFN(name);
+ if (stat(buffer.c_str(), &buf) == -1) {
return 0;
} else {
return buf.st_mtime;
@@ -585,8 +581,7 @@ void DorisFSDirectory::touchFile(const char* name) {
int64_t DorisFSDirectory::fileLength(const char* name) const {
CND_PRECONDITION(directory[0] != 0, "directory is not open");
- char buffer[CL_MAX_DIR];
- priv_getFN(buffer, name);
+ std::string buffer = priv_getFN(name);
int64_t size = -1;
Status st = _fs->file_size(buffer, &size);
DBUG_EXECUTE_IF("inverted file read error: index file not found",
@@ -605,9 +600,8 @@ int64_t DorisFSDirectory::fileLength(const char* name)
const {
bool DorisFSDirectory::openInput(const char* name, lucene::store::IndexInput*&
ret,
CLuceneError& error, int32_t bufferSize) {
CND_PRECONDITION(directory[0] != 0, "directory is not open");
- char fl[CL_MAX_DIR];
- priv_getFN(fl, name);
- return FSIndexInput::open(_fs, fl, ret, error, bufferSize);
+ std::string fl = priv_getFN(name);
+ return FSIndexInput::open(_fs, fl.c_str(), ret, error, bufferSize);
}
void DorisFSDirectory::close() {
@@ -617,8 +611,7 @@ void DorisFSDirectory::close() {
bool DorisFSDirectory::doDeleteFile(const char* name) {
CND_PRECONDITION(directory[0] != 0, "directory is not open");
- char fl[CL_MAX_DIR];
- priv_getFN(fl, name);
+ std::string fl = priv_getFN(name);
auto st = _fs->delete_file(fl);
DBUG_EXECUTE_IF("DorisFSDirectory::doDeleteFile_status_is_not_ok", {
st = Status::Error<ErrorCode::INTERNAL_ERROR>(
@@ -630,8 +623,7 @@ bool DorisFSDirectory::doDeleteFile(const char* name) {
bool DorisFSDirectory::deleteDirectory() {
CND_PRECONDITION(directory[0] != 0, "directory is not open");
- char fl[CL_MAX_DIR];
- priv_getFN(fl, "");
+ std::string fl = priv_getFN("");
auto st = _fs->delete_directory(fl);
DBUG_EXECUTE_IF("DorisFSDirectory::deleteDirectory_throw_is_not_directory", {
st = Status::Error<ErrorCode::NOT_FOUND>(
@@ -644,11 +636,9 @@ bool DorisFSDirectory::deleteDirectory() {
void DorisFSDirectory::renameFile(const char* from, const char* to) {
CND_PRECONDITION(directory[0] != 0, "directory is not open");
std::lock_guard<std::mutex> wlock(_this_lock);
- char old[CL_MAX_DIR];
- priv_getFN(old, from);
- char nu[CL_MAX_DIR];
- priv_getFN(nu, to);
+ std::string old = priv_getFN(from);
+ std::string nu = priv_getFN(to);
bool exists = false;
auto st = _fs->exists(nu, &exists);
@@ -675,8 +665,7 @@ void DorisFSDirectory::renameFile(const char* from, const
char* to) {
lucene::store::IndexOutput* DorisFSDirectory::createOutput(const char* name) {
CND_PRECONDITION(directory[0] != 0, "directory is not open");
- char fl[CL_MAX_DIR];
- priv_getFN(fl, name);
+ std::string fl = priv_getFN(name);
bool exists = false;
auto st = _fs->exists(fl, &exists);
DBUG_EXECUTE_IF("DorisFSDirectory::createOutput_exists_status_is_not_ok", {
@@ -709,7 +698,7 @@ lucene::store::IndexOutput*
DorisFSDirectory::createOutput(const char* name) {
ErrorContext error_context;
ret->set_file_writer_opts(_opts);
try {
- ret->init(_fs, fl);
+ ret->init(_fs, fl.c_str());
} catch (CLuceneError& err) {
error_context.eptr = std::current_exception();
error_context.err_msg.append("FSIndexOutput init error: ");
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h
b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h
index 60e3a132aa4..da525ebb86a 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h
@@ -53,7 +53,7 @@ protected:
io::FileSystemSPtr _fs;
std::string directory;
- void priv_getFN(char* buffer, const char* name) const;
+ std::string priv_getFN(const std::string& name) const;
/// Removes an existing file in the directory.
bool doDeleteFile(const char* name) override;
@@ -96,6 +96,8 @@ public:
private:
int32_t filemode;
io::FileWriterOptions _opts;
+
+ friend class DorisFSDirectoryTest;
};
class CLUCENE_EXPORT DorisRAMFSDirectory : public DorisFSDirectory {
diff --git
a/be/test/olap/rowset/segment_v2/inverted_index_fs_directory_test.cpp
b/be/test/olap/rowset/segment_v2/inverted_index_fs_directory_test.cpp
index fa2145544af..0663b7722de 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index_fs_directory_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index_fs_directory_test.cpp
@@ -99,4 +99,25 @@ TEST_F(DorisFSDirectoryTest, FSIndexInputReadInternalTimer) {
_CLDELETE(input1);
}
+TEST_F(DorisFSDirectoryTest, PrivGetFN) {
+ {
+ std::string file_name = "my_file.txt";
+ std::string result = _directory->priv_getFN(file_name);
+ std::string expected_path = (_tmp_dir / file_name).string();
+ EXPECT_EQ(result, expected_path);
+ }
+ {
+ std::string file_name = "";
+ std::string result = _directory->priv_getFN(file_name);
+ std::string expected_path = (_tmp_dir / file_name).string();
+ EXPECT_EQ(result, expected_path);
+ }
+ {
+ std::string file_name = "subdir/another_file.log";
+ std::string result = _directory->priv_getFN(file_name);
+ std::string expected_path = (_tmp_dir / file_name).string();
+ EXPECT_EQ(result, expected_path);
+ }
+}
+
} // namespace doris::segment_v2
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
