Repository: qpid-cpp Updated Branches: refs/heads/master 7c54578c6 -> 40a133d32
QPID-7383: Fix for excessive time taken to overwrite files when overwrite-before-return option is in use. Also improves time taken to create new journal files for the pool. Project: http://git-wip-us.apache.org/repos/asf/qpid-cpp/repo Commit: http://git-wip-us.apache.org/repos/asf/qpid-cpp/commit/2dc01547 Tree: http://git-wip-us.apache.org/repos/asf/qpid-cpp/tree/2dc01547 Diff: http://git-wip-us.apache.org/repos/asf/qpid-cpp/diff/2dc01547 Branch: refs/heads/master Commit: 2dc01547ba5481bfa7ad8e31c183f3ecf18704cd Parents: 6136161 Author: Kim van der Riet <[email protected]> Authored: Tue Aug 9 16:56:20 2016 -0400 Committer: Kim van der Riet <[email protected]> Committed: Tue Aug 9 16:56:20 2016 -0400 ---------------------------------------------------------------------- src/qpid/linearstore/journal/EmptyFilePool.cpp | 63 +++++++++++++++------ src/qpid/linearstore/journal/EmptyFilePool.h | 8 +++ 2 files changed, 54 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/qpid-cpp/blob/2dc01547/src/qpid/linearstore/journal/EmptyFilePool.cpp ---------------------------------------------------------------------- diff --git a/src/qpid/linearstore/journal/EmptyFilePool.cpp b/src/qpid/linearstore/journal/EmptyFilePool.cpp index 681ced3..3f3b61e 100644 --- a/src/qpid/linearstore/journal/EmptyFilePool.cpp +++ b/src/qpid/linearstore/journal/EmptyFilePool.cpp @@ -23,7 +23,6 @@ #include <fstream> #include "qpid/linearstore/journal/EmptyFilePoolPartition.h" -#include "qpid/linearstore/journal/jcfg.h" #include "qpid/linearstore/journal/jdir.h" #include "qpid/linearstore/journal/JournalLog.h" #include "qpid/linearstore/journal/slock.h" @@ -37,11 +36,20 @@ namespace qpid { namespace linearstore { namespace journal { -// static -std::string EmptyFilePool::s_inuseFileDirectory_ = "in_use"; +#define FHDR_BUFF_SIZE (QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB) * 1024 +#define ZERO_BUFF_SIZE QLS_SBLK_SIZE_KIB * 1024 -// static +// tatic declarations + +std::string EmptyFilePool::s_inuseFileDirectory_ = "in_use"; std::string EmptyFilePool::s_returnedFileDirectory_ = "returned"; +size_t EmptyFilePool::s_fhdr_buff_size_ = FHDR_BUFF_SIZE; +unsigned char EmptyFilePool::s_fhdr_buff_[FHDR_BUFF_SIZE]; +smutex EmptyFilePool::s_fhdr_buff_mutex_; +size_t EmptyFilePool::s_zero_buff_size_ = ZERO_BUFF_SIZE; +unsigned char EmptyFilePool::s_zero_buff_[ZERO_BUFF_SIZE]; +bool EmptyFilePool::s_static_initializer_flag_ = false; + EmptyFilePool::EmptyFilePool(const std::string& efpDirectory, const EmptyFilePoolPartition* partitionPtr, @@ -54,7 +62,12 @@ EmptyFilePool::EmptyFilePool(const std::string& efpDirectory, overwriteBeforeReturnFlag_(overwriteBeforeReturnFlag), truncateFlag_(truncateFlag), journalLogRef_(journalLogRef) -{} +{ + if (!s_static_initializer_flag_) { + initializeStaticBuffers(); + s_static_initializer_flag_ = true; + } +} EmptyFilePool::~EmptyFilePool() {} @@ -250,19 +263,28 @@ void EmptyFilePool::initializeSubDirectory(const std::string& fqDirName) { } void EmptyFilePool::overwriteFileContents(const std::string& fqFileName) { - ::file_hdr_t fh; - ::file_hdr_create(&fh, QLS_FILE_MAGIC, QLS_JRNL_VERSION, QLS_JRNL_FHDR_RES_SIZE_SBLKS, partitionPtr_->getPartitionNumber(), efpDataSize_kib_); - std::ofstream ofs(fqFileName.c_str(), std::ofstream::out | std::ofstream::binary); - checkIosState(errno, ofs, jerrno::JERR_EFP_FOPEN, fqFileName, "constructor", "Failed to create file", "EmptyFilePool", "overwriteFileContents"); - ofs.write((char*)&fh, sizeof(::file_hdr_t)); - checkIosState(errno, ofs, jerrno::JERR_EFP_FWRITE, fqFileName, "write()", "Failed to write header", "EmptyFilePool", "overwriteFileContents"); - uint64_t rem = ((efpDataSize_kib_ + (QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB)) * 1024) - sizeof(::file_hdr_t); - while (rem--) { - ofs.put('\0'); - checkIosState(errno, ofs, jerrno::JERR_EFP_FWRITE, fqFileName, "put()", "Failed to put \0", "EmptyFilePool", "overwriteFileContents"); + FILE* pFile; + pFile = ::fopen(fqFileName.c_str(), "wb"); + { + slock l(s_fhdr_buff_mutex_); + + // Initialize file header + ::file_hdr_create((::file_hdr_t*)s_fhdr_buff_, + QLS_FILE_MAGIC, + QLS_JRNL_VERSION, + QLS_JRNL_FHDR_RES_SIZE_SBLKS, + partitionPtr_->getPartitionNumber(), + efpDataSize_kib_); + + // Write file header + ::fwrite((void*)s_fhdr_buff_, 1, s_fhdr_buff_size_, pFile); + } + + // Fill rest of file with zeros (buffer is 1 sblk in size) + for (efpDataSize_sblks_t i = 0; i < dataSize_sblks(); ++i) { + ::fwrite((void*)s_zero_buff_, 1, s_zero_buff_size_, pFile); } - ofs.close(); -//std::cout << "*** WARNING: EFP " << efpDirectory_ << " is empty - created new journal file " << fqFileName.substr(fqFileName.rfind('/') + 1) << " on the fly" << std::endl; // DEBUG + ::fclose(pFile); } std::string EmptyFilePool::popEmptyFile() { @@ -476,4 +498,11 @@ bool EmptyFilePool::moveFile(const std::string& from, return true; } +//static +void EmptyFilePool::initializeStaticBuffers() { + // Overwrite buffers with zeros + ::memset(s_fhdr_buff_, 0, s_fhdr_buff_size_); + ::memset(s_zero_buff_, 0, s_zero_buff_size_); +} + }}} http://git-wip-us.apache.org/repos/asf/qpid-cpp/blob/2dc01547/src/qpid/linearstore/journal/EmptyFilePool.h ---------------------------------------------------------------------- diff --git a/src/qpid/linearstore/journal/EmptyFilePool.h b/src/qpid/linearstore/journal/EmptyFilePool.h index 935057f..aaf09d5 100644 --- a/src/qpid/linearstore/journal/EmptyFilePool.h +++ b/src/qpid/linearstore/journal/EmptyFilePool.h @@ -30,6 +30,7 @@ namespace journal { #include <deque> #include "qpid/linearstore/journal/EmptyFilePoolTypes.h" +#include "qpid/linearstore/journal/jcfg.h" #include "qpid/linearstore/journal/smutex.h" namespace qpid { @@ -48,6 +49,12 @@ protected: static std::string s_inuseFileDirectory_; static std::string s_returnedFileDirectory_; + static size_t s_fhdr_buff_size_; + static unsigned char s_fhdr_buff_[(QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB) * 1024]; + static smutex s_fhdr_buff_mutex_; + static size_t s_zero_buff_size_; + static unsigned char s_zero_buff_[QLS_SBLK_SIZE_KIB * 1024]; + static bool s_static_initializer_flag_; const std::string efpDirectory_; const efpDataSize_kib_t efpDataSize_kib_; @@ -112,6 +119,7 @@ protected: static bool isSymlink(const std::string& fqName); static bool moveFile(const std::string& fromFqPath, const std::string& toFqPath); + static void initializeStaticBuffers(); }; }}} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
