http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/d3a13a49/thirdparty/rocksdb/util/fault_injection_test_env.cc ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/util/fault_injection_test_env.cc b/thirdparty/rocksdb/util/fault_injection_test_env.cc deleted file mode 100644 index 3b3a8b9..0000000 --- a/thirdparty/rocksdb/util/fault_injection_test_env.cc +++ /dev/null @@ -1,313 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). -// -// Copyright 2014 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -// This test uses a custom Env to keep track of the state of a filesystem as of -// the last "sync". It then checks for data loss errors by purposely dropping -// file data (or entire files) not protected by a "sync". - -#include "util/fault_injection_test_env.h" -#include <functional> -#include <utility> - -namespace rocksdb { - -// Assume a filename, and not a directory name like "/foo/bar/" -std::string GetDirName(const std::string filename) { - size_t found = filename.find_last_of("/\\"); - if (found == std::string::npos) { - return ""; - } else { - return filename.substr(0, found); - } -} - -// A basic file truncation function suitable for this test. -Status Truncate(Env* env, const std::string& filename, uint64_t length) { - unique_ptr<SequentialFile> orig_file; - const EnvOptions options; - Status s = env->NewSequentialFile(filename, &orig_file, options); - if (!s.ok()) { - fprintf(stderr, "Cannot truncate file %s: %s\n", filename.c_str(), - s.ToString().c_str()); - return s; - } - - std::unique_ptr<char[]> scratch(new char[length]); - rocksdb::Slice result; - s = orig_file->Read(length, &result, scratch.get()); -#ifdef OS_WIN - orig_file.reset(); -#endif - if (s.ok()) { - std::string tmp_name = GetDirName(filename) + "/truncate.tmp"; - unique_ptr<WritableFile> tmp_file; - s = env->NewWritableFile(tmp_name, &tmp_file, options); - if (s.ok()) { - s = tmp_file->Append(result); - if (s.ok()) { - s = env->RenameFile(tmp_name, filename); - } else { - fprintf(stderr, "Cannot rename file %s to %s: %s\n", tmp_name.c_str(), - filename.c_str(), s.ToString().c_str()); - env->DeleteFile(tmp_name); - } - } - } - if (!s.ok()) { - fprintf(stderr, "Cannot truncate file %s: %s\n", filename.c_str(), - s.ToString().c_str()); - } - - return s; -} - -// Trim the tailing "/" in the end of `str` -std::string TrimDirname(const std::string& str) { - size_t found = str.find_last_not_of("/"); - if (found == std::string::npos) { - return str; - } - return str.substr(0, found + 1); -} - -// Return pair <parent directory name, file name> of a full path. -std::pair<std::string, std::string> GetDirAndName(const std::string& name) { - std::string dirname = GetDirName(name); - std::string fname = name.substr(dirname.size() + 1); - return std::make_pair(dirname, fname); -} - -Status FileState::DropUnsyncedData(Env* env) const { - ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_; - return Truncate(env, filename_, sync_pos); -} - -Status FileState::DropRandomUnsyncedData(Env* env, Random* rand) const { - ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_; - assert(pos_ >= sync_pos); - int range = static_cast<int>(pos_ - sync_pos); - uint64_t truncated_size = - static_cast<uint64_t>(sync_pos) + rand->Uniform(range); - return Truncate(env, filename_, truncated_size); -} - -Status TestDirectory::Fsync() { - env_->SyncDir(dirname_); - return dir_->Fsync(); -} - -TestWritableFile::TestWritableFile(const std::string& fname, - unique_ptr<WritableFile>&& f, - FaultInjectionTestEnv* env) - : state_(fname), - target_(std::move(f)), - writable_file_opened_(true), - env_(env) { - assert(target_ != nullptr); - state_.pos_ = 0; -} - -TestWritableFile::~TestWritableFile() { - if (writable_file_opened_) { - Close(); - } -} - -Status TestWritableFile::Append(const Slice& data) { - if (!env_->IsFilesystemActive()) { - return Status::Corruption("Not Active"); - } - Status s = target_->Append(data); - if (s.ok()) { - state_.pos_ += data.size(); - } - return s; -} - -Status TestWritableFile::Close() { - writable_file_opened_ = false; - Status s = target_->Close(); - if (s.ok()) { - env_->WritableFileClosed(state_); - } - return s; -} - -Status TestWritableFile::Flush() { - Status s = target_->Flush(); - if (s.ok() && env_->IsFilesystemActive()) { - state_.pos_at_last_flush_ = state_.pos_; - } - return s; -} - -Status TestWritableFile::Sync() { - if (!env_->IsFilesystemActive()) { - return Status::IOError("FaultInjectionTestEnv: not active"); - } - // No need to actual sync. - state_.pos_at_last_sync_ = state_.pos_; - return Status::OK(); -} - -Status FaultInjectionTestEnv::NewDirectory(const std::string& name, - unique_ptr<Directory>* result) { - unique_ptr<Directory> r; - Status s = target()->NewDirectory(name, &r); - assert(s.ok()); - if (!s.ok()) { - return s; - } - result->reset(new TestDirectory(this, TrimDirname(name), r.release())); - return Status::OK(); -} - -Status FaultInjectionTestEnv::NewWritableFile(const std::string& fname, - unique_ptr<WritableFile>* result, - const EnvOptions& soptions) { - if (!IsFilesystemActive()) { - return Status::Corruption("Not Active"); - } - // Not allow overwriting files - Status s = target()->FileExists(fname); - if (s.ok()) { - return Status::Corruption("File already exists."); - } else if (!s.IsNotFound()) { - assert(s.IsIOError()); - return s; - } - s = target()->NewWritableFile(fname, result, soptions); - if (s.ok()) { - result->reset(new TestWritableFile(fname, std::move(*result), this)); - // WritableFileWriter* file is opened - // again then it will be truncated - so forget our saved state. - UntrackFile(fname); - MutexLock l(&mutex_); - open_files_.insert(fname); - auto dir_and_name = GetDirAndName(fname); - auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first]; - list.insert(dir_and_name.second); - } - return s; -} - -Status FaultInjectionTestEnv::DeleteFile(const std::string& f) { - if (!IsFilesystemActive()) { - return Status::Corruption("Not Active"); - } - Status s = EnvWrapper::DeleteFile(f); - if (!s.ok()) { - fprintf(stderr, "Cannot delete file %s: %s\n", f.c_str(), - s.ToString().c_str()); - } - assert(s.ok()); - if (s.ok()) { - UntrackFile(f); - } - return s; -} - -Status FaultInjectionTestEnv::RenameFile(const std::string& s, - const std::string& t) { - if (!IsFilesystemActive()) { - return Status::Corruption("Not Active"); - } - Status ret = EnvWrapper::RenameFile(s, t); - - if (ret.ok()) { - MutexLock l(&mutex_); - if (db_file_state_.find(s) != db_file_state_.end()) { - db_file_state_[t] = db_file_state_[s]; - db_file_state_.erase(s); - } - - auto sdn = GetDirAndName(s); - auto tdn = GetDirAndName(t); - if (dir_to_new_files_since_last_sync_[sdn.first].erase(sdn.second) != 0) { - auto& tlist = dir_to_new_files_since_last_sync_[tdn.first]; - assert(tlist.find(tdn.second) == tlist.end()); - tlist.insert(tdn.second); - } - } - - return ret; -} - -void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) { - MutexLock l(&mutex_); - if (open_files_.find(state.filename_) != open_files_.end()) { - db_file_state_[state.filename_] = state; - open_files_.erase(state.filename_); - } -} - -// For every file that is not fully synced, make a call to `func` with -// FileState of the file as the parameter. -Status FaultInjectionTestEnv::DropFileData( - std::function<Status(Env*, FileState)> func) { - Status s; - MutexLock l(&mutex_); - for (std::map<std::string, FileState>::const_iterator it = - db_file_state_.begin(); - s.ok() && it != db_file_state_.end(); ++it) { - const FileState& state = it->second; - if (!state.IsFullySynced()) { - s = func(target(), state); - } - } - return s; -} - -Status FaultInjectionTestEnv::DropUnsyncedFileData() { - return DropFileData([&](Env* env, const FileState& state) { - return state.DropUnsyncedData(env); - }); -} - -Status FaultInjectionTestEnv::DropRandomUnsyncedFileData(Random* rnd) { - return DropFileData([&](Env* env, const FileState& state) { - return state.DropRandomUnsyncedData(env, rnd); - }); -} - -Status FaultInjectionTestEnv::DeleteFilesCreatedAfterLastDirSync() { - // Because DeleteFile access this container make a copy to avoid deadlock - std::map<std::string, std::set<std::string>> map_copy; - { - MutexLock l(&mutex_); - map_copy.insert(dir_to_new_files_since_last_sync_.begin(), - dir_to_new_files_since_last_sync_.end()); - } - - for (auto& pair : map_copy) { - for (std::string name : pair.second) { - Status s = DeleteFile(pair.first + "/" + name); - if (!s.ok()) { - return s; - } - } - } - return Status::OK(); -} -void FaultInjectionTestEnv::ResetState() { - MutexLock l(&mutex_); - db_file_state_.clear(); - dir_to_new_files_since_last_sync_.clear(); - SetFilesystemActiveNoLock(true); -} - -void FaultInjectionTestEnv::UntrackFile(const std::string& f) { - MutexLock l(&mutex_); - auto dir_and_name = GetDirAndName(f); - dir_to_new_files_since_last_sync_[dir_and_name.first].erase( - dir_and_name.second); - db_file_state_.erase(f); - open_files_.erase(f); -} -} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/d3a13a49/thirdparty/rocksdb/util/fault_injection_test_env.h ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/util/fault_injection_test_env.h b/thirdparty/rocksdb/util/fault_injection_test_env.h deleted file mode 100644 index 5d0ae63..0000000 --- a/thirdparty/rocksdb/util/fault_injection_test_env.h +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). -// -// Copyright 2014 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -// This test uses a custom Env to keep track of the state of a filesystem as of -// the last "sync". It then checks for data loss errors by purposely dropping -// file data (or entire files) not protected by a "sync". - -#ifndef UTIL_FAULT_INJECTION_TEST_ENV_H_ -#define UTIL_FAULT_INJECTION_TEST_ENV_H_ - -#include <map> -#include <set> -#include <string> - -#include "db/version_set.h" -#include "env/mock_env.h" -#include "rocksdb/db.h" -#include "rocksdb/env.h" -#include "util/filename.h" -#include "util/mutexlock.h" -#include "util/random.h" - -namespace rocksdb { - -class TestWritableFile; -class FaultInjectionTestEnv; - -struct FileState { - std::string filename_; - ssize_t pos_; - ssize_t pos_at_last_sync_; - ssize_t pos_at_last_flush_; - - explicit FileState(const std::string& filename) - : filename_(filename), - pos_(-1), - pos_at_last_sync_(-1), - pos_at_last_flush_(-1) {} - - FileState() : pos_(-1), pos_at_last_sync_(-1), pos_at_last_flush_(-1) {} - - bool IsFullySynced() const { return pos_ <= 0 || pos_ == pos_at_last_sync_; } - - Status DropUnsyncedData(Env* env) const; - - Status DropRandomUnsyncedData(Env* env, Random* rand) const; -}; - -// A wrapper around WritableFileWriter* file -// is written to or sync'ed. -class TestWritableFile : public WritableFile { - public: - explicit TestWritableFile(const std::string& fname, - unique_ptr<WritableFile>&& f, - FaultInjectionTestEnv* env); - virtual ~TestWritableFile(); - virtual Status Append(const Slice& data) override; - virtual Status Truncate(uint64_t size) override { - return target_->Truncate(size); - } - virtual Status Close() override; - virtual Status Flush() override; - virtual Status Sync() override; - virtual bool IsSyncThreadSafe() const override { return true; } - - private: - FileState state_; - unique_ptr<WritableFile> target_; - bool writable_file_opened_; - FaultInjectionTestEnv* env_; -}; - -class TestDirectory : public Directory { - public: - explicit TestDirectory(FaultInjectionTestEnv* env, std::string dirname, - Directory* dir) - : env_(env), dirname_(dirname), dir_(dir) {} - ~TestDirectory() {} - - virtual Status Fsync() override; - - private: - FaultInjectionTestEnv* env_; - std::string dirname_; - unique_ptr<Directory> dir_; -}; - -class FaultInjectionTestEnv : public EnvWrapper { - public: - explicit FaultInjectionTestEnv(Env* base) - : EnvWrapper(base), filesystem_active_(true) {} - virtual ~FaultInjectionTestEnv() {} - - Status NewDirectory(const std::string& name, - unique_ptr<Directory>* result) override; - - Status NewWritableFile(const std::string& fname, - unique_ptr<WritableFile>* result, - const EnvOptions& soptions) override; - - virtual Status DeleteFile(const std::string& f) override; - - virtual Status RenameFile(const std::string& s, - const std::string& t) override; - - void WritableFileClosed(const FileState& state); - - // For every file that is not fully synced, make a call to `func` with - // FileState of the file as the parameter. - Status DropFileData(std::function<Status(Env*, FileState)> func); - - Status DropUnsyncedFileData(); - - Status DropRandomUnsyncedFileData(Random* rnd); - - Status DeleteFilesCreatedAfterLastDirSync(); - - void ResetState(); - - void UntrackFile(const std::string& f); - - void SyncDir(const std::string& dirname) { - MutexLock l(&mutex_); - dir_to_new_files_since_last_sync_.erase(dirname); - } - - // Setting the filesystem to inactive is the test equivalent to simulating a - // system reset. Setting to inactive will freeze our saved filesystem state so - // that it will stop being recorded. It can then be reset back to the state at - // the time of the reset. - bool IsFilesystemActive() { - MutexLock l(&mutex_); - return filesystem_active_; - } - void SetFilesystemActiveNoLock(bool active) { filesystem_active_ = active; } - void SetFilesystemActive(bool active) { - MutexLock l(&mutex_); - SetFilesystemActiveNoLock(active); - } - void AssertNoOpenFile() { assert(open_files_.empty()); } - - private: - port::Mutex mutex_; - std::map<std::string, FileState> db_file_state_; - std::set<std::string> open_files_; - std::unordered_map<std::string, std::set<std::string>> - dir_to_new_files_since_last_sync_; - bool filesystem_active_; // Record flushes, syncs, writes -}; - -} // namespace rocksdb - -#endif // UTIL_FAULT_INJECTION_TEST_ENV_H_ http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/d3a13a49/thirdparty/rocksdb/util/file_reader_writer_test.cc ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/util/file_reader_writer_test.cc b/thirdparty/rocksdb/util/file_reader_writer_test.cc deleted file mode 100644 index 45675e9..0000000 --- a/thirdparty/rocksdb/util/file_reader_writer_test.cc +++ /dev/null @@ -1,325 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). -// -#include "util/file_reader_writer.h" -#include <algorithm> -#include <vector> -#include "util/random.h" -#include "util/testharness.h" -#include "util/testutil.h" - -namespace rocksdb { - -class WritableFileWriterTest : public testing::Test {}; - -const uint32_t kMb = 1 << 20; - -TEST_F(WritableFileWriterTest, RangeSync) { - class FakeWF : public WritableFile { - public: - explicit FakeWF() : size_(0), last_synced_(0) {} - ~FakeWF() {} - - Status Append(const Slice& data) override { - size_ += data.size(); - return Status::OK(); - } - virtual Status Truncate(uint64_t size) override { - return Status::OK(); - } - Status Close() override { - EXPECT_GE(size_, last_synced_ + kMb); - EXPECT_LT(size_, last_synced_ + 2 * kMb); - // Make sure random writes generated enough writes. - EXPECT_GT(size_, 10 * kMb); - return Status::OK(); - } - Status Flush() override { return Status::OK(); } - Status Sync() override { return Status::OK(); } - Status Fsync() override { return Status::OK(); } - void SetIOPriority(Env::IOPriority pri) override {} - uint64_t GetFileSize() override { return size_; } - void GetPreallocationStatus(size_t* block_size, - size_t* last_allocated_block) override {} - size_t GetUniqueId(char* id, size_t max_size) const override { return 0; } - Status InvalidateCache(size_t offset, size_t length) override { - return Status::OK(); - } - - protected: - Status Allocate(uint64_t offset, uint64_t len) override { return Status::OK(); } - Status RangeSync(uint64_t offset, uint64_t nbytes) override { - EXPECT_EQ(offset % 4096, 0u); - EXPECT_EQ(nbytes % 4096, 0u); - - EXPECT_EQ(offset, last_synced_); - last_synced_ = offset + nbytes; - EXPECT_GE(size_, last_synced_ + kMb); - if (size_ > 2 * kMb) { - EXPECT_LT(size_, last_synced_ + 2 * kMb); - } - return Status::OK(); - } - - uint64_t size_; - uint64_t last_synced_; - }; - - EnvOptions env_options; - env_options.bytes_per_sync = kMb; - unique_ptr<FakeWF> wf(new FakeWF); - unique_ptr<WritableFileWriter> writer( - new WritableFileWriter(std::move(wf), env_options)); - Random r(301); - std::unique_ptr<char[]> large_buf(new char[10 * kMb]); - for (int i = 0; i < 1000; i++) { - int skew_limit = (i < 700) ? 10 : 15; - uint32_t num = r.Skewed(skew_limit) * 100 + r.Uniform(100); - writer->Append(Slice(large_buf.get(), num)); - - // Flush in a chance of 1/10. - if (r.Uniform(10) == 0) { - writer->Flush(); - } - } - writer->Close(); -} - -TEST_F(WritableFileWriterTest, IncrementalBuffer) { - class FakeWF : public WritableFile { - public: - explicit FakeWF(std::string* _file_data, bool _use_direct_io, - bool _no_flush) - : file_data_(_file_data), - use_direct_io_(_use_direct_io), - no_flush_(_no_flush) {} - ~FakeWF() {} - - Status Append(const Slice& data) override { - file_data_->append(data.data(), data.size()); - size_ += data.size(); - return Status::OK(); - } - Status PositionedAppend(const Slice& data, uint64_t pos) override { - EXPECT_TRUE(pos % 512 == 0); - EXPECT_TRUE(data.size() % 512 == 0); - file_data_->resize(pos); - file_data_->append(data.data(), data.size()); - size_ += data.size(); - return Status::OK(); - } - - virtual Status Truncate(uint64_t size) override { - file_data_->resize(size); - return Status::OK(); - } - Status Close() override { return Status::OK(); } - Status Flush() override { return Status::OK(); } - Status Sync() override { return Status::OK(); } - Status Fsync() override { return Status::OK(); } - void SetIOPriority(Env::IOPriority pri) override {} - uint64_t GetFileSize() override { return size_; } - void GetPreallocationStatus(size_t* block_size, - size_t* last_allocated_block) override {} - size_t GetUniqueId(char* id, size_t max_size) const override { return 0; } - Status InvalidateCache(size_t offset, size_t length) override { - return Status::OK(); - } - bool use_direct_io() const override { return use_direct_io_; } - - std::string* file_data_; - bool use_direct_io_; - bool no_flush_; - size_t size_ = 0; - }; - - Random r(301); - const int kNumAttempts = 50; - for (int attempt = 0; attempt < kNumAttempts; attempt++) { - bool no_flush = (attempt % 3 == 0); - EnvOptions env_options; - env_options.writable_file_max_buffer_size = - (attempt < kNumAttempts / 2) ? 512 * 1024 : 700 * 1024; - std::string actual; - unique_ptr<FakeWF> wf(new FakeWF(&actual, -#ifndef ROCKSDB_LITE - attempt % 2 == 1, -#else - false, -#endif - no_flush)); - unique_ptr<WritableFileWriter> writer( - new WritableFileWriter(std::move(wf), env_options)); - - std::string target; - for (int i = 0; i < 20; i++) { - uint32_t num = r.Skewed(16) * 100 + r.Uniform(100); - std::string random_string; - test::RandomString(&r, num, &random_string); - writer->Append(Slice(random_string.c_str(), num)); - target.append(random_string.c_str(), num); - - // In some attempts, flush in a chance of 1/10. - if (!no_flush && r.Uniform(10) == 0) { - writer->Flush(); - } - } - writer->Flush(); - writer->Close(); - ASSERT_EQ(target.size(), actual.size()); - ASSERT_EQ(target, actual); - } -} - -#ifndef ROCKSDB_LITE -TEST_F(WritableFileWriterTest, AppendStatusReturn) { - class FakeWF : public WritableFile { - public: - explicit FakeWF() : use_direct_io_(false), io_error_(false) {} - - virtual bool use_direct_io() const override { return use_direct_io_; } - Status Append(const Slice& data) override { - if (io_error_) { - return Status::IOError("Fake IO error"); - } - return Status::OK(); - } - Status PositionedAppend(const Slice& data, uint64_t) override { - if (io_error_) { - return Status::IOError("Fake IO error"); - } - return Status::OK(); - } - Status Close() override { return Status::OK(); } - Status Flush() override { return Status::OK(); } - Status Sync() override { return Status::OK(); } - void Setuse_direct_io(bool val) { use_direct_io_ = val; } - void SetIOError(bool val) { io_error_ = val; } - - protected: - bool use_direct_io_; - bool io_error_; - }; - unique_ptr<FakeWF> wf(new FakeWF()); - wf->Setuse_direct_io(true); - unique_ptr<WritableFileWriter> writer( - new WritableFileWriter(std::move(wf), EnvOptions())); - - ASSERT_OK(writer->Append(std::string(2 * kMb, 'a'))); - - // Next call to WritableFile::Append() should fail - dynamic_cast<FakeWF*>(writer->writable_file())->SetIOError(true); - ASSERT_NOK(writer->Append(std::string(2 * kMb, 'b'))); -} -#endif - -class ReadaheadRandomAccessFileTest - : public testing::Test, - public testing::WithParamInterface<size_t> { - public: - static std::vector<size_t> GetReadaheadSizeList() { - return {1lu << 12, 1lu << 16}; - } - virtual void SetUp() override { - readahead_size_ = GetParam(); - scratch_.reset(new char[2 * readahead_size_]); - ResetSourceStr(); - } - ReadaheadRandomAccessFileTest() : control_contents_() {} - std::string Read(uint64_t offset, size_t n) { - Slice result; - test_read_holder_->Read(offset, n, &result, scratch_.get()); - return std::string(result.data(), result.size()); - } - void ResetSourceStr(const std::string& str = "") { - auto write_holder = std::unique_ptr<WritableFileWriter>( - test::GetWritableFileWriter(new test::StringSink(&control_contents_))); - write_holder->Append(Slice(str)); - write_holder->Flush(); - auto read_holder = std::unique_ptr<RandomAccessFile>( - new test::StringSource(control_contents_)); - test_read_holder_ = - NewReadaheadRandomAccessFile(std::move(read_holder), readahead_size_); - } - size_t GetReadaheadSize() const { return readahead_size_; } - - private: - size_t readahead_size_; - Slice control_contents_; - std::unique_ptr<RandomAccessFile> test_read_holder_; - std::unique_ptr<char[]> scratch_; -}; - -TEST_P(ReadaheadRandomAccessFileTest, EmptySourceStrTest) { - ASSERT_EQ("", Read(0, 1)); - ASSERT_EQ("", Read(0, 0)); - ASSERT_EQ("", Read(13, 13)); -} - -TEST_P(ReadaheadRandomAccessFileTest, SourceStrLenLessThanReadaheadSizeTest) { - std::string str = "abcdefghijklmnopqrs"; - ResetSourceStr(str); - ASSERT_EQ(str.substr(3, 4), Read(3, 4)); - ASSERT_EQ(str.substr(0, 3), Read(0, 3)); - ASSERT_EQ(str, Read(0, str.size())); - ASSERT_EQ(str.substr(7, std::min(static_cast<int>(str.size()) - 7, 30)), - Read(7, 30)); - ASSERT_EQ("", Read(100, 100)); -} - -TEST_P(ReadaheadRandomAccessFileTest, - SourceStrLenCanBeGreaterThanReadaheadSizeTest) { - Random rng(42); - for (int k = 0; k < 100; ++k) { - size_t strLen = k * GetReadaheadSize() + - rng.Uniform(static_cast<int>(GetReadaheadSize())); - std::string str = - test::RandomHumanReadableString(&rng, static_cast<int>(strLen)); - ResetSourceStr(str); - for (int test = 1; test <= 100; ++test) { - size_t offset = rng.Uniform(static_cast<int>(strLen)); - size_t n = rng.Uniform(static_cast<int>(GetReadaheadSize())); - ASSERT_EQ(str.substr(offset, std::min(n, str.size() - offset)), - Read(offset, n)); - } - } -} - -TEST_P(ReadaheadRandomAccessFileTest, NExceedReadaheadTest) { - Random rng(7); - size_t strLen = 4 * GetReadaheadSize() + - rng.Uniform(static_cast<int>(GetReadaheadSize())); - std::string str = - test::RandomHumanReadableString(&rng, static_cast<int>(strLen)); - ResetSourceStr(str); - for (int test = 1; test <= 100; ++test) { - size_t offset = rng.Uniform(static_cast<int>(strLen)); - size_t n = - GetReadaheadSize() + rng.Uniform(static_cast<int>(GetReadaheadSize())); - ASSERT_EQ(str.substr(offset, std::min(n, str.size() - offset)), - Read(offset, n)); - } -} - -INSTANTIATE_TEST_CASE_P( - EmptySourceStrTest, ReadaheadRandomAccessFileTest, - ::testing::ValuesIn(ReadaheadRandomAccessFileTest::GetReadaheadSizeList())); -INSTANTIATE_TEST_CASE_P( - SourceStrLenLessThanReadaheadSizeTest, ReadaheadRandomAccessFileTest, - ::testing::ValuesIn(ReadaheadRandomAccessFileTest::GetReadaheadSizeList())); -INSTANTIATE_TEST_CASE_P( - SourceStrLenCanBeGreaterThanReadaheadSizeTest, - ReadaheadRandomAccessFileTest, - ::testing::ValuesIn(ReadaheadRandomAccessFileTest::GetReadaheadSizeList())); -INSTANTIATE_TEST_CASE_P( - NExceedReadaheadTest, ReadaheadRandomAccessFileTest, - ::testing::ValuesIn(ReadaheadRandomAccessFileTest::GetReadaheadSizeList())); - -} // namespace rocksdb - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/d3a13a49/thirdparty/rocksdb/util/filelock_test.cc ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/util/filelock_test.cc b/thirdparty/rocksdb/util/filelock_test.cc deleted file mode 100644 index cb4bd43..0000000 --- a/thirdparty/rocksdb/util/filelock_test.cc +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). -// -#include "rocksdb/status.h" -#include "rocksdb/env.h" - -#include <vector> -#include "util/coding.h" -#include "util/testharness.h" - -namespace rocksdb { - -class LockTest : public testing::Test { - public: - static LockTest* current_; - std::string file_; - rocksdb::Env* env_; - - LockTest() : file_(test::TmpDir() + "/db_testlock_file"), - env_(rocksdb::Env::Default()) { - current_ = this; - } - - ~LockTest() { - } - - Status LockFile(FileLock** db_lock) { - return env_->LockFile(file_, db_lock); - } - - Status UnlockFile(FileLock* db_lock) { - return env_->UnlockFile(db_lock); - } -}; -LockTest* LockTest::current_; - -TEST_F(LockTest, LockBySameThread) { - FileLock* lock1; - FileLock* lock2; - - // acquire a lock on a file - ASSERT_OK(LockFile(&lock1)); - - // re-acquire the lock on the same file. This should fail. - ASSERT_TRUE(LockFile(&lock2).IsIOError()); - - // release the lock - ASSERT_OK(UnlockFile(lock1)); - -} - -} // namespace rocksdb - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/d3a13a49/thirdparty/rocksdb/util/hash_test.cc ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/util/hash_test.cc b/thirdparty/rocksdb/util/hash_test.cc deleted file mode 100644 index 959e8cd..0000000 --- a/thirdparty/rocksdb/util/hash_test.cc +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). -// -// Copyright (c) 2012 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#include <vector> - -#include "util/hash.h" -#include "util/testharness.h" - -// The hash algorithm is part of the file format, for example for the Bloom -// filters. Test that the hash values are stable for a set of random strings of -// varying lengths. -TEST(HashTest, Values) { - using rocksdb::Hash; - constexpr uint32_t kSeed = 0xbc9f1d34; // Same as BloomHash. - - EXPECT_EQ(Hash("", 0, kSeed), 3164544308); - EXPECT_EQ(Hash("\x08", 1, kSeed), 422599524); - EXPECT_EQ(Hash("\x17", 1, kSeed), 3168152998); - EXPECT_EQ(Hash("\x9a", 1, kSeed), 3195034349); - EXPECT_EQ(Hash("\x1c", 1, kSeed), 2651681383); - EXPECT_EQ(Hash("\x4d\x76", 2, kSeed), 2447836956); - EXPECT_EQ(Hash("\x52\xd5", 2, kSeed), 3854228105); - EXPECT_EQ(Hash("\x91\xf7", 2, kSeed), 31066776); - EXPECT_EQ(Hash("\xd6\x27", 2, kSeed), 1806091603); - EXPECT_EQ(Hash("\x30\x46\x0b", 3, kSeed), 3808221797); - EXPECT_EQ(Hash("\x56\xdc\xd6", 3, kSeed), 2157698265); - EXPECT_EQ(Hash("\xd4\x52\x33", 3, kSeed), 1721992661); - EXPECT_EQ(Hash("\x6a\xb5\xf4", 3, kSeed), 2469105222); - EXPECT_EQ(Hash("\x67\x53\x81\x1c", 4, kSeed), 118283265); - EXPECT_EQ(Hash("\x69\xb8\xc0\x88", 4, kSeed), 3416318611); - EXPECT_EQ(Hash("\x1e\x84\xaf\x2d", 4, kSeed), 3315003572); - EXPECT_EQ(Hash("\x46\xdc\x54\xbe", 4, kSeed), 447346355); - EXPECT_EQ(Hash("\xd0\x7a\x6e\xea\x56", 5, kSeed), 4255445370); - EXPECT_EQ(Hash("\x86\x83\xd5\xa4\xd8", 5, kSeed), 2390603402); - EXPECT_EQ(Hash("\xb7\x46\xbb\x77\xce", 5, kSeed), 2048907743); - EXPECT_EQ(Hash("\x6c\xa8\xbc\xe5\x99", 5, kSeed), 2177978500); - EXPECT_EQ(Hash("\x5c\x5e\xe1\xa0\x73\x81", 6, kSeed), 1036846008); - EXPECT_EQ(Hash("\x08\x5d\x73\x1c\xe5\x2e", 6, kSeed), 229980482); - EXPECT_EQ(Hash("\x42\xfb\xf2\x52\xb4\x10", 6, kSeed), 3655585422); - EXPECT_EQ(Hash("\x73\xe1\xff\x56\x9c\xce", 6, kSeed), 3502708029); - EXPECT_EQ(Hash("\x5c\xbe\x97\x75\x54\x9a\x52", 7, kSeed), 815120748); - EXPECT_EQ(Hash("\x16\x82\x39\x49\x88\x2b\x36", 7, kSeed), 3056033698); - EXPECT_EQ(Hash("\x59\x77\xf0\xa7\x24\xf4\x78", 7, kSeed), 587205227); - EXPECT_EQ(Hash("\xd3\xa5\x7c\x0e\xc0\x02\x07", 7, kSeed), 2030937252); - EXPECT_EQ(Hash("\x31\x1b\x98\x75\x96\x22\xd3\x9a", 8, kSeed), 469635402); - EXPECT_EQ(Hash("\x38\xd6\xf7\x28\x20\xb4\x8a\xe9", 8, kSeed), 3530274698); - EXPECT_EQ(Hash("\xbb\x18\x5d\xf4\x12\x03\xf7\x99", 8, kSeed), 1974545809); - EXPECT_EQ(Hash("\x80\xd4\x3b\x3b\xae\x22\xa2\x78", 8, kSeed), 3563570120); - EXPECT_EQ(Hash("\x1a\xb5\xd0\xfe\xab\xc3\x61\xb2\x99", 9, kSeed), 2706087434); - EXPECT_EQ(Hash("\x8e\x4a\xc3\x18\x20\x2f\x06\xe6\x3c", 9, kSeed), 1534654151); - EXPECT_EQ(Hash("\xb6\xc0\xdd\x05\x3f\xc4\x86\x4c\xef", 9, kSeed), 2355554696); - EXPECT_EQ(Hash("\x9a\x5f\x78\x0d\xaf\x50\xe1\x1f\x55", 9, kSeed), 1400800912); - EXPECT_EQ(Hash("\x22\x6f\x39\x1f\xf8\xdd\x4f\x52\x17\x94", 10, kSeed), - 3420325137); - EXPECT_EQ(Hash("\x32\x89\x2a\x75\x48\x3a\x4a\x02\x69\xdd", 10, kSeed), - 3427803584); - EXPECT_EQ(Hash("\x06\x92\x5c\xf4\x88\x0e\x7e\x68\x38\x3e", 10, kSeed), - 1152407945); - EXPECT_EQ(Hash("\xbd\x2c\x63\x38\xbf\xe9\x78\xb7\xbf\x15", 10, kSeed), - 3382479516); -} - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - - return RUN_ALL_TESTS(); -} http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/d3a13a49/thirdparty/rocksdb/util/heap_test.cc ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/util/heap_test.cc b/thirdparty/rocksdb/util/heap_test.cc deleted file mode 100644 index b415615..0000000 --- a/thirdparty/rocksdb/util/heap_test.cc +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). - -#include <gtest/gtest.h> - -#include <climits> - -#include <queue> -#include <random> -#include <utility> - -#include "util/heap.h" - -#ifndef GFLAGS -const int64_t FLAGS_iters = 100000; -#else -#include <gflags/gflags.h> -DEFINE_int64(iters, 100000, "number of pseudo-random operations in each test"); -#endif // GFLAGS - -/* - * Compares the custom heap implementation in util/heap.h against - * std::priority_queue on a pseudo-random sequence of operations. - */ - -namespace rocksdb { - -using HeapTestValue = uint64_t; -using Params = std::tuple<size_t, HeapTestValue, int64_t>; - -class HeapTest : public ::testing::TestWithParam<Params> { -}; - -TEST_P(HeapTest, Test) { - // This test performs the same pseudorandom sequence of operations on a - // BinaryHeap and an std::priority_queue, comparing output. The three - // possible operations are insert, replace top and pop. - // - // Insert is chosen slightly more often than the others so that the size of - // the heap slowly grows. Once the size heats the MAX_HEAP_SIZE limit, we - // disallow inserting until the heap becomes empty, testing the "draining" - // scenario. - - const auto MAX_HEAP_SIZE = std::get<0>(GetParam()); - const auto MAX_VALUE = std::get<1>(GetParam()); - const auto RNG_SEED = std::get<2>(GetParam()); - - BinaryHeap<HeapTestValue> heap; - std::priority_queue<HeapTestValue> ref; - - std::mt19937 rng(static_cast<unsigned int>(RNG_SEED)); - std::uniform_int_distribution<HeapTestValue> value_dist(0, MAX_VALUE); - int ndrains = 0; - bool draining = false; // hit max size, draining until we empty the heap - size_t size = 0; - for (int64_t i = 0; i < FLAGS_iters; ++i) { - if (size == 0) { - draining = false; - } - - if (!draining && - (size == 0 || std::bernoulli_distribution(0.4)(rng))) { - // insert - HeapTestValue val = value_dist(rng); - heap.push(val); - ref.push(val); - ++size; - if (size == MAX_HEAP_SIZE) { - draining = true; - ++ndrains; - } - } else if (std::bernoulli_distribution(0.5)(rng)) { - // replace top - HeapTestValue val = value_dist(rng); - heap.replace_top(val); - ref.pop(); - ref.push(val); - } else { - // pop - assert(size > 0); - heap.pop(); - ref.pop(); - --size; - } - - // After every operation, check that the public methods give the same - // results - assert((size == 0) == ref.empty()); - ASSERT_EQ(size == 0, heap.empty()); - if (size > 0) { - ASSERT_EQ(ref.top(), heap.top()); - } - } - - // Probabilities should be set up to occasionally hit the max heap size and - // drain it - assert(ndrains > 0); - - heap.clear(); - ASSERT_TRUE(heap.empty()); -} - -// Basic test, MAX_VALUE = 3*MAX_HEAP_SIZE (occasional duplicates) -INSTANTIATE_TEST_CASE_P( - Basic, HeapTest, - ::testing::Values(Params(1000, 3000, 0x1b575cf05b708945)) -); -// Mid-size heap with small values (many duplicates) -INSTANTIATE_TEST_CASE_P( - SmallValues, HeapTest, - ::testing::Values(Params(100, 10, 0x5ae213f7bd5dccd0)) -); -// Small heap, large value range (no duplicates) -INSTANTIATE_TEST_CASE_P( - SmallHeap, HeapTest, - ::testing::Values(Params(10, ULLONG_MAX, 0x3e1fa8f4d01707cf)) -); -// Two-element heap -INSTANTIATE_TEST_CASE_P( - TwoElementHeap, HeapTest, - ::testing::Values(Params(2, 5, 0x4b5e13ea988c6abc)) -); -// One-element heap -INSTANTIATE_TEST_CASE_P( - OneElementHeap, HeapTest, - ::testing::Values(Params(1, 3, 0x176a1019ab0b612e)) -); - -} // namespace rocksdb - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); -#ifdef GFLAGS - GFLAGS::ParseCommandLineFlags(&argc, &argv, true); -#endif // GFLAGS - return RUN_ALL_TESTS(); -} http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/d3a13a49/thirdparty/rocksdb/util/rate_limiter_test.cc ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/util/rate_limiter_test.cc b/thirdparty/rocksdb/util/rate_limiter_test.cc deleted file mode 100644 index f099808..0000000 --- a/thirdparty/rocksdb/util/rate_limiter_test.cc +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). -// -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#ifndef __STDC_FORMAT_MACROS -#define __STDC_FORMAT_MACROS -#endif - -#include "util/rate_limiter.h" -#include <inttypes.h> -#include <limits> -#include "rocksdb/env.h" -#include "util/random.h" -#include "util/sync_point.h" -#include "util/testharness.h" - -namespace rocksdb { - -// TODO(yhchiang): the rate will not be accurate when we run test in parallel. -class RateLimiterTest : public testing::Test {}; - -TEST_F(RateLimiterTest, OverflowRate) { - GenericRateLimiter limiter(port::kMaxInt64, 1000, 10); - ASSERT_GT(limiter.GetSingleBurstBytes(), 1000000000ll); -} - -TEST_F(RateLimiterTest, StartStop) { - std::unique_ptr<RateLimiter> limiter(NewGenericRateLimiter(100, 100, 10)); -} - -TEST_F(RateLimiterTest, Modes) { - for (auto mode : {RateLimiter::Mode::kWritesOnly, - RateLimiter::Mode::kReadsOnly, RateLimiter::Mode::kAllIo}) { - GenericRateLimiter limiter(2000 /* rate_bytes_per_sec */, - 1000 * 1000 /* refill_period_us */, - 10 /* fairness */, mode); - limiter.Request(1000 /* bytes */, Env::IO_HIGH, nullptr /* stats */, - RateLimiter::OpType::kRead); - if (mode == RateLimiter::Mode::kWritesOnly) { - ASSERT_EQ(0, limiter.GetTotalBytesThrough(Env::IO_HIGH)); - } else { - ASSERT_EQ(1000, limiter.GetTotalBytesThrough(Env::IO_HIGH)); - } - - limiter.Request(1000 /* bytes */, Env::IO_HIGH, nullptr /* stats */, - RateLimiter::OpType::kWrite); - if (mode == RateLimiter::Mode::kAllIo) { - ASSERT_EQ(2000, limiter.GetTotalBytesThrough(Env::IO_HIGH)); - } else { - ASSERT_EQ(1000, limiter.GetTotalBytesThrough(Env::IO_HIGH)); - } - } -} - -#if !(defined(TRAVIS) && defined(OS_MACOSX)) -TEST_F(RateLimiterTest, Rate) { - auto* env = Env::Default(); - struct Arg { - Arg(int32_t _target_rate, int _burst) - : limiter(NewGenericRateLimiter(_target_rate, 100 * 1000, 10)), - request_size(_target_rate / 10), - burst(_burst) {} - std::unique_ptr<RateLimiter> limiter; - int32_t request_size; - int burst; - }; - - auto writer = [](void* p) { - auto* thread_env = Env::Default(); - auto* arg = static_cast<Arg*>(p); - // Test for 2 seconds - auto until = thread_env->NowMicros() + 2 * 1000000; - Random r((uint32_t)(thread_env->NowNanos() % - std::numeric_limits<uint32_t>::max())); - while (thread_env->NowMicros() < until) { - for (int i = 0; i < static_cast<int>(r.Skewed(arg->burst) + 1); ++i) { - arg->limiter->Request(r.Uniform(arg->request_size - 1) + 1, - Env::IO_HIGH, nullptr /* stats */, - RateLimiter::OpType::kWrite); - } - arg->limiter->Request(r.Uniform(arg->request_size - 1) + 1, Env::IO_LOW, - nullptr /* stats */, RateLimiter::OpType::kWrite); - } - }; - - for (int i = 1; i <= 16; i *= 2) { - int32_t target = i * 1024 * 10; - Arg arg(target, i / 4 + 1); - int64_t old_total_bytes_through = 0; - for (int iter = 1; iter <= 2; ++iter) { - // second iteration changes the target dynamically - if (iter == 2) { - target *= 2; - arg.limiter->SetBytesPerSecond(target); - } - auto start = env->NowMicros(); - for (int t = 0; t < i; ++t) { - env->StartThread(writer, &arg); - } - env->WaitForJoin(); - - auto elapsed = env->NowMicros() - start; - double rate = - (arg.limiter->GetTotalBytesThrough() - old_total_bytes_through) * - 1000000.0 / elapsed; - old_total_bytes_through = arg.limiter->GetTotalBytesThrough(); - fprintf(stderr, - "request size [1 - %" PRIi32 "], limit %" PRIi32 - " KB/sec, actual rate: %lf KB/sec, elapsed %.2lf seconds\n", - arg.request_size - 1, target / 1024, rate / 1024, - elapsed / 1000000.0); - - ASSERT_GE(rate / target, 0.80); - ASSERT_LE(rate / target, 1.25); - } - } -} -#endif - -TEST_F(RateLimiterTest, LimitChangeTest) { - // starvation test when limit changes to a smaller value - int64_t refill_period = 1000 * 1000; - auto* env = Env::Default(); - rocksdb::SyncPoint::GetInstance()->EnableProcessing(); - struct Arg { - Arg(int32_t _request_size, Env::IOPriority _pri, - std::shared_ptr<RateLimiter> _limiter) - : request_size(_request_size), pri(_pri), limiter(_limiter) {} - int32_t request_size; - Env::IOPriority pri; - std::shared_ptr<RateLimiter> limiter; - }; - - auto writer = [](void* p) { - auto* arg = static_cast<Arg*>(p); - arg->limiter->Request(arg->request_size, arg->pri, nullptr /* stats */, - RateLimiter::OpType::kWrite); - }; - - for (uint32_t i = 1; i <= 16; i <<= 1) { - int32_t target = i * 1024 * 10; - // refill per second - for (int iter = 0; iter < 2; iter++) { - std::shared_ptr<RateLimiter> limiter = - std::make_shared<GenericRateLimiter>(target, refill_period, 10); - rocksdb::SyncPoint::GetInstance()->LoadDependency( - {{"GenericRateLimiter::Request", - "RateLimiterTest::LimitChangeTest:changeLimitStart"}, - {"RateLimiterTest::LimitChangeTest:changeLimitEnd", - "GenericRateLimiter::Refill"}}); - Arg arg(target, Env::IO_HIGH, limiter); - // The idea behind is to start a request first, then before it refills, - // update limit to a different value (2X/0.5X). No starvation should - // be guaranteed under any situation - // TODO(lightmark): more test cases are welcome. - env->StartThread(writer, &arg); - int32_t new_limit = (target << 1) >> (iter << 1); - TEST_SYNC_POINT("RateLimiterTest::LimitChangeTest:changeLimitStart"); - arg.limiter->SetBytesPerSecond(new_limit); - TEST_SYNC_POINT("RateLimiterTest::LimitChangeTest:changeLimitEnd"); - env->WaitForJoin(); - fprintf(stderr, - "[COMPLETE] request size %" PRIi32 " KB, new limit %" PRIi32 - "KB/sec, refill period %" PRIi64 " ms\n", - target / 1024, new_limit / 1024, refill_period / 1000); - } - } -} - -} // namespace rocksdb - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/d3a13a49/thirdparty/rocksdb/util/slice_transform_test.cc ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/util/slice_transform_test.cc b/thirdparty/rocksdb/util/slice_transform_test.cc deleted file mode 100644 index 0b0e564..0000000 --- a/thirdparty/rocksdb/util/slice_transform_test.cc +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). -// -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#include "rocksdb/slice_transform.h" - -#include "rocksdb/db.h" -#include "rocksdb/env.h" -#include "rocksdb/filter_policy.h" -#include "rocksdb/statistics.h" -#include "rocksdb/table.h" -#include "util/testharness.h" - -namespace rocksdb { - -class SliceTransformTest : public testing::Test {}; - -TEST_F(SliceTransformTest, CapPrefixTransform) { - std::string s; - s = "abcdefge"; - - unique_ptr<const SliceTransform> transform; - - transform.reset(NewCappedPrefixTransform(6)); - ASSERT_EQ(transform->Transform(s).ToString(), "abcdef"); - ASSERT_TRUE(transform->SameResultWhenAppended("123456")); - ASSERT_TRUE(transform->SameResultWhenAppended("1234567")); - ASSERT_TRUE(!transform->SameResultWhenAppended("12345")); - - transform.reset(NewCappedPrefixTransform(8)); - ASSERT_EQ(transform->Transform(s).ToString(), "abcdefge"); - - transform.reset(NewCappedPrefixTransform(10)); - ASSERT_EQ(transform->Transform(s).ToString(), "abcdefge"); - - transform.reset(NewCappedPrefixTransform(0)); - ASSERT_EQ(transform->Transform(s).ToString(), ""); - - transform.reset(NewCappedPrefixTransform(0)); - ASSERT_EQ(transform->Transform("").ToString(), ""); -} - -class SliceTransformDBTest : public testing::Test { - private: - std::string dbname_; - Env* env_; - DB* db_; - - public: - SliceTransformDBTest() : env_(Env::Default()), db_(nullptr) { - dbname_ = test::TmpDir() + "/slice_transform_db_test"; - EXPECT_OK(DestroyDB(dbname_, last_options_)); - } - - ~SliceTransformDBTest() { - delete db_; - EXPECT_OK(DestroyDB(dbname_, last_options_)); - } - - DB* db() { return db_; } - - // Return the current option configuration. - Options* GetOptions() { return &last_options_; } - - void DestroyAndReopen() { - // Destroy using last options - Destroy(); - ASSERT_OK(TryReopen()); - } - - void Destroy() { - delete db_; - db_ = nullptr; - ASSERT_OK(DestroyDB(dbname_, last_options_)); - } - - Status TryReopen() { - delete db_; - db_ = nullptr; - last_options_.create_if_missing = true; - - return DB::Open(last_options_, dbname_, &db_); - } - - Options last_options_; -}; - -namespace { -uint64_t TestGetTickerCount(const Options& options, Tickers ticker_type) { - return options.statistics->getTickerCount(ticker_type); -} -} // namespace - -TEST_F(SliceTransformDBTest, CapPrefix) { - last_options_.prefix_extractor.reset(NewCappedPrefixTransform(8)); - last_options_.statistics = rocksdb::CreateDBStatistics(); - BlockBasedTableOptions bbto; - bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); - bbto.whole_key_filtering = false; - last_options_.table_factory.reset(NewBlockBasedTableFactory(bbto)); - ASSERT_OK(TryReopen()); - - ReadOptions ro; - FlushOptions fo; - WriteOptions wo; - - ASSERT_OK(db()->Put(wo, "barbarbar", "foo")); - ASSERT_OK(db()->Put(wo, "barbarbar2", "foo2")); - ASSERT_OK(db()->Put(wo, "foo", "bar")); - ASSERT_OK(db()->Put(wo, "foo3", "bar3")); - ASSERT_OK(db()->Flush(fo)); - - unique_ptr<Iterator> iter(db()->NewIterator(ro)); - - iter->Seek("foo"); - ASSERT_OK(iter->status()); - ASSERT_TRUE(iter->Valid()); - ASSERT_EQ(iter->value().ToString(), "bar"); - ASSERT_EQ(TestGetTickerCount(last_options_, BLOOM_FILTER_PREFIX_USEFUL), 0U); - - iter->Seek("foo2"); - ASSERT_OK(iter->status()); - ASSERT_TRUE(!iter->Valid()); - ASSERT_EQ(TestGetTickerCount(last_options_, BLOOM_FILTER_PREFIX_USEFUL), 1U); - - iter->Seek("barbarbar"); - ASSERT_OK(iter->status()); - ASSERT_TRUE(iter->Valid()); - ASSERT_EQ(iter->value().ToString(), "foo"); - ASSERT_EQ(TestGetTickerCount(last_options_, BLOOM_FILTER_PREFIX_USEFUL), 1U); - - iter->Seek("barfoofoo"); - ASSERT_OK(iter->status()); - ASSERT_TRUE(!iter->Valid()); - ASSERT_EQ(TestGetTickerCount(last_options_, BLOOM_FILTER_PREFIX_USEFUL), 2U); - - iter->Seek("foobarbar"); - ASSERT_OK(iter->status()); - ASSERT_TRUE(!iter->Valid()); - ASSERT_EQ(TestGetTickerCount(last_options_, BLOOM_FILTER_PREFIX_USEFUL), 3U); -} - -} // namespace rocksdb - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/d3a13a49/thirdparty/rocksdb/util/testutil.cc ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/util/testutil.cc b/thirdparty/rocksdb/util/testutil.cc deleted file mode 100644 index f3010f3..0000000 --- a/thirdparty/rocksdb/util/testutil.cc +++ /dev/null @@ -1,401 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). -// -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#include "util/testutil.h" - -#include <cctype> -#include <sstream> - -#include "db/memtable_list.h" -#include "port/port.h" -#include "util/file_reader_writer.h" - -namespace rocksdb { -namespace test { - -Slice RandomString(Random* rnd, int len, std::string* dst) { - dst->resize(len); - for (int i = 0; i < len; i++) { - (*dst)[i] = static_cast<char>(' ' + rnd->Uniform(95)); // ' ' .. '~' - } - return Slice(*dst); -} - -extern std::string RandomHumanReadableString(Random* rnd, int len) { - std::string ret; - ret.resize(len); - for (int i = 0; i < len; ++i) { - ret[i] = static_cast<char>('a' + rnd->Uniform(26)); - } - return ret; -} - -std::string RandomKey(Random* rnd, int len, RandomKeyType type) { - // Make sure to generate a wide variety of characters so we - // test the boundary conditions for short-key optimizations. - static const char kTestChars[] = { - '\0', '\1', 'a', 'b', 'c', 'd', 'e', '\xfd', '\xfe', '\xff' - }; - std::string result; - for (int i = 0; i < len; i++) { - std::size_t indx = 0; - switch (type) { - case RandomKeyType::RANDOM: - indx = rnd->Uniform(sizeof(kTestChars)); - break; - case RandomKeyType::LARGEST: - indx = sizeof(kTestChars) - 1; - break; - case RandomKeyType::MIDDLE: - indx = sizeof(kTestChars) / 2; - break; - case RandomKeyType::SMALLEST: - indx = 0; - break; - } - result += kTestChars[indx]; - } - return result; -} - - -extern Slice CompressibleString(Random* rnd, double compressed_fraction, - int len, std::string* dst) { - int raw = static_cast<int>(len * compressed_fraction); - if (raw < 1) raw = 1; - std::string raw_data; - RandomString(rnd, raw, &raw_data); - - // Duplicate the random data until we have filled "len" bytes - dst->clear(); - while (dst->size() < (unsigned int)len) { - dst->append(raw_data); - } - dst->resize(len); - return Slice(*dst); -} - -namespace { -class Uint64ComparatorImpl : public Comparator { - public: - Uint64ComparatorImpl() { } - - virtual const char* Name() const override { - return "rocksdb.Uint64Comparator"; - } - - virtual int Compare(const Slice& a, const Slice& b) const override { - assert(a.size() == sizeof(uint64_t) && b.size() == sizeof(uint64_t)); - const uint64_t* left = reinterpret_cast<const uint64_t*>(a.data()); - const uint64_t* right = reinterpret_cast<const uint64_t*>(b.data()); - uint64_t leftValue; - uint64_t rightValue; - GetUnaligned(left, &leftValue); - GetUnaligned(right, &rightValue); - if (leftValue == rightValue) { - return 0; - } else if (leftValue < rightValue) { - return -1; - } else { - return 1; - } - } - - virtual void FindShortestSeparator(std::string* start, - const Slice& limit) const override { - return; - } - - virtual void FindShortSuccessor(std::string* key) const override { - return; - } -}; -} // namespace - -static port::OnceType once; -static const Comparator* uint64comp; - -static void InitModule() { - uint64comp = new Uint64ComparatorImpl; -} - -const Comparator* Uint64Comparator() { - port::InitOnce(&once, InitModule); - return uint64comp; -} - -WritableFileWriter* GetWritableFileWriter(WritableFile* wf) { - unique_ptr<WritableFile> file(wf); - return new WritableFileWriter(std::move(file), EnvOptions()); -} - -RandomAccessFileReader* GetRandomAccessFileReader(RandomAccessFile* raf) { - unique_ptr<RandomAccessFile> file(raf); - return new RandomAccessFileReader(std::move(file), - "[test RandomAccessFileReader]"); -} - -SequentialFileReader* GetSequentialFileReader(SequentialFile* se) { - unique_ptr<SequentialFile> file(se); - return new SequentialFileReader(std::move(file)); -} - -void CorruptKeyType(InternalKey* ikey) { - std::string keystr = ikey->Encode().ToString(); - keystr[keystr.size() - 8] = kTypeLogData; - ikey->DecodeFrom(Slice(keystr.data(), keystr.size())); -} - -std::string KeyStr(const std::string& user_key, const SequenceNumber& seq, - const ValueType& t, bool corrupt) { - InternalKey k(user_key, seq, t); - if (corrupt) { - CorruptKeyType(&k); - } - return k.Encode().ToString(); -} - -std::string RandomName(Random* rnd, const size_t len) { - std::stringstream ss; - for (size_t i = 0; i < len; ++i) { - ss << static_cast<char>(rnd->Uniform(26) + 'a'); - } - return ss.str(); -} - -CompressionType RandomCompressionType(Random* rnd) { - return static_cast<CompressionType>(rnd->Uniform(6)); -} - -void RandomCompressionTypeVector(const size_t count, - std::vector<CompressionType>* types, - Random* rnd) { - types->clear(); - for (size_t i = 0; i < count; ++i) { - types->emplace_back(RandomCompressionType(rnd)); - } -} - -const SliceTransform* RandomSliceTransform(Random* rnd, int pre_defined) { - int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(4); - switch (random_num) { - case 0: - return NewFixedPrefixTransform(rnd->Uniform(20) + 1); - case 1: - return NewCappedPrefixTransform(rnd->Uniform(20) + 1); - case 2: - return NewNoopTransform(); - default: - return nullptr; - } -} - -BlockBasedTableOptions RandomBlockBasedTableOptions(Random* rnd) { - BlockBasedTableOptions opt; - opt.cache_index_and_filter_blocks = rnd->Uniform(2); - opt.pin_l0_filter_and_index_blocks_in_cache = rnd->Uniform(2); - opt.index_type = rnd->Uniform(2) ? BlockBasedTableOptions::kBinarySearch - : BlockBasedTableOptions::kHashSearch; - opt.hash_index_allow_collision = rnd->Uniform(2); - opt.checksum = static_cast<ChecksumType>(rnd->Uniform(3)); - opt.block_size = rnd->Uniform(10000000); - opt.block_size_deviation = rnd->Uniform(100); - opt.block_restart_interval = rnd->Uniform(100); - opt.index_block_restart_interval = rnd->Uniform(100); - opt.whole_key_filtering = rnd->Uniform(2); - - return opt; -} - -TableFactory* RandomTableFactory(Random* rnd, int pre_defined) { -#ifndef ROCKSDB_LITE - int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(4); - switch (random_num) { - case 0: - return NewPlainTableFactory(); - case 1: - return NewCuckooTableFactory(); - default: - return NewBlockBasedTableFactory(); - } -#else - return NewBlockBasedTableFactory(); -#endif // !ROCKSDB_LITE -} - -MergeOperator* RandomMergeOperator(Random* rnd) { - return new ChanglingMergeOperator(RandomName(rnd, 10)); -} - -CompactionFilter* RandomCompactionFilter(Random* rnd) { - return new ChanglingCompactionFilter(RandomName(rnd, 10)); -} - -CompactionFilterFactory* RandomCompactionFilterFactory(Random* rnd) { - return new ChanglingCompactionFilterFactory(RandomName(rnd, 10)); -} - -void RandomInitDBOptions(DBOptions* db_opt, Random* rnd) { - // boolean options - db_opt->advise_random_on_open = rnd->Uniform(2); - db_opt->allow_mmap_reads = rnd->Uniform(2); - db_opt->allow_mmap_writes = rnd->Uniform(2); - db_opt->use_direct_reads = rnd->Uniform(2); - db_opt->use_direct_io_for_flush_and_compaction = rnd->Uniform(2); - db_opt->create_if_missing = rnd->Uniform(2); - db_opt->create_missing_column_families = rnd->Uniform(2); - db_opt->enable_thread_tracking = rnd->Uniform(2); - db_opt->error_if_exists = rnd->Uniform(2); - db_opt->is_fd_close_on_exec = rnd->Uniform(2); - db_opt->paranoid_checks = rnd->Uniform(2); - db_opt->skip_log_error_on_recovery = rnd->Uniform(2); - db_opt->skip_stats_update_on_db_open = rnd->Uniform(2); - db_opt->use_adaptive_mutex = rnd->Uniform(2); - db_opt->use_fsync = rnd->Uniform(2); - db_opt->recycle_log_file_num = rnd->Uniform(2); - db_opt->avoid_flush_during_recovery = rnd->Uniform(2); - db_opt->avoid_flush_during_shutdown = rnd->Uniform(2); - - // int options - db_opt->max_background_compactions = rnd->Uniform(100); - db_opt->max_background_flushes = rnd->Uniform(100); - db_opt->max_file_opening_threads = rnd->Uniform(100); - db_opt->max_open_files = rnd->Uniform(100); - db_opt->table_cache_numshardbits = rnd->Uniform(100); - - // size_t options - db_opt->db_write_buffer_size = rnd->Uniform(10000); - db_opt->keep_log_file_num = rnd->Uniform(10000); - db_opt->log_file_time_to_roll = rnd->Uniform(10000); - db_opt->manifest_preallocation_size = rnd->Uniform(10000); - db_opt->max_log_file_size = rnd->Uniform(10000); - - // std::string options - db_opt->db_log_dir = "path/to/db_log_dir"; - db_opt->wal_dir = "path/to/wal_dir"; - - // uint32_t options - db_opt->max_subcompactions = rnd->Uniform(100000); - - // uint64_t options - static const uint64_t uint_max = static_cast<uint64_t>(UINT_MAX); - db_opt->WAL_size_limit_MB = uint_max + rnd->Uniform(100000); - db_opt->WAL_ttl_seconds = uint_max + rnd->Uniform(100000); - db_opt->bytes_per_sync = uint_max + rnd->Uniform(100000); - db_opt->delayed_write_rate = uint_max + rnd->Uniform(100000); - db_opt->delete_obsolete_files_period_micros = uint_max + rnd->Uniform(100000); - db_opt->max_manifest_file_size = uint_max + rnd->Uniform(100000); - db_opt->max_total_wal_size = uint_max + rnd->Uniform(100000); - db_opt->wal_bytes_per_sync = uint_max + rnd->Uniform(100000); - - // unsigned int options - db_opt->stats_dump_period_sec = rnd->Uniform(100000); -} - -void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) { - cf_opt->compaction_style = (CompactionStyle)(rnd->Uniform(4)); - - // boolean options - cf_opt->report_bg_io_stats = rnd->Uniform(2); - cf_opt->disable_auto_compactions = rnd->Uniform(2); - cf_opt->inplace_update_support = rnd->Uniform(2); - cf_opt->level_compaction_dynamic_level_bytes = rnd->Uniform(2); - cf_opt->optimize_filters_for_hits = rnd->Uniform(2); - cf_opt->paranoid_file_checks = rnd->Uniform(2); - cf_opt->purge_redundant_kvs_while_flush = rnd->Uniform(2); - cf_opt->force_consistency_checks = rnd->Uniform(2); - - // double options - cf_opt->hard_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13; - cf_opt->soft_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13; - cf_opt->memtable_prefix_bloom_size_ratio = - static_cast<double>(rnd->Uniform(10000)) / 20000.0; - - // int options - cf_opt->level0_file_num_compaction_trigger = rnd->Uniform(100); - cf_opt->level0_slowdown_writes_trigger = rnd->Uniform(100); - cf_opt->level0_stop_writes_trigger = rnd->Uniform(100); - cf_opt->max_bytes_for_level_multiplier = rnd->Uniform(100); - cf_opt->max_mem_compaction_level = rnd->Uniform(100); - cf_opt->max_write_buffer_number = rnd->Uniform(100); - cf_opt->max_write_buffer_number_to_maintain = rnd->Uniform(100); - cf_opt->min_write_buffer_number_to_merge = rnd->Uniform(100); - cf_opt->num_levels = rnd->Uniform(100); - cf_opt->target_file_size_multiplier = rnd->Uniform(100); - - // vector int options - cf_opt->max_bytes_for_level_multiplier_additional.resize(cf_opt->num_levels); - for (int i = 0; i < cf_opt->num_levels; i++) { - cf_opt->max_bytes_for_level_multiplier_additional[i] = rnd->Uniform(100); - } - - // size_t options - cf_opt->arena_block_size = rnd->Uniform(10000); - cf_opt->inplace_update_num_locks = rnd->Uniform(10000); - cf_opt->max_successive_merges = rnd->Uniform(10000); - cf_opt->memtable_huge_page_size = rnd->Uniform(10000); - cf_opt->write_buffer_size = rnd->Uniform(10000); - - // uint32_t options - cf_opt->bloom_locality = rnd->Uniform(10000); - cf_opt->max_bytes_for_level_base = rnd->Uniform(10000); - - // uint64_t options - static const uint64_t uint_max = static_cast<uint64_t>(UINT_MAX); - cf_opt->max_sequential_skip_in_iterations = uint_max + rnd->Uniform(10000); - cf_opt->target_file_size_base = uint_max + rnd->Uniform(10000); - cf_opt->max_compaction_bytes = - cf_opt->target_file_size_base * rnd->Uniform(100); - - // unsigned int options - cf_opt->rate_limit_delay_max_milliseconds = rnd->Uniform(10000); - - // pointer typed options - cf_opt->prefix_extractor.reset(RandomSliceTransform(rnd)); - cf_opt->table_factory.reset(RandomTableFactory(rnd)); - cf_opt->merge_operator.reset(RandomMergeOperator(rnd)); - if (cf_opt->compaction_filter) { - delete cf_opt->compaction_filter; - } - cf_opt->compaction_filter = RandomCompactionFilter(rnd); - cf_opt->compaction_filter_factory.reset(RandomCompactionFilterFactory(rnd)); - - // custom typed options - cf_opt->compression = RandomCompressionType(rnd); - RandomCompressionTypeVector(cf_opt->num_levels, - &cf_opt->compression_per_level, rnd); -} - -Status DestroyDir(Env* env, const std::string& dir) { - Status s; - if (env->FileExists(dir).IsNotFound()) { - return s; - } - std::vector<std::string> files_in_dir; - s = env->GetChildren(dir, &files_in_dir); - if (s.ok()) { - for (auto& file_in_dir : files_in_dir) { - if (file_in_dir == "." || file_in_dir == "..") { - continue; - } - s = env->DeleteFile(dir + "/" + file_in_dir); - if (!s.ok()) { - break; - } - } - } - - if (s.ok()) { - s = env->DeleteDir(dir); - } - return s; -} - -} // namespace test -} // namespace rocksdb http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/d3a13a49/thirdparty/rocksdb/util/testutil.h ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/util/testutil.h b/thirdparty/rocksdb/util/testutil.h deleted file mode 100644 index 02bfb0f..0000000 --- a/thirdparty/rocksdb/util/testutil.h +++ /dev/null @@ -1,744 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). -// -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#pragma once -#include <algorithm> -#include <deque> -#include <string> -#include <vector> - -#include "rocksdb/compaction_filter.h" -#include "rocksdb/env.h" -#include "rocksdb/iterator.h" -#include "rocksdb/merge_operator.h" -#include "rocksdb/options.h" -#include "rocksdb/slice.h" -#include "rocksdb/table.h" -#include "table/block_based_table_factory.h" -#include "table/internal_iterator.h" -#include "table/plain_table_factory.h" -#include "util/mutexlock.h" -#include "util/random.h" - -namespace rocksdb { -class SequentialFile; -class SequentialFileReader; - -namespace test { - -// Store in *dst a random string of length "len" and return a Slice that -// references the generated data. -extern Slice RandomString(Random* rnd, int len, std::string* dst); - -extern std::string RandomHumanReadableString(Random* rnd, int len); - -// Return a random key with the specified length that may contain interesting -// characters (e.g. \x00, \xff, etc.). -enum RandomKeyType : char { RANDOM, LARGEST, SMALLEST, MIDDLE }; -extern std::string RandomKey(Random* rnd, int len, - RandomKeyType type = RandomKeyType::RANDOM); - -// Store in *dst a string of length "len" that will compress to -// "N*compressed_fraction" bytes and return a Slice that references -// the generated data. -extern Slice CompressibleString(Random* rnd, double compressed_fraction, - int len, std::string* dst); - -// A wrapper that allows injection of errors. -class ErrorEnv : public EnvWrapper { - public: - bool writable_file_error_; - int num_writable_file_errors_; - - ErrorEnv() : EnvWrapper(Env::Default()), - writable_file_error_(false), - num_writable_file_errors_(0) { } - - virtual Status NewWritableFile(const std::string& fname, - unique_ptr<WritableFile>* result, - const EnvOptions& soptions) override { - result->reset(); - if (writable_file_error_) { - ++num_writable_file_errors_; - return Status::IOError(fname, "fake error"); - } - return target()->NewWritableFile(fname, result, soptions); - } -}; - -// An internal comparator that just forward comparing results from the -// user comparator in it. Can be used to test entities that have no dependency -// on internal key structure but consumes InternalKeyComparator, like -// BlockBasedTable. -class PlainInternalKeyComparator : public InternalKeyComparator { - public: - explicit PlainInternalKeyComparator(const Comparator* c) - : InternalKeyComparator(c) {} - - virtual ~PlainInternalKeyComparator() {} - - virtual int Compare(const Slice& a, const Slice& b) const override { - return user_comparator()->Compare(a, b); - } - virtual void FindShortestSeparator(std::string* start, - const Slice& limit) const override { - user_comparator()->FindShortestSeparator(start, limit); - } - virtual void FindShortSuccessor(std::string* key) const override { - user_comparator()->FindShortSuccessor(key); - } -}; - -// A test comparator which compare two strings in this way: -// (1) first compare prefix of 8 bytes in alphabet order, -// (2) if two strings share the same prefix, sort the other part of the string -// in the reverse alphabet order. -// This helps simulate the case of compounded key of [entity][timestamp] and -// latest timestamp first. -class SimpleSuffixReverseComparator : public Comparator { - public: - SimpleSuffixReverseComparator() {} - - virtual const char* Name() const override { - return "SimpleSuffixReverseComparator"; - } - - virtual int Compare(const Slice& a, const Slice& b) const override { - Slice prefix_a = Slice(a.data(), 8); - Slice prefix_b = Slice(b.data(), 8); - int prefix_comp = prefix_a.compare(prefix_b); - if (prefix_comp != 0) { - return prefix_comp; - } else { - Slice suffix_a = Slice(a.data() + 8, a.size() - 8); - Slice suffix_b = Slice(b.data() + 8, b.size() - 8); - return -(suffix_a.compare(suffix_b)); - } - } - virtual void FindShortestSeparator(std::string* start, - const Slice& limit) const override {} - - virtual void FindShortSuccessor(std::string* key) const override {} -}; - -// Returns a user key comparator that can be used for comparing two uint64_t -// slices. Instead of comparing slices byte-wise, it compares all the 8 bytes -// at once. Assumes same endian-ness is used though the database's lifetime. -// Symantics of comparison would differ from Bytewise comparator in little -// endian machines. -extern const Comparator* Uint64Comparator(); - -// Iterator over a vector of keys/values -class VectorIterator : public InternalIterator { - public: - explicit VectorIterator(const std::vector<std::string>& keys) - : keys_(keys), current_(keys.size()) { - std::sort(keys_.begin(), keys_.end()); - values_.resize(keys.size()); - } - - VectorIterator(const std::vector<std::string>& keys, - const std::vector<std::string>& values) - : keys_(keys), values_(values), current_(keys.size()) { - assert(keys_.size() == values_.size()); - } - - virtual bool Valid() const override { return current_ < keys_.size(); } - - virtual void SeekToFirst() override { current_ = 0; } - virtual void SeekToLast() override { current_ = keys_.size() - 1; } - - virtual void Seek(const Slice& target) override { - current_ = std::lower_bound(keys_.begin(), keys_.end(), target.ToString()) - - keys_.begin(); - } - - virtual void SeekForPrev(const Slice& target) override { - current_ = std::upper_bound(keys_.begin(), keys_.end(), target.ToString()) - - keys_.begin(); - if (!Valid()) { - SeekToLast(); - } else { - Prev(); - } - } - - virtual void Next() override { current_++; } - virtual void Prev() override { current_--; } - - virtual Slice key() const override { return Slice(keys_[current_]); } - virtual Slice value() const override { return Slice(values_[current_]); } - - virtual Status status() const override { return Status::OK(); } - - private: - std::vector<std::string> keys_; - std::vector<std::string> values_; - size_t current_; -}; -extern WritableFileWriter* GetWritableFileWriter(WritableFile* wf); - -extern RandomAccessFileReader* GetRandomAccessFileReader(RandomAccessFile* raf); - -extern SequentialFileReader* GetSequentialFileReader(SequentialFile* se); - -class StringSink: public WritableFile { - public: - std::string contents_; - - explicit StringSink(Slice* reader_contents = nullptr) : - WritableFile(), - contents_(""), - reader_contents_(reader_contents), - last_flush_(0) { - if (reader_contents_ != nullptr) { - *reader_contents_ = Slice(contents_.data(), 0); - } - } - - const std::string& contents() const { return contents_; } - - virtual Status Truncate(uint64_t size) override { - contents_.resize(static_cast<size_t>(size)); - return Status::OK(); - } - virtual Status Close() override { return Status::OK(); } - virtual Status Flush() override { - if (reader_contents_ != nullptr) { - assert(reader_contents_->size() <= last_flush_); - size_t offset = last_flush_ - reader_contents_->size(); - *reader_contents_ = Slice( - contents_.data() + offset, - contents_.size() - offset); - last_flush_ = contents_.size(); - } - - return Status::OK(); - } - virtual Status Sync() override { return Status::OK(); } - virtual Status Append(const Slice& slice) override { - contents_.append(slice.data(), slice.size()); - return Status::OK(); - } - void Drop(size_t bytes) { - if (reader_contents_ != nullptr) { - contents_.resize(contents_.size() - bytes); - *reader_contents_ = Slice( - reader_contents_->data(), reader_contents_->size() - bytes); - last_flush_ = contents_.size(); - } - } - - private: - Slice* reader_contents_; - size_t last_flush_; -}; - -// A wrapper around a StringSink to give it a RandomRWFile interface -class RandomRWStringSink : public RandomRWFile { - public: - explicit RandomRWStringSink(StringSink* ss) : ss_(ss) {} - - Status Write(uint64_t offset, const Slice& data) { - if (offset + data.size() > ss_->contents_.size()) { - ss_->contents_.resize(offset + data.size(), '\0'); - } - - char* pos = const_cast<char*>(ss_->contents_.data() + offset); - memcpy(pos, data.data(), data.size()); - return Status::OK(); - } - - Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const { - *result = Slice(nullptr, 0); - if (offset < ss_->contents_.size()) { - size_t str_res_sz = - std::min(static_cast<size_t>(ss_->contents_.size() - offset), n); - *result = Slice(ss_->contents_.data() + offset, str_res_sz); - } - return Status::OK(); - } - - Status Flush() { return Status::OK(); } - - Status Sync() { return Status::OK(); } - - Status Close() { return Status::OK(); } - - const std::string& contents() const { return ss_->contents(); } - - private: - StringSink* ss_; -}; - -// Like StringSink, this writes into a string. Unlink StringSink, it -// has some initial content and overwrites it, just like a recycled -// log file. -class OverwritingStringSink : public WritableFile { - public: - explicit OverwritingStringSink(Slice* reader_contents) - : WritableFile(), - contents_(""), - reader_contents_(reader_contents), - last_flush_(0) {} - - const std::string& contents() const { return contents_; } - - virtual Status Truncate(uint64_t size) override { - contents_.resize(static_cast<size_t>(size)); - return Status::OK(); - } - virtual Status Close() override { return Status::OK(); } - virtual Status Flush() override { - if (last_flush_ < contents_.size()) { - assert(reader_contents_->size() >= contents_.size()); - memcpy((char*)reader_contents_->data() + last_flush_, - contents_.data() + last_flush_, contents_.size() - last_flush_); - last_flush_ = contents_.size(); - } - return Status::OK(); - } - virtual Status Sync() override { return Status::OK(); } - virtual Status Append(const Slice& slice) override { - contents_.append(slice.data(), slice.size()); - return Status::OK(); - } - void Drop(size_t bytes) { - contents_.resize(contents_.size() - bytes); - if (last_flush_ > contents_.size()) last_flush_ = contents_.size(); - } - - private: - std::string contents_; - Slice* reader_contents_; - size_t last_flush_; -}; - -class StringSource: public RandomAccessFile { - public: - explicit StringSource(const Slice& contents, uint64_t uniq_id = 0, - bool mmap = false) - : contents_(contents.data(), contents.size()), - uniq_id_(uniq_id), - mmap_(mmap), - total_reads_(0) {} - - virtual ~StringSource() { } - - uint64_t Size() const { return contents_.size(); } - - virtual Status Read(uint64_t offset, size_t n, Slice* result, - char* scratch) const override { - total_reads_++; - if (offset > contents_.size()) { - return Status::InvalidArgument("invalid Read offset"); - } - if (offset + n > contents_.size()) { - n = contents_.size() - static_cast<size_t>(offset); - } - if (!mmap_) { - memcpy(scratch, &contents_[static_cast<size_t>(offset)], n); - *result = Slice(scratch, n); - } else { - *result = Slice(&contents_[static_cast<size_t>(offset)], n); - } - return Status::OK(); - } - - virtual size_t GetUniqueId(char* id, size_t max_size) const override { - if (max_size < 20) { - return 0; - } - - char* rid = id; - rid = EncodeVarint64(rid, uniq_id_); - rid = EncodeVarint64(rid, 0); - return static_cast<size_t>(rid-id); - } - - int total_reads() const { return total_reads_; } - - void set_total_reads(int tr) { total_reads_ = tr; } - - private: - std::string contents_; - uint64_t uniq_id_; - bool mmap_; - mutable int total_reads_; -}; - -class NullLogger : public Logger { - public: - using Logger::Logv; - virtual void Logv(const char* format, va_list ap) override {} - virtual size_t GetLogFileSize() const override { return 0; } -}; - -// Corrupts key by changing the type -extern void CorruptKeyType(InternalKey* ikey); - -extern std::string KeyStr(const std::string& user_key, - const SequenceNumber& seq, const ValueType& t, - bool corrupt = false); - -class SleepingBackgroundTask { - public: - SleepingBackgroundTask() - : bg_cv_(&mutex_), - should_sleep_(true), - done_with_sleep_(false), - sleeping_(false) {} - - bool IsSleeping() { - MutexLock l(&mutex_); - return sleeping_; - } - void DoSleep() { - MutexLock l(&mutex_); - sleeping_ = true; - bg_cv_.SignalAll(); - while (should_sleep_) { - bg_cv_.Wait(); - } - sleeping_ = false; - done_with_sleep_ = true; - bg_cv_.SignalAll(); - } - void WaitUntilSleeping() { - MutexLock l(&mutex_); - while (!sleeping_ || !should_sleep_) { - bg_cv_.Wait(); - } - } - void WakeUp() { - MutexLock l(&mutex_); - should_sleep_ = false; - bg_cv_.SignalAll(); - } - void WaitUntilDone() { - MutexLock l(&mutex_); - while (!done_with_sleep_) { - bg_cv_.Wait(); - } - } - bool WokenUp() { - MutexLock l(&mutex_); - return should_sleep_ == false; - } - - void Reset() { - MutexLock l(&mutex_); - should_sleep_ = true; - done_with_sleep_ = false; - } - - static void DoSleepTask(void* arg) { - reinterpret_cast<SleepingBackgroundTask*>(arg)->DoSleep(); - } - - private: - port::Mutex mutex_; - port::CondVar bg_cv_; // Signalled when background work finishes - bool should_sleep_; - bool done_with_sleep_; - bool sleeping_; -}; - -// Filters merge operands and values that are equal to `num`. -class FilterNumber : public CompactionFilter { - public: - explicit FilterNumber(uint64_t num) : num_(num) {} - - std::string last_merge_operand_key() { return last_merge_operand_key_; } - - bool Filter(int level, const rocksdb::Slice& key, const rocksdb::Slice& value, - std::string* new_value, bool* value_changed) const override { - if (value.size() == sizeof(uint64_t)) { - return num_ == DecodeFixed64(value.data()); - } - return true; - } - - bool FilterMergeOperand(int level, const rocksdb::Slice& key, - const rocksdb::Slice& value) const override { - last_merge_operand_key_ = key.ToString(); - if (value.size() == sizeof(uint64_t)) { - return num_ == DecodeFixed64(value.data()); - } - return true; - } - - const char* Name() const override { return "FilterBadMergeOperand"; } - - private: - mutable std::string last_merge_operand_key_; - uint64_t num_; -}; - -inline std::string EncodeInt(uint64_t x) { - std::string result; - PutFixed64(&result, x); - return result; -} - -class StringEnv : public EnvWrapper { - public: - class SeqStringSource : public SequentialFile { - public: - explicit SeqStringSource(const std::string& data) - : data_(data), offset_(0) {} - ~SeqStringSource() {} - Status Read(size_t n, Slice* result, char* scratch) override { - std::string output; - if (offset_ < data_.size()) { - n = std::min(data_.size() - offset_, n); - memcpy(scratch, data_.data() + offset_, n); - offset_ += n; - *result = Slice(scratch, n); - } else { - return Status::InvalidArgument( - "Attemp to read when it already reached eof."); - } - return Status::OK(); - } - Status Skip(uint64_t n) override { - if (offset_ >= data_.size()) { - return Status::InvalidArgument( - "Attemp to read when it already reached eof."); - } - // TODO(yhchiang): Currently doesn't handle the overflow case. - offset_ += n; - return Status::OK(); - } - - private: - std::string data_; - size_t offset_; - }; - - class StringSink : public WritableFile { - public: - explicit StringSink(std::string* contents) - : WritableFile(), contents_(contents) {} - virtual Status Truncate(uint64_t size) override { - contents_->resize(size); - return Status::OK(); - } - virtual Status Close() override { return Status::OK(); } - virtual Status Flush() override { return Status::OK(); } - virtual Status Sync() override { return Status::OK(); } - virtual Status Append(const Slice& slice) override { - contents_->append(slice.data(), slice.size()); - return Status::OK(); - } - - private: - std::string* contents_; - }; - - explicit StringEnv(Env* t) : EnvWrapper(t) {} - virtual ~StringEnv() {} - - const std::string& GetContent(const std::string& f) { return files_[f]; } - - const Status WriteToNewFile(const std::string& file_name, - const std::string& content) { - unique_ptr<WritableFile> r; - auto s = NewWritableFile(file_name, &r, EnvOptions()); - if (!s.ok()) { - return s; - } - r->Append(content); - r->Flush(); - r->Close(); - assert(files_[file_name] == content); - return Status::OK(); - } - - // The following text is boilerplate that forwards all methods to target() - Status NewSequentialFile(const std::string& f, unique_ptr<SequentialFile>* r, - const EnvOptions& options) override { - auto iter = files_.find(f); - if (iter == files_.end()) { - return Status::NotFound("The specified file does not exist", f); - } - r->reset(new SeqStringSource(iter->second)); - return Status::OK(); - } - Status NewRandomAccessFile(const std::string& f, - unique_ptr<RandomAccessFile>* r, - const EnvOptions& options) override { - return Status::NotSupported(); - } - Status NewWritableFile(const std::string& f, unique_ptr<WritableFile>* r, - const EnvOptions& options) override { - auto iter = files_.find(f); - if (iter != files_.end()) { - return Status::IOError("The specified file already exists", f); - } - r->reset(new StringSink(&files_[f])); - return Status::OK(); - } - virtual Status NewDirectory(const std::string& name, - unique_ptr<Directory>* result) override { - return Status::NotSupported(); - } - Status FileExists(const std::string& f) override { - if (files_.find(f) == files_.end()) { - return Status::NotFound(); - } - return Status::OK(); - } - Status GetChildren(const std::string& dir, - std::vector<std::string>* r) override { - return Status::NotSupported(); - } - Status DeleteFile(const std::string& f) override { - files_.erase(f); - return Status::OK(); - } - Status CreateDir(const std::string& d) override { - return Status::NotSupported(); - } - Status CreateDirIfMissing(const std::string& d) override { - return Status::NotSupported(); - } - Status DeleteDir(const std::string& d) override { - return Status::NotSupported(); - } - Status GetFileSize(const std::string& f, uint64_t* s) override { - auto iter = files_.find(f); - if (iter == files_.end()) { - return Status::NotFound("The specified file does not exist:", f); - } - *s = iter->second.size(); - return Status::OK(); - } - - Status GetFileModificationTime(const std::string& fname, - uint64_t* file_mtime) override { - return Status::NotSupported(); - } - - Status RenameFile(const std::string& s, const std::string& t) override { - return Status::NotSupported(); - } - - Status LinkFile(const std::string& s, const std::string& t) override { - return Status::NotSupported(); - } - - Status LockFile(const std::string& f, FileLock** l) override { - return Status::NotSupported(); - } - - Status UnlockFile(FileLock* l) override { return Status::NotSupported(); } - - protected: - std::unordered_map<std::string, std::string> files_; -}; - -// Randomly initialize the given DBOptions -void RandomInitDBOptions(DBOptions* db_opt, Random* rnd); - -// Randomly initialize the given ColumnFamilyOptions -// Note that the caller is responsible for releasing non-null -// cf_opt->compaction_filter. -void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd); - -// A dummy merge operator which can change its name -class ChanglingMergeOperator : public MergeOperator { - public: - explicit ChanglingMergeOperator(const std::string& name) - : name_(name + "MergeOperator") {} - ~ChanglingMergeOperator() {} - - void SetName(const std::string& name) { name_ = name; } - - virtual bool FullMergeV2(const MergeOperationInput& merge_in, - MergeOperationOutput* merge_out) const override { - return false; - } - virtual bool PartialMergeMulti(const Slice& key, - const std::deque<Slice>& operand_list, - std::string* new_value, - Logger* logger) const override { - return false; - } - virtual const char* Name() const override { return name_.c_str(); } - - protected: - std::string name_; -}; - -// Returns a dummy merge operator with random name. -MergeOperator* RandomMergeOperator(Random* rnd); - -// A dummy compaction filter which can change its name -class ChanglingCompactionFilter : public CompactionFilter { - public: - explicit ChanglingCompactionFilter(const std::string& name) - : name_(name + "CompactionFilter") {} - ~ChanglingCompactionFilter() {} - - void SetName(const std::string& name) { name_ = name; } - - bool Filter(int level, const Slice& key, const Slice& existing_value, - std::string* new_value, bool* value_changed) const override { - return false; - } - - const char* Name() const override { return name_.c_str(); } - - private: - std::string name_; -}; - -// Returns a dummy compaction filter with a random name. -CompactionFilter* RandomCompactionFilter(Random* rnd); - -// A dummy compaction filter factory which can change its name -class ChanglingCompactionFilterFactory : public CompactionFilterFactory { - public: - explicit ChanglingCompactionFilterFactory(const std::string& name) - : name_(name + "CompactionFilterFactory") {} - ~ChanglingCompactionFilterFactory() {} - - void SetName(const std::string& name) { name_ = name; } - - std::unique_ptr<CompactionFilter> CreateCompactionFilter( - const CompactionFilter::Context& context) override { - return std::unique_ptr<CompactionFilter>(); - } - - // Returns a name that identifies this compaction filter factory. - const char* Name() const override { return name_.c_str(); } - - protected: - std::string name_; -}; - -CompressionType RandomCompressionType(Random* rnd); - -void RandomCompressionTypeVector(const size_t count, - std::vector<CompressionType>* types, - Random* rnd); - -CompactionFilterFactory* RandomCompactionFilterFactory(Random* rnd); - -const SliceTransform* RandomSliceTransform(Random* rnd, int pre_defined = -1); - -TableFactory* RandomTableFactory(Random* rnd, int pre_defined = -1); - -std::string RandomName(Random* rnd, const size_t len); - -Status DestroyDir(Env* env, const std::string& dir); - -} // namespace test -} // namespace rocksdb
