This is an automated email from the ASF dual-hosted git repository. junchao pushed a commit to branch cassandra in repository https://gitbox.apache.org/repos/asf/incubator-resilientdb.git
commit a082d4416ce98be29a538f54f89855ed33cd2def Author: cjcchen <[email protected]> AuthorDate: Tue Oct 1 09:33:52 2024 +0000 add multipaxos --- benchmark/protocols/multipaxos/BUILD | 16 ++ .../protocols/multipaxos/kv_server_performance.cpp | 88 +++++++ .../ordering/cassandra/algorithm/cassandra.cpp | 5 +- .../consensus/ordering/multipaxos/algorithm/BUILD | 39 ++++ .../ordering/multipaxos/algorithm/multipaxos.cpp | 174 ++++++++++++++ .../ordering/multipaxos/algorithm/multipaxos.h | 60 +++++ .../multipaxos/algorithm/proposal_manager.cpp | 36 +++ .../multipaxos/algorithm/proposal_manager.h | 25 ++ .../consensus/ordering/multipaxos/framework/BUILD | 20 ++ .../ordering/multipaxos/framework/consensus.cpp | 113 +++++++++ .../ordering/multipaxos/framework/consensus.h | 53 +++++ platform/consensus/ordering/multipaxos/proto/BUILD | 16 ++ .../ordering/multipaxos/proto/proposal.proto | 35 +++ .../consensus/ordering/tusk/algorithm/tusk.cpp | 16 +- scripts/deploy/config/cassandra.config | 4 +- scripts/deploy/config/kv_performance_server.conf | 10 +- .../deploy/config/kv_performance_server_16.conf | 64 +++--- .../deploy/config/kv_performance_server_32.conf | 128 +++++------ .../deploy/config/kv_performance_server_64.conf | 256 ++++++++++----------- .../config/{rcc.config => multipaxos.config} | 2 +- scripts/deploy/config/rcc.config | 2 +- scripts/deploy/config/tusk.config | 6 +- .../deploy/performance/multipaxos_performance.sh | 6 + scripts/deploy/performance/set_delay.sh | 91 ++++++++ 24 files changed, 1025 insertions(+), 240 deletions(-) diff --git a/benchmark/protocols/multipaxos/BUILD b/benchmark/protocols/multipaxos/BUILD new file mode 100644 index 00000000..69b373d8 --- /dev/null +++ b/benchmark/protocols/multipaxos/BUILD @@ -0,0 +1,16 @@ +package(default_visibility = ["//visibility:private"]) + +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") + +cc_binary( + name = "kv_server_performance", + srcs = ["kv_server_performance.cpp"], + deps = [ + "//chain/storage:memory_db", + "//executor/kv:kv_executor", + "//platform/config:resdb_config_utils", + "//platform/consensus/ordering/multipaxos/framework:consensus", + "//service/utils:server_factory", + ], +) + diff --git a/benchmark/protocols/multipaxos/kv_server_performance.cpp b/benchmark/protocols/multipaxos/kv_server_performance.cpp new file mode 100644 index 00000000..17fba4d1 --- /dev/null +++ b/benchmark/protocols/multipaxos/kv_server_performance.cpp @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2019-2022 ExpoLab, UC Davis + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + */ + +#include <glog/logging.h> + +#include "chain/storage/memory_db.h" +#include "executor/kv/kv_executor.h" +#include "platform/config/resdb_config_utils.h" +#include "platform/consensus/ordering/multipaxos/framework/consensus.h" +#include "platform/networkstrate/service_network.h" +#include "platform/statistic/stats.h" +#include "proto/kv/kv.pb.h" + +using namespace resdb; +using namespace resdb::multipaxos; +using namespace resdb::storage; + +void ShowUsage() { + printf("<config> <private_key> <cert_file> [logging_dir]\n"); +} + +std::string GetRandomKey() { + int num1 = rand() % 10; + int num2 = rand() % 10; + return std::to_string(num1) + std::to_string(num2); +} + +int main(int argc, char** argv) { + if (argc < 3) { + ShowUsage(); + exit(0); + } + + // google::InitGoogleLogging(argv[0]); + // FLAGS_minloglevel = google::GLOG_WARNING; + + char* config_file = argv[1]; + char* private_key_file = argv[2]; + char* cert_file = argv[3]; + + if (argc >= 5) { + auto monitor_port = Stats::GetGlobalStats(5); + monitor_port->SetPrometheus(argv[4]); + } + + std::unique_ptr<ResDBConfig> config = + GenerateResDBConfig(config_file, private_key_file, cert_file); + + config->RunningPerformance(true); + + auto performance_consens = std::make_unique<multipaxos::Consensus>( + *config, std::make_unique<KVExecutor>(std::make_unique<MemoryDB>())); + performance_consens->SetupPerformanceDataFunc([]() { + KVRequest request; + request.set_cmd(KVRequest::SET); + request.set_key(GetRandomKey()); + request.set_value("helloworld"); + std::string request_data; + request.SerializeToString(&request_data); + return request_data; + }); + + auto server = + std::make_unique<ServiceNetwork>(*config, std::move(performance_consens)); + server->Run(); +} diff --git a/platform/consensus/ordering/cassandra/algorithm/cassandra.cpp b/platform/consensus/ordering/cassandra/algorithm/cassandra.cpp index 7b32c5dc..f32ba2d9 100644 --- a/platform/consensus/ordering/cassandra/algorithm/cassandra.cpp +++ b/platform/consensus/ordering/cassandra/algorithm/cassandra.cpp @@ -17,10 +17,11 @@ Cassandra::Cassandra(int id, int f, int total_num, SignatureVerifier* verifier) total_num_ = total_num; f_ = f; is_stop_ = false; + //timeout_ms_ = 10000; timeout_ms_ = 60000; local_txn_id_ = 1; local_proposal_id_ = 1; - batch_size_ = 10; + batch_size_ = 15; recv_num_ = 0; execute_num_ = 0; @@ -248,7 +249,7 @@ void Cassandra::BroadcastTxn() { */ for(int i = 1; i < batch_size_; ++i){ - std::unique_ptr<Transaction> txn = txns_.Pop(10); + std::unique_ptr<Transaction> txn = txns_.Pop(100); if(txn == nullptr){ break; } diff --git a/platform/consensus/ordering/multipaxos/algorithm/BUILD b/platform/consensus/ordering/multipaxos/algorithm/BUILD new file mode 100644 index 00000000..6eef34d4 --- /dev/null +++ b/platform/consensus/ordering/multipaxos/algorithm/BUILD @@ -0,0 +1,39 @@ +package(default_visibility = ["//platform/consensus/ordering/multipaxos:__subpackages__"]) + +cc_library( + name = "proposal_manager", + srcs = ["proposal_manager.cpp"], + hdrs = ["proposal_manager.h"], + deps = [ + "//common:comm", + "//common/crypto:signature_verifier", + "//common/utils", + "//platform/consensus/ordering/multipaxos/proto:proposal_cc_proto", + ], +) + +cc_library( + name = "multipaxos", + srcs = ["multipaxos.cpp"], + hdrs = ["multipaxos.h"], + deps = [ + ":proposal_manager", + "//common:comm", + "//platform/common/queue:lock_free_queue", + "//platform/consensus/ordering/common/algorithm:protocol_base", + ], +) + +cc_library( + name = "tusk", + srcs = ["tusk.cpp"], + hdrs = ["tusk.h"], + deps = [ + ":proposal_manager", + "//common/crypto:signature_verifier", + "//common:comm", + "//platform/common/queue:lock_free_queue", + "//platform/consensus/ordering/common/algorithm:protocol_base", + ], +) + diff --git a/platform/consensus/ordering/multipaxos/algorithm/multipaxos.cpp b/platform/consensus/ordering/multipaxos/algorithm/multipaxos.cpp new file mode 100644 index 00000000..a7966898 --- /dev/null +++ b/platform/consensus/ordering/multipaxos/algorithm/multipaxos.cpp @@ -0,0 +1,174 @@ +#include "platform/consensus/ordering/multipaxos/algorithm/multipaxos.h" + +#include <glog/logging.h> +#include "common/utils/utils.h" + + +namespace resdb { +namespace multipaxos { + +MultiPaxos::MultiPaxos(int id, int f, int total_num, SignatureVerifier * verifier) + : ProtocolBase(id, f, total_num){ + + LOG(ERROR)<<"id:"<<id<<" f:"<<f<<" total:"<<total_num_; + + proposal_manager_ = std::make_unique<ProposalManager>(id, 2*f_+1, verifier); + send_thread_ = std::thread(&MultiPaxos::AsyncSend, this); + commit_thread_ = std::thread(&MultiPaxos::AsyncCommit, this); + commit_seq_thread_ = std::thread(&MultiPaxos::AsyncCommitSeq, this); + batch_size_ = 10; + start_seq_ = 1; + } + +MultiPaxos::~MultiPaxos() { +} + + +void MultiPaxos::AsyncSend() { + while (!IsStop()) { + auto txn = txns_.Pop(); + if(txn == nullptr){ + continue; + } + + std::vector<std::unique_ptr<Transaction> > txns; + txns.push_back(std::move(txn)); + for(int i = 1; i < batch_size_; ++i){ + auto txn = txns_.Pop(); + if(txn == nullptr){ + break; + } + txns.push_back(std::move(txn)); + } + + std::unique_ptr<Proposal> proposal = nullptr; + { + proposal = proposal_manager_ -> GenerateProposal(txns); + //LOG(ERROR)<<"propose view:"<<proposal->header().view(); + } + broadcast_call_(MessageType::Propose, *proposal); + } +} + + +void MultiPaxos::AddCommitData(std::unique_ptr<Proposal> p){ + std::unique_lock<std::mutex> lk(n_mutex_); + commit_data_[p->header().proposal_id()] = std::move(p); + vote_cv_.notify_one(); +} + +std::unique_ptr<Proposal> MultiPaxos::GetCommitData(){ + std::unique_lock<std::mutex> lk(n_mutex_); + auto p = std::move(commit_data_.begin()->second); + commit_data_.erase(commit_data_.begin()); + start_seq_++; + return p; +} + +bool MultiPaxos::Wait() { + auto Ready = [&]() { + return !commit_data_.empty() && commit_data_.begin()->first == start_seq_; + }; + + while(!IsStop()){ + std::unique_lock<std::mutex> lk(n_mutex_); + vote_cv_.wait_for(lk, std::chrono::microseconds(1000), + [&] { return Ready(); }); + if(Ready()){ + return true; + } + } + return false; +} + +void MultiPaxos::AsyncCommit() { + while (!IsStop()) { + auto p = commit_q_.Pop(); + if(p == nullptr){ + continue; + } + + int proposer = p->header().proposer(); + int round = p->header().view(); + std::unique_ptr<Proposal> data = nullptr; + { + std::unique_lock<std::mutex> lk(mutex_); + auto it = receive_.find(round); + assert(it != receive_.end()); + auto dit = it->second.find(proposer); + assert(dit != it->second.end()); + data = std::move(dit->second); + assert(data != nullptr); + } + AddCommitData(std::move(data)); + } +} + +void MultiPaxos::AsyncCommitSeq() { + int seq = 1; + while (!IsStop()) { + if(!Wait()){ + break; + } + std::unique_ptr<Proposal> data = GetCommitData(); + int proposer = data->header().proposer(); + int round = data->header().view(); + int proposal_seq = data->header().proposal_id(); + LOG(ERROR)<<" commit proposer:"<<proposer<< " round:"<<round<<" seq:"<<proposal_seq; + for(Transaction& txn : *data->mutable_transactions()){ + txn.set_id(seq++); + Commit(txn); + } + } +} + +bool MultiPaxos::ReceiveTransaction(std::unique_ptr<Transaction> txn) { + txn->set_proposer(id_); + txns_.Push(std::move(txn)); + return true; +} + +bool MultiPaxos::ReceiveProposal(std::unique_ptr<Proposal> proposal) { + int round = proposal->header().view(); + int proposer = proposal->sender(); + int seq = proposal->header().proposal_id(); + + bool done = false; + { + //LOG(ERROR)<<"recv proposal from:"<<proposer<<" round:"<<round<<" seq:"<<seq; + std::unique_lock<std::mutex> lk(mutex_); + receive_[round][proposer] = std::move(proposal); + done = true; + } + if(done) { + Proposal learn_proposal; + learn_proposal.mutable_header()->set_view(round); + learn_proposal.mutable_header()->set_proposer(proposer); + learn_proposal.mutable_header()->set_proposal_id(seq); + learn_proposal.set_sender(id_); + Broadcast(MessageType::Learn, learn_proposal); + } + return true; +} + +bool MultiPaxos::ReceiveLearn(std::unique_ptr<Proposal> proposal) { + int round = proposal->header().view(); + int sender = proposal->sender(); + int proposer = proposal->header().proposer(); + + std::unique_lock<std::mutex> lk(learn_mutex_); + learn_receive_[round].insert(sender); + LOG(ERROR)<<"RECEIVE proposer learn:"<<round<<" size:"<<learn_receive_[round].size()<<" proposer:"<<proposer; + if(learn_receive_[round].size() == f_+1){ + CommitProposal(std::move(proposal)); + } + return true; +} + +void MultiPaxos::CommitProposal(std::unique_ptr<Proposal> p){ + commit_q_.Push(std::move(p)); +} + + +} // namespace tusk +} // namespace resdb diff --git a/platform/consensus/ordering/multipaxos/algorithm/multipaxos.h b/platform/consensus/ordering/multipaxos/algorithm/multipaxos.h new file mode 100644 index 00000000..6ade5312 --- /dev/null +++ b/platform/consensus/ordering/multipaxos/algorithm/multipaxos.h @@ -0,0 +1,60 @@ +#pragma once + +#include <thread> + +#include "platform/common/queue/lock_free_queue.h" +#include "platform/consensus/ordering/multipaxos/algorithm/proposal_manager.h" +#include "platform/consensus/ordering/multipaxos/proto/proposal.pb.h" +#include "platform/consensus/ordering/common/algorithm/protocol_base.h" + +namespace resdb { +namespace multipaxos { + +class MultiPaxos: public common::ProtocolBase { + public: + MultiPaxos(int id, int f, int total_num, SignatureVerifier* verifier ); + ~MultiPaxos(); + + // recv txn -> send block with links -> rec block ack -> send block with certs + bool ReceiveTransaction(std::unique_ptr<Transaction> txn); + bool ReceiveProposal(std::unique_ptr<Proposal> proposal); + bool ReceiveLearn(std::unique_ptr<Proposal> proposal); + + + private: + bool Ready(); + void AsyncSend(); + void AsyncCommit(); + void AsyncCommitSeq(); + + void CommitProposal(std::unique_ptr<Proposal> p); + + void AddCommitData(std::unique_ptr<Proposal> p); + std::unique_ptr<Proposal> GetCommitData(); + bool Wait() ; + + + private: + LockFreeQueue<Transaction> txns_; + LockFreeQueue<Proposal> commit_q_; + + std::unique_ptr<ProposalManager> proposal_manager_; + + std::thread send_thread_, commit_thread_, commit_seq_thread_; + + int batch_size_; + + std::mutex mutex_; + std::map<int, std::map<int, std::unique_ptr<Proposal>> > receive_; + + std::mutex learn_mutex_; + std::map<int, std::set<int>> learn_receive_; + + std::mutex n_mutex_; + std::map<int, std::unique_ptr<Proposal> > commit_data_; + std::condition_variable vote_cv_; + int start_seq_; +}; + +} // namespace tusk +} // namespace resdb diff --git a/platform/consensus/ordering/multipaxos/algorithm/proposal_manager.cpp b/platform/consensus/ordering/multipaxos/algorithm/proposal_manager.cpp new file mode 100644 index 00000000..1a55f8db --- /dev/null +++ b/platform/consensus/ordering/multipaxos/algorithm/proposal_manager.cpp @@ -0,0 +1,36 @@ +#include "platform/consensus/ordering/multipaxos/algorithm/proposal_manager.h" + +#include <glog/logging.h> +#include "common/utils/utils.h" +#include "common/crypto/signature_verifier.h" + +namespace resdb { +namespace multipaxos { + +ProposalManager::ProposalManager(int32_t id, int limit_count, SignatureVerifier* verifier) + : id_(id) { + round_ = 1; + seq_ = 1; + assert(verifier_ != nullptr); +} + +std::unique_ptr<Proposal> ProposalManager::GenerateProposal( + const std::vector<std::unique_ptr<Transaction>>& txns) { + std::unique_ptr<Proposal> proposal = std::make_unique<Proposal>(); + { + for(const auto& txn: txns){ + *proposal->add_transactions() = *txn; + } + proposal->mutable_header()->set_proposal_id(seq_++); + proposal->mutable_header()->set_proposer(id_); + proposal->mutable_header()->set_view(round_++); + proposal->set_sender(id_); + } + return proposal; +} + +int ProposalManager::CurrentView(){ + return round_; +} +} // namespace tusk +} // namespace resdb diff --git a/platform/consensus/ordering/multipaxos/algorithm/proposal_manager.h b/platform/consensus/ordering/multipaxos/algorithm/proposal_manager.h new file mode 100644 index 00000000..a8678b46 --- /dev/null +++ b/platform/consensus/ordering/multipaxos/algorithm/proposal_manager.h @@ -0,0 +1,25 @@ +#pragma once + +#include "platform/consensus/ordering/multipaxos/proto/proposal.pb.h" +#include "common/crypto/signature_verifier.h" + +namespace resdb { +namespace multipaxos { + +class ProposalManager { + public: + ProposalManager(int32_t id, int limit_count, SignatureVerifier * verifier); + + std::unique_ptr<Proposal> GenerateProposal(const std::vector<std::unique_ptr<Transaction>>& txns); + int CurrentView(); + + private: + int32_t id_; + int round_; + int seq_; + + SignatureVerifier* verifier_; +}; + +} // namespace tusk +} // namespace resdb diff --git a/platform/consensus/ordering/multipaxos/framework/BUILD b/platform/consensus/ordering/multipaxos/framework/BUILD new file mode 100644 index 00000000..7dcc0ec3 --- /dev/null +++ b/platform/consensus/ordering/multipaxos/framework/BUILD @@ -0,0 +1,20 @@ +package(default_visibility = ["//visibility:private"]) + +cc_library( + name = "consensus", + srcs = ["consensus.cpp"], + hdrs = ["consensus.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + "//common/utils", + "//executor/common:transaction_manager", + "//platform/consensus/ordering/common/framework:consensus", + "//platform/consensus/execution:transaction_executor", + "//platform/consensus/ordering/multipaxos/algorithm:multipaxos", + "//platform/networkstrate:consensus_manager", + ], +) + + diff --git a/platform/consensus/ordering/multipaxos/framework/consensus.cpp b/platform/consensus/ordering/multipaxos/framework/consensus.cpp new file mode 100644 index 00000000..b20d4e5e --- /dev/null +++ b/platform/consensus/ordering/multipaxos/framework/consensus.cpp @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2019-2022 ExpoLab, UC Davis + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + */ + +#include "platform/consensus/ordering/multipaxos/framework/consensus.h" + +#include <glog/logging.h> +#include <unistd.h> + +#include "common/utils/utils.h" + +namespace resdb { +namespace multipaxos { + +Consensus::Consensus(const ResDBConfig& config, + std::unique_ptr<TransactionManager> executor) + : common::Consensus(config, std::move(executor)) { + + Init(); + + int total_replicas = config_.GetReplicaNum(); + int f = (total_replicas - 1) / 3; + + if (config_.GetPublicKeyCertificateInfo() + .public_key() + .public_key_info() + .type() != CertificateKeyInfo::CLIENT) { + multipaxos_= std::make_unique<MultiPaxos>(config_.GetSelfInfo().id(), f, total_replicas, GetSignatureVerifier()); + InitProtocol(multipaxos_.get()); + } +} + +int Consensus::ProcessCustomConsensus(std::unique_ptr<Request> request) { + //LOG(ERROR)<<"recv request:"<<MessageType_Name(request->user_type()); + //int64_t current_time = GetCurrentTime(); + + if(request->user_type() == MessageType::Propose) { + std::unique_ptr<Proposal> p = std::make_unique<Proposal>(); + if (!p->ParseFromString(request->data())) { + LOG(ERROR) << "parse proposal fail"; + assert(1 == 0); + return -1; + } + multipaxos_->ReceiveProposal(std::move(p)); + } + else if(request->user_type() == MessageType::Query) { + std::unique_ptr<Proposal> p = std::make_unique<Proposal>(); + if (!p->ParseFromString(request->data())) { + LOG(ERROR) << "parse proposal fail"; + assert(1 == 0); + return -1; + } + //multipaxos_->ReceivePropose(std::move(p)); + } + else if(request->user_type() == MessageType::Learn) { + std::unique_ptr<Proposal> p = std::make_unique<Proposal>(); + if (!p->ParseFromString(request->data())) { + LOG(ERROR) << "parse proposal fail"; + assert(1 == 0); + return -1; + } + multipaxos_->ReceiveLearn(std::move(p)); + } + return 0; +} + +int Consensus::ProcessNewTransaction(std::unique_ptr<Request> request) { + std::unique_ptr<Transaction> txn = std::make_unique<Transaction>(); + txn->set_data(request->data()); + txn->set_hash(request->hash()); + txn->set_proxy_id(request->proxy_id()); + txn->set_user_seq(request->user_seq()); + return multipaxos_->ReceiveTransaction(std::move(txn)); +} + +int Consensus::CommitMsg(const google::protobuf::Message& msg) { + return CommitMsgInternal(dynamic_cast<const Transaction&>(msg)); +} + +int Consensus::CommitMsgInternal(const Transaction& txn) { + //LOG(ERROR)<<"commit txn:"<<txn.id()<<" proxy id:"<<txn.proxy_id(); + std::unique_ptr<Request> request = std::make_unique<Request>(); + request->set_data(txn.data()); + request->set_seq(txn.id()); + request->set_proxy_id(txn.proxy_id()); + transaction_executor_->AddExecuteMessage(std::move(request)); + return 0; +} + + +} // namespace fairdag +} // namespace resdb diff --git a/platform/consensus/ordering/multipaxos/framework/consensus.h b/platform/consensus/ordering/multipaxos/framework/consensus.h new file mode 100644 index 00000000..4d08c3ac --- /dev/null +++ b/platform/consensus/ordering/multipaxos/framework/consensus.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019-2022 ExpoLab, UC Davis + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + */ + +#pragma once + +#include "executor/common/transaction_manager.h" +#include "platform/consensus/execution/transaction_executor.h" +#include "platform/consensus/ordering/multipaxos/algorithm/multipaxos.h" +#include "platform/consensus/ordering/common/framework/consensus.h" +#include "platform/networkstrate/consensus_manager.h" + +namespace resdb { +namespace multipaxos { + +class Consensus : public common::Consensus{ + public: + Consensus(const ResDBConfig& config, + std::unique_ptr<TransactionManager> transaction_manager); + + protected: + int ProcessCustomConsensus(std::unique_ptr<Request> request) override; + int ProcessNewTransaction(std::unique_ptr<Request> request) override; + int CommitMsg(const google::protobuf::Message& msg) override; + int CommitMsgInternal(const Transaction& txn); + + private: + std::unique_ptr<MultiPaxos> multipaxos_; +}; + +} // namespace tusk +} // namespace resdb diff --git a/platform/consensus/ordering/multipaxos/proto/BUILD b/platform/consensus/ordering/multipaxos/proto/BUILD new file mode 100644 index 00000000..1fe488c3 --- /dev/null +++ b/platform/consensus/ordering/multipaxos/proto/BUILD @@ -0,0 +1,16 @@ +package(default_visibility = ["//platform/consensus/ordering/multipaxos:__subpackages__"]) + +load("@rules_cc//cc:defs.bzl", "cc_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") +load("@rules_proto_grpc//python:defs.bzl", "python_proto_library") + +proto_library( + name = "proposal_proto", + srcs = ["proposal.proto"], + deps = ["//common/proto:signature_info_proto"], +) + +cc_proto_library( + name = "proposal_cc_proto", + deps = [":proposal_proto"], +) diff --git a/platform/consensus/ordering/multipaxos/proto/proposal.proto b/platform/consensus/ordering/multipaxos/proto/proposal.proto new file mode 100644 index 00000000..83875513 --- /dev/null +++ b/platform/consensus/ordering/multipaxos/proto/proposal.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; +import "common/proto/signature_info.proto"; + +package resdb.multipaxos; + +message Transaction{ + int32 id = 1; + bytes data = 2; + bytes hash = 3; + int32 proxy_id = 4; + int32 proposer = 5; + uint64 timestamp = 6; + int64 user_seq = 7; +} + +message Header { + int32 proposer = 1; + int32 proposal_id = 2; + int32 view = 3; +} + +message Proposal { + Header header = 1; + repeated Transaction transactions = 2; + int32 sender = 3; + bytes hash = 4; +}; + +enum MessageType { + NewProposal = 0; + Propose = 1; + Learn = 2; + Query = 3; +}; + diff --git a/platform/consensus/ordering/tusk/algorithm/tusk.cpp b/platform/consensus/ordering/tusk/algorithm/tusk.cpp index 044e1e3d..0156e38f 100644 --- a/platform/consensus/ordering/tusk/algorithm/tusk.cpp +++ b/platform/consensus/ordering/tusk/algorithm/tusk.cpp @@ -300,9 +300,21 @@ bool Tusk::ReceiveTransaction(std::unique_ptr<Transaction> txn) { } bool Tusk::ReceiveBlock(std::unique_ptr<Proposal> proposal) { - //LOG(ERROR) << "recv block from " << proposal->header().proposer_id() - // << " round:" << proposal->header().round(); + /* + LOG(ERROR) << "recv block from " << proposal->header().proposer_id() + << " round:" << proposal->header().round(); + */ + if(proposal->header().round() == 200 && id_ >= 23 && proposal->header().proposer_id() < 23){ + sleep(10); + } + + if(proposal->header().round() == 200 && id_ <= 23 && proposal->header().proposer_id() > 23){ + //LOG(ERROR) << "recv block from " << proposal->header().proposer_id() + // << " round:" << proposal->header().round(); + sleep(10); + } + std::unique_lock<std::mutex> lk(check_block_mutex_); { proposal->set_queuing_time(GetCurrentTime()); diff --git a/scripts/deploy/config/cassandra.config b/scripts/deploy/config/cassandra.config index 193b4830..2d91c531 100644 --- a/scripts/deploy/config/cassandra.config +++ b/scripts/deploy/config/cassandra.config @@ -1,9 +1,9 @@ { - "clientBatchNum": 800, + "clientBatchNum": 400, "enable_viewchange": false, "recovery_enabled": false, "max_client_complaint_num":10, - "max_process_txn": 256, + "max_process_txn": 512, "worker_num": 10, "input_worker_num": 1, "output_worker_num": 5 diff --git a/scripts/deploy/config/kv_performance_server.conf b/scripts/deploy/config/kv_performance_server.conf index 498f5a98..98253c0a 100644 --- a/scripts/deploy/config/kv_performance_server.conf +++ b/scripts/deploy/config/kv_performance_server.conf @@ -1,8 +1,8 @@ iplist=( -172.31.23.110 -172.31.31.183 -172.31.22.246 -172.31.26.117 -172.31.21.196 +172.31.25.232 +172.31.17.37 +172.31.30.17 +172.31.27.41 ) +client_num=1 diff --git a/scripts/deploy/config/kv_performance_server_16.conf b/scripts/deploy/config/kv_performance_server_16.conf index f5b61302..981ff4eb 100644 --- a/scripts/deploy/config/kv_performance_server_16.conf +++ b/scripts/deploy/config/kv_performance_server_16.conf @@ -1,36 +1,36 @@ iplist=( -172.31.40.76 -172.31.37.77 -172.31.45.72 -172.31.41.9 -172.31.42.143 -172.31.38.17 -172.31.47.205 -172.31.41.14 -172.31.45.115 -172.31.45.118 -172.31.40.240 -172.31.45.242 -172.31.42.5 -172.31.39.133 -172.31.47.250 -172.31.46.126 -172.31.47.231 -172.31.44.39 -172.31.47.38 -172.31.38.102 -172.31.34.108 -172.31.42.173 -172.31.42.40 -172.31.43.42 -172.31.42.225 -172.31.39.35 -172.31.37.83 -172.31.40.218 -172.31.38.82 -172.31.34.146 -172.31.34.157 -172.31.35.222 +172.31.29.163 +172.31.22.39 +172.31.16.32 +172.31.27.98 +172.31.23.49 +172.31.18.178 +172.31.25.109 +172.31.17.109 +172.31.18.116 +172.31.25.117 +172.31.28.50 +172.31.24.115 +172.31.17.251 +172.31.17.252 +172.31.27.122 +172.31.26.123 +172.31.20.132 +172.31.20.70 +172.31.24.255 +172.31.25.132 +172.31.24.15 +172.31.20.16 +172.31.26.76 +172.31.18.15 +172.31.29.23 +172.31.19.215 +172.31.19.17 +172.31.31.149 +172.31.17.92 +172.31.31.159 +172.31.26.155 +172.31.24.155 ) key=~/.ssh/junchao.pem diff --git a/scripts/deploy/config/kv_performance_server_32.conf b/scripts/deploy/config/kv_performance_server_32.conf index bb245dda..aab16fae 100644 --- a/scripts/deploy/config/kv_performance_server_32.conf +++ b/scripts/deploy/config/kv_performance_server_32.conf @@ -1,68 +1,68 @@ iplist=( -172.31.35.170 -172.31.45.211 -172.31.40.83 -172.31.43.214 -172.31.40.20 -172.31.44.90 -172.31.36.216 -172.31.42.92 -172.31.34.198 -172.31.43.197 -172.31.36.136 -172.31.36.7 -172.31.39.203 -172.31.47.75 -172.31.36.14 -172.31.46.11 -172.31.42.185 -172.31.35.247 -172.31.32.252 -172.31.45.252 -172.31.45.189 -172.31.44.125 -172.31.39.129 -172.31.34.126 -172.31.40.111 -172.31.32.170 -172.31.42.178 -172.31.39.48 -172.31.47.53 -172.31.41.245 -172.31.41.246 -172.31.37.118 -172.31.40.76 -172.31.37.77 -172.31.45.72 -172.31.41.9 -172.31.42.143 -172.31.38.17 -172.31.47.205 -172.31.41.14 -172.31.45.115 -172.31.45.118 -172.31.40.240 -172.31.45.242 -172.31.42.5 -172.31.39.133 -172.31.47.250 -172.31.46.126 -172.31.47.231 -172.31.44.39 -172.31.47.38 -172.31.38.102 -172.31.34.108 -172.31.42.173 -172.31.42.40 -172.31.43.42 -172.31.42.225 -172.31.39.35 -172.31.37.83 -172.31.40.218 -172.31.38.82 -172.31.34.146 -172.31.34.157 -172.31.35.222 +172.31.31.59 +172.31.19.187 +172.31.28.180 +172.31.27.51 +172.31.20.3 +172.31.24.2 +172.31.29.63 +172.31.18.59 +172.31.26.174 +172.31.28.43 +172.31.27.40 +172.31.27.36 +172.31.20.179 +172.31.17.176 +172.31.27.48 +172.31.31.174 +172.31.20.20 +172.31.27.19 +172.31.17.17 +172.31.16.16 +172.31.23.31 +172.31.21.158 +172.31.24.152 +172.31.18.23 +172.31.16.11 +172.31.27.137 +172.31.26.6 +172.31.24.133 +172.31.20.14 +172.31.17.13 +172.31.18.141 +172.31.24.11 +172.31.30.254 +172.31.16.252 +172.31.29.120 +172.31.22.248 +172.31.28.200 +172.31.26.71 +172.31.24.197 +172.31.26.255 +172.31.24.233 +172.31.19.227 +172.31.28.98 +172.31.18.224 +172.31.19.119 +172.31.31.113 +172.31.25.111 +172.31.26.233 +172.31.17.218 +172.31.26.89 +172.31.16.87 +172.31.18.87 +172.31.26.34 +172.31.27.160 +172.31.24.93 +172.31.24.92 +172.31.18.83 +172.31.29.210 +172.31.30.76 +172.31.19.73 +172.31.21.87 +172.31.28.213 +172.31.23.211 +172.31.16.211 ) key=~/.ssh/junchao.pem diff --git a/scripts/deploy/config/kv_performance_server_64.conf b/scripts/deploy/config/kv_performance_server_64.conf index b2d40276..7e8be748 100644 --- a/scripts/deploy/config/kv_performance_server_64.conf +++ b/scripts/deploy/config/kv_performance_server_64.conf @@ -1,132 +1,132 @@ iplist=( -172.31.35.170 -172.31.45.211 -172.31.40.83 -172.31.43.214 -172.31.40.20 -172.31.44.90 -172.31.36.216 -172.31.42.92 -172.31.34.198 -172.31.43.197 -172.31.36.136 -172.31.36.7 -172.31.39.203 -172.31.47.75 -172.31.36.14 -172.31.46.11 -172.31.42.185 -172.31.35.247 -172.31.32.252 -172.31.45.252 -172.31.45.189 -172.31.44.125 -172.31.39.129 -172.31.34.126 -172.31.40.111 -172.31.32.170 -172.31.42.178 -172.31.39.48 -172.31.47.53 -172.31.41.245 -172.31.41.246 -172.31.37.118 -172.31.40.76 -172.31.37.77 -172.31.45.72 -172.31.41.9 -172.31.42.143 -172.31.38.17 -172.31.47.205 -172.31.41.14 -172.31.45.115 -172.31.45.118 -172.31.40.240 -172.31.45.242 -172.31.42.5 -172.31.39.133 -172.31.47.250 -172.31.46.126 -172.31.47.231 -172.31.44.39 -172.31.47.38 -172.31.38.102 -172.31.34.108 -172.31.42.173 -172.31.42.40 -172.31.43.42 -172.31.42.225 -172.31.39.35 -172.31.37.83 -172.31.40.218 -172.31.38.82 -172.31.34.146 -172.31.34.157 -172.31.35.222 -172.31.34.145 -172.31.35.17 -172.31.42.15 -172.31.40.145 -172.31.36.13 -172.31.47.142 -172.31.36.11 -172.31.43.140 -172.31.43.133 -172.31.41.137 -172.31.37.3 -172.31.43.132 -172.31.34.62 -172.31.38.0 -172.31.43.190 -172.31.35.190 -172.31.32.188 -172.31.43.61 -172.31.45.185 -172.31.47.186 -172.31.34.184 -172.31.40.184 -172.31.32.183 -172.31.38.55 -172.31.32.181 -172.31.41.181 -172.31.46.50 -172.31.35.179 -172.31.41.45 -172.31.34.178 -172.31.43.234 -172.31.34.107 -172.31.40.43 -172.31.41.172 -172.31.32.169 -172.31.41.170 -172.31.44.37 -172.31.39.38 -172.31.43.92 -172.31.34.35 -172.31.32.88 -172.31.47.91 -172.31.37.213 -172.31.41.86 -172.31.40.211 -172.31.40.212 -172.31.46.210 -172.31.43.82 -172.31.33.209 -172.31.40.82 -172.31.36.78 -172.31.39.207 -172.31.43.200 -172.31.46.206 -172.31.38.69 -172.31.43.198 -172.31.34.124 -172.31.45.65 -172.31.36.123 -172.31.35.124 -172.31.37.246 -172.31.41.247 -172.31.38.112 -172.31.44.240 +172.31.29.160 +172.31.22.239 +172.31.23.112 +172.31.21.177 +172.31.23.242 +172.31.26.233 +172.31.23.235 +172.31.16.108 +172.31.22.174 +172.31.23.56 +172.31.30.57 +172.31.22.58 +172.31.16.127 +172.31.16.245 +172.31.17.246 +172.31.17.118 +172.31.24.247 +172.31.24.204 +172.31.23.12 +172.31.29.140 +172.31.28.141 +172.31.31.3 +172.31.30.199 +172.31.29.73 +172.31.27.75 +172.31.19.87 +172.31.25.31 +172.31.18.95 +172.31.23.208 +172.31.22.83 +172.31.20.214 +172.31.16.87 +172.31.26.116 +172.31.27.244 +172.31.18.245 +172.31.25.121 +172.31.29.121 +172.31.27.254 +172.31.21.255 +172.31.27.255 +172.31.28.96 +172.31.21.231 +172.31.21.236 +172.31.24.237 +172.31.30.240 +172.31.22.242 +172.31.18.49 +172.31.18.178 +172.31.17.179 +172.31.26.181 +172.31.31.54 +172.31.29.55 +172.31.30.188 +172.31.27.190 +172.31.17.162 +172.31.31.168 +172.31.20.169 +172.31.22.171 +172.31.28.172 +172.31.24.173 +172.31.28.45 +172.31.19.174 +172.31.31.215 +172.31.22.216 +172.31.19.217 +172.31.28.217 +172.31.19.220 +172.31.18.221 +172.31.21.32 +172.31.17.34 +172.31.29.69 +172.31.23.202 +172.31.16.205 +172.31.30.208 +172.31.20.82 +172.31.28.211 +172.31.30.83 +172.31.30.84 +172.31.25.28 +172.31.18.30 +172.31.28.14 +172.31.19.14 +172.31.30.144 +172.31.31.149 +172.31.28.22 +172.31.31.22 +172.31.20.23 +172.31.22.155 +172.31.26.191 +172.31.21.1 +172.31.17.130 +172.31.17.6 +172.31.29.9 +172.31.29.12 +172.31.18.142 +172.31.26.142 +172.31.29.220 +172.31.24.157 +172.31.27.64 +172.31.30.5 +172.31.30.139 +172.31.27.13 +172.31.22.60 +172.31.22.189 +172.31.31.62 +172.31.24.192 +172.31.23.82 +172.31.23.83 +172.31.29.25 +172.31.23.154 +172.31.23.143 +172.31.29.16 +172.31.29.144 +172.31.22.144 +172.31.20.35 +172.31.25.228 +172.31.24.164 +172.31.16.165 +172.31.22.33 +172.31.29.161 +172.31.26.168 +172.31.31.173 +172.31.31.53 +172.31.20.54 +172.31.24.165 +172.31.26.230 +172.31.18.230 +172.31.22.104 ) key=~/.ssh/junchao.pem diff --git a/scripts/deploy/config/rcc.config b/scripts/deploy/config/multipaxos.config similarity index 88% copy from scripts/deploy/config/rcc.config copy to scripts/deploy/config/multipaxos.config index 70ccdcc7..1ac9fe46 100644 --- a/scripts/deploy/config/rcc.config +++ b/scripts/deploy/config/multipaxos.config @@ -1,5 +1,5 @@ { - "clientBatchNum": 200, + "clientBatchNum": 100, "enable_viewchange": false, "recovery_enabled": false, "max_client_complaint_num":10, diff --git a/scripts/deploy/config/rcc.config b/scripts/deploy/config/rcc.config index 70ccdcc7..985c7f80 100644 --- a/scripts/deploy/config/rcc.config +++ b/scripts/deploy/config/rcc.config @@ -1,5 +1,5 @@ { - "clientBatchNum": 200, + "clientBatchNum": 400, "enable_viewchange": false, "recovery_enabled": false, "max_client_complaint_num":10, diff --git a/scripts/deploy/config/tusk.config b/scripts/deploy/config/tusk.config index 424cad47..80213d34 100644 --- a/scripts/deploy/config/tusk.config +++ b/scripts/deploy/config/tusk.config @@ -1,10 +1,10 @@ { - "clientBatchNum": 200, + "clientBatchNum": 400, "enable_viewchange": false, "recovery_enabled": false, "max_client_complaint_num":10, - "max_process_txn": 384, - "worker_num": 10, + "max_process_txn": 4096, + "worker_num": 40, "input_worker_num": 1, "output_worker_num": 5 } diff --git a/scripts/deploy/performance/multipaxos_performance.sh b/scripts/deploy/performance/multipaxos_performance.sh new file mode 100755 index 00000000..d9586847 --- /dev/null +++ b/scripts/deploy/performance/multipaxos_performance.sh @@ -0,0 +1,6 @@ +export server=//benchmark/protocols/multipaxos:kv_server_performance +export TEMPLATE_PATH=$PWD/config/multipaxos.config +#export COPTS="--define enable_leveldb=True" +#export COPTS="-pg" + +./performance/run_performance.sh $* diff --git a/scripts/deploy/performance/set_delay.sh b/scripts/deploy/performance/set_delay.sh new file mode 100755 index 00000000..9bfe1048 --- /dev/null +++ b/scripts/deploy/performance/set_delay.sh @@ -0,0 +1,91 @@ +. ./script/env.sh + + +KEY_FILE="config/key.conf" + +. ${KEY_FILE} +iplist=( +172.31.29.163 +172.31.22.39 +172.31.16.32 +172.31.27.98 +172.31.23.49 +172.31.18.178 +172.31.25.109 +172.31.17.109 +172.31.18.116 +172.31.25.117 +172.31.28.50 +172.31.24.115 +172.31.17.251 +172.31.17.252 +172.31.27.122 +172.31.26.123 +172.31.25.31 +#172.31.18.95 +#172.31.23.208 +#172.31.22.83 +#172.31.20.214 +#172.31.16.87 +#172.31.26.116 +#172.31.27.244 +#172.31.18.245 +#172.31.25.121 +#172.31.29.121 +#172.31.27.254 +#172.31.21.255 +#172.31.27.255 +#172.31.28.96 +#172.31.21.231 +#172.31.21.236 +#172.31.24.237 +#172.31.30.240 +#172.31.22.242 +#172.31.18.49 +#172.31.18.178 +#172.31.17.179 +#172.31.26.181 +#172.31.31.54 +#172.31.29.55 +#172.31.30.188 +#172.31.27.190 +#172.31.17.162 +#172.31.31.168 +#172.31.20.169 +#172.31.22.171 +#172.31.28.172 +#172.31.24.173 +#172.31.28.45 +#172.31.19.174 +#172.31.31.215 +#172.31.22.216 + +) + +command1="sudo tc qdisc add dev ens5 root netem delay 200ms" +command2="sudo tc qdisc change dev ens5 root netem delay 200ms" +command3="tc qdisc show" + +echo ${command1} + +function run_cmd(){ + ip=$1 + cmd=$2 + ssh -i ${key} -n -o BatchMode=yes -o StrictHostKeyChecking=no ubuntu@${ip} "${cmd}" + echo "ssh -i ${key} -n -o BatchMode=yes -o StrictHostKeyChecking=no ubuntu@${ip} \"${cmd}\"" +} + +for ip in ${iplist[@]}; +do + run_cmd ${ip} "${command1}" & + #run_cmd ${ip} "${command2}" & +done + +wait + +for ip in ${iplist[@]}; +do + run_cmd ${ip} "${command3}" +done + +wait
