This is an automated email from the ASF dual-hosted git repository.

junchao pushed a commit to branch cassandra
in repository https://gitbox.apache.org/repos/asf/incubator-resilientdb.git

commit a082d4416ce98be29a538f54f89855ed33cd2def
Author: cjcchen <[email protected]>
AuthorDate: Tue Oct 1 09:33:52 2024 +0000

    add multipaxos
---
 benchmark/protocols/multipaxos/BUILD               |  16 ++
 .../protocols/multipaxos/kv_server_performance.cpp |  88 +++++++
 .../ordering/cassandra/algorithm/cassandra.cpp     |   5 +-
 .../consensus/ordering/multipaxos/algorithm/BUILD  |  39 ++++
 .../ordering/multipaxos/algorithm/multipaxos.cpp   | 174 ++++++++++++++
 .../ordering/multipaxos/algorithm/multipaxos.h     |  60 +++++
 .../multipaxos/algorithm/proposal_manager.cpp      |  36 +++
 .../multipaxos/algorithm/proposal_manager.h        |  25 ++
 .../consensus/ordering/multipaxos/framework/BUILD  |  20 ++
 .../ordering/multipaxos/framework/consensus.cpp    | 113 +++++++++
 .../ordering/multipaxos/framework/consensus.h      |  53 +++++
 platform/consensus/ordering/multipaxos/proto/BUILD |  16 ++
 .../ordering/multipaxos/proto/proposal.proto       |  35 +++
 .../consensus/ordering/tusk/algorithm/tusk.cpp     |  16 +-
 scripts/deploy/config/cassandra.config             |   4 +-
 scripts/deploy/config/kv_performance_server.conf   |  10 +-
 .../deploy/config/kv_performance_server_16.conf    |  64 +++---
 .../deploy/config/kv_performance_server_32.conf    | 128 +++++------
 .../deploy/config/kv_performance_server_64.conf    | 256 ++++++++++-----------
 .../config/{rcc.config => multipaxos.config}       |   2 +-
 scripts/deploy/config/rcc.config                   |   2 +-
 scripts/deploy/config/tusk.config                  |   6 +-
 .../deploy/performance/multipaxos_performance.sh   |   6 +
 scripts/deploy/performance/set_delay.sh            |  91 ++++++++
 24 files changed, 1025 insertions(+), 240 deletions(-)

diff --git a/benchmark/protocols/multipaxos/BUILD 
b/benchmark/protocols/multipaxos/BUILD
new file mode 100644
index 00000000..69b373d8
--- /dev/null
+++ b/benchmark/protocols/multipaxos/BUILD
@@ -0,0 +1,16 @@
+package(default_visibility = ["//visibility:private"])
+
+load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
+
+cc_binary(
+    name = "kv_server_performance",
+    srcs = ["kv_server_performance.cpp"],
+    deps = [
+        "//chain/storage:memory_db",
+        "//executor/kv:kv_executor",
+        "//platform/config:resdb_config_utils",
+        "//platform/consensus/ordering/multipaxos/framework:consensus",
+        "//service/utils:server_factory",
+    ],
+)
+
diff --git a/benchmark/protocols/multipaxos/kv_server_performance.cpp 
b/benchmark/protocols/multipaxos/kv_server_performance.cpp
new file mode 100644
index 00000000..17fba4d1
--- /dev/null
+++ b/benchmark/protocols/multipaxos/kv_server_performance.cpp
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2019-2022 ExpoLab, UC Davis
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <glog/logging.h>
+
+#include "chain/storage/memory_db.h"
+#include "executor/kv/kv_executor.h"
+#include "platform/config/resdb_config_utils.h"
+#include "platform/consensus/ordering/multipaxos/framework/consensus.h"
+#include "platform/networkstrate/service_network.h"
+#include "platform/statistic/stats.h"
+#include "proto/kv/kv.pb.h"
+
+using namespace resdb;
+using namespace resdb::multipaxos;
+using namespace resdb::storage;
+
+void ShowUsage() {
+  printf("<config> <private_key> <cert_file> [logging_dir]\n");
+}
+
+std::string GetRandomKey() {
+  int num1 = rand() % 10;
+  int num2 = rand() % 10;
+  return std::to_string(num1) + std::to_string(num2);
+}
+
+int main(int argc, char** argv) {
+  if (argc < 3) {
+    ShowUsage();
+    exit(0);
+  }
+
+  // google::InitGoogleLogging(argv[0]);
+  // FLAGS_minloglevel = google::GLOG_WARNING;
+
+  char* config_file = argv[1];
+  char* private_key_file = argv[2];
+  char* cert_file = argv[3];
+
+  if (argc >= 5) {
+    auto monitor_port = Stats::GetGlobalStats(5);
+    monitor_port->SetPrometheus(argv[4]);
+  }
+
+  std::unique_ptr<ResDBConfig> config =
+      GenerateResDBConfig(config_file, private_key_file, cert_file);
+
+  config->RunningPerformance(true);
+
+  auto performance_consens = std::make_unique<multipaxos::Consensus>(
+      *config, std::make_unique<KVExecutor>(std::make_unique<MemoryDB>()));
+  performance_consens->SetupPerformanceDataFunc([]() {
+    KVRequest request;
+    request.set_cmd(KVRequest::SET);
+    request.set_key(GetRandomKey());
+    request.set_value("helloworld");
+    std::string request_data;
+    request.SerializeToString(&request_data);
+    return request_data;
+  });
+
+  auto server =
+      std::make_unique<ServiceNetwork>(*config, 
std::move(performance_consens));
+  server->Run();
+}
diff --git a/platform/consensus/ordering/cassandra/algorithm/cassandra.cpp 
b/platform/consensus/ordering/cassandra/algorithm/cassandra.cpp
index 7b32c5dc..f32ba2d9 100644
--- a/platform/consensus/ordering/cassandra/algorithm/cassandra.cpp
+++ b/platform/consensus/ordering/cassandra/algorithm/cassandra.cpp
@@ -17,10 +17,11 @@ Cassandra::Cassandra(int id, int f, int total_num, 
SignatureVerifier* verifier)
   total_num_ = total_num;
   f_ = f;
   is_stop_ = false;
+  //timeout_ms_ = 10000;
   timeout_ms_ = 60000;
   local_txn_id_ = 1;
   local_proposal_id_ = 1;
-  batch_size_ = 10;
+  batch_size_ = 15;
 
   recv_num_ = 0;
   execute_num_ = 0;
@@ -248,7 +249,7 @@ void Cassandra::BroadcastTxn() {
     */
 
     for(int i = 1; i < batch_size_; ++i){
-      std::unique_ptr<Transaction> txn = txns_.Pop(10);
+      std::unique_ptr<Transaction> txn = txns_.Pop(100);
       if(txn == nullptr){
         break;
       }
diff --git a/platform/consensus/ordering/multipaxos/algorithm/BUILD 
b/platform/consensus/ordering/multipaxos/algorithm/BUILD
new file mode 100644
index 00000000..6eef34d4
--- /dev/null
+++ b/platform/consensus/ordering/multipaxos/algorithm/BUILD
@@ -0,0 +1,39 @@
+package(default_visibility = 
["//platform/consensus/ordering/multipaxos:__subpackages__"])
+
+cc_library(
+    name = "proposal_manager",
+    srcs = ["proposal_manager.cpp"],
+    hdrs = ["proposal_manager.h"],
+    deps = [
+        "//common:comm",
+        "//common/crypto:signature_verifier",
+        "//common/utils",
+        "//platform/consensus/ordering/multipaxos/proto:proposal_cc_proto",
+    ],
+)
+
+cc_library(
+    name = "multipaxos",
+    srcs = ["multipaxos.cpp"],
+    hdrs = ["multipaxos.h"],
+    deps = [
+        ":proposal_manager",
+        "//common:comm",
+        "//platform/common/queue:lock_free_queue",
+        "//platform/consensus/ordering/common/algorithm:protocol_base",
+    ],
+)
+
+cc_library(
+    name = "tusk",
+    srcs = ["tusk.cpp"],
+    hdrs = ["tusk.h"],
+    deps = [
+        ":proposal_manager",
+        "//common/crypto:signature_verifier",
+        "//common:comm",
+        "//platform/common/queue:lock_free_queue",
+        "//platform/consensus/ordering/common/algorithm:protocol_base",
+    ],
+)
+
diff --git a/platform/consensus/ordering/multipaxos/algorithm/multipaxos.cpp 
b/platform/consensus/ordering/multipaxos/algorithm/multipaxos.cpp
new file mode 100644
index 00000000..a7966898
--- /dev/null
+++ b/platform/consensus/ordering/multipaxos/algorithm/multipaxos.cpp
@@ -0,0 +1,174 @@
+#include "platform/consensus/ordering/multipaxos/algorithm/multipaxos.h"
+
+#include <glog/logging.h>
+#include "common/utils/utils.h"
+
+
+namespace resdb {
+namespace multipaxos {
+
+MultiPaxos::MultiPaxos(int id, int f, int total_num, SignatureVerifier * 
verifier)
+  : ProtocolBase(id, f, total_num){
+
+    LOG(ERROR)<<"id:"<<id<<" f:"<<f<<" total:"<<total_num_;
+
+    proposal_manager_ = std::make_unique<ProposalManager>(id, 2*f_+1, 
verifier);
+    send_thread_ = std::thread(&MultiPaxos::AsyncSend, this);
+    commit_thread_ = std::thread(&MultiPaxos::AsyncCommit, this);
+    commit_seq_thread_ = std::thread(&MultiPaxos::AsyncCommitSeq, this);
+    batch_size_ = 10;
+    start_seq_ = 1;
+  }
+
+MultiPaxos::~MultiPaxos() {
+}
+
+
+void MultiPaxos::AsyncSend() {
+  while (!IsStop()) {
+    auto txn = txns_.Pop();
+    if(txn == nullptr){
+      continue;
+    }
+
+    std::vector<std::unique_ptr<Transaction> > txns;
+    txns.push_back(std::move(txn));
+    for(int i = 1; i < batch_size_; ++i){
+      auto txn = txns_.Pop();
+      if(txn == nullptr){
+        break;
+      }
+      txns.push_back(std::move(txn));
+    }
+
+    std::unique_ptr<Proposal> proposal =  nullptr;
+    {
+      proposal = proposal_manager_ -> GenerateProposal(txns);
+      //LOG(ERROR)<<"propose view:"<<proposal->header().view();
+    }
+    broadcast_call_(MessageType::Propose, *proposal);
+  }
+}
+
+
+void MultiPaxos::AddCommitData(std::unique_ptr<Proposal> p){
+  std::unique_lock<std::mutex> lk(n_mutex_);
+  commit_data_[p->header().proposal_id()] = std::move(p);
+  vote_cv_.notify_one();
+}
+
+std::unique_ptr<Proposal> MultiPaxos::GetCommitData(){
+  std::unique_lock<std::mutex> lk(n_mutex_);
+  auto p = std::move(commit_data_.begin()->second);
+  commit_data_.erase(commit_data_.begin());
+  start_seq_++;
+  return p;
+}
+
+bool MultiPaxos::Wait() {
+  auto Ready = [&]() {
+    return !commit_data_.empty() && commit_data_.begin()->first == start_seq_;
+  };
+
+  while(!IsStop()){
+      std::unique_lock<std::mutex> lk(n_mutex_);
+      vote_cv_.wait_for(lk, std::chrono::microseconds(1000),
+          [&] { return Ready(); });
+      if(Ready()){
+        return true;
+      }
+    }
+    return false;
+}
+
+void MultiPaxos::AsyncCommit() {
+  while (!IsStop()) {
+    auto p = commit_q_.Pop();
+    if(p == nullptr){
+      continue;
+    }
+
+    int proposer = p->header().proposer();
+    int round = p->header().view();
+    std::unique_ptr<Proposal> data = nullptr;
+    {
+      std::unique_lock<std::mutex> lk(mutex_);
+      auto it = receive_.find(round);
+      assert(it != receive_.end());
+      auto dit = it->second.find(proposer);
+      assert(dit != it->second.end());
+      data = std::move(dit->second);
+      assert(data != nullptr);
+    }
+    AddCommitData(std::move(data));
+  }
+}
+
+void MultiPaxos::AsyncCommitSeq() {
+  int seq = 1;
+  while (!IsStop()) {
+    if(!Wait()){
+      break;
+    }
+    std::unique_ptr<Proposal> data = GetCommitData();
+    int proposer = data->header().proposer();
+    int round = data->header().view();
+    int proposal_seq = data->header().proposal_id();
+    LOG(ERROR)<<" commit proposer:"<<proposer<< " round:"<<round<<" 
seq:"<<proposal_seq;
+    for(Transaction& txn : *data->mutable_transactions()){
+      txn.set_id(seq++);
+      Commit(txn);
+    }
+  }
+}
+
+bool MultiPaxos::ReceiveTransaction(std::unique_ptr<Transaction> txn) {
+  txn->set_proposer(id_);
+  txns_.Push(std::move(txn));
+  return true;
+}
+
+bool MultiPaxos::ReceiveProposal(std::unique_ptr<Proposal> proposal) {
+  int round = proposal->header().view();
+  int proposer = proposal->sender();
+  int seq = proposal->header().proposal_id();
+
+  bool done = false;
+  {
+    //LOG(ERROR)<<"recv proposal from:"<<proposer<<" round:"<<round<<" 
seq:"<<seq;
+    std::unique_lock<std::mutex> lk(mutex_);
+    receive_[round][proposer] = std::move(proposal);
+    done = true;
+  }
+  if(done) {
+    Proposal learn_proposal;
+    learn_proposal.mutable_header()->set_view(round);
+    learn_proposal.mutable_header()->set_proposer(proposer);
+    learn_proposal.mutable_header()->set_proposal_id(seq);
+    learn_proposal.set_sender(id_);
+    Broadcast(MessageType::Learn, learn_proposal);
+  }
+  return true;
+}
+
+bool MultiPaxos::ReceiveLearn(std::unique_ptr<Proposal> proposal) {
+  int round = proposal->header().view();
+  int sender = proposal->sender();
+  int proposer = proposal->header().proposer();
+
+  std::unique_lock<std::mutex> lk(learn_mutex_);
+  learn_receive_[round].insert(sender);
+  LOG(ERROR)<<"RECEIVE proposer learn:"<<round<<" 
size:"<<learn_receive_[round].size()<<" proposer:"<<proposer;
+  if(learn_receive_[round].size() == f_+1){
+    CommitProposal(std::move(proposal));
+  }
+  return true;
+}
+
+void MultiPaxos::CommitProposal(std::unique_ptr<Proposal> p){
+  commit_q_.Push(std::move(p));
+}
+
+
+}  // namespace tusk
+}  // namespace resdb
diff --git a/platform/consensus/ordering/multipaxos/algorithm/multipaxos.h 
b/platform/consensus/ordering/multipaxos/algorithm/multipaxos.h
new file mode 100644
index 00000000..6ade5312
--- /dev/null
+++ b/platform/consensus/ordering/multipaxos/algorithm/multipaxos.h
@@ -0,0 +1,60 @@
+#pragma once
+
+#include <thread>
+
+#include "platform/common/queue/lock_free_queue.h"
+#include "platform/consensus/ordering/multipaxos/algorithm/proposal_manager.h"
+#include "platform/consensus/ordering/multipaxos/proto/proposal.pb.h"
+#include "platform/consensus/ordering/common/algorithm/protocol_base.h"
+
+namespace resdb {
+namespace multipaxos {
+
+class MultiPaxos: public common::ProtocolBase {
+ public:
+  MultiPaxos(int id, int f, int total_num, SignatureVerifier* verifier );
+  ~MultiPaxos();
+
+  //  recv txn -> send block with links -> rec block ack -> send block with 
certs
+  bool ReceiveTransaction(std::unique_ptr<Transaction> txn);
+  bool ReceiveProposal(std::unique_ptr<Proposal> proposal);
+  bool ReceiveLearn(std::unique_ptr<Proposal> proposal);
+
+
+  private:
+    bool Ready();
+    void AsyncSend();
+    void AsyncCommit();
+    void AsyncCommitSeq();
+
+    void CommitProposal(std::unique_ptr<Proposal> p);
+
+    void AddCommitData(std::unique_ptr<Proposal> p);
+    std::unique_ptr<Proposal> GetCommitData();
+    bool Wait() ;
+
+
+ private:
+  LockFreeQueue<Transaction> txns_;
+  LockFreeQueue<Proposal> commit_q_;
+
+  std::unique_ptr<ProposalManager> proposal_manager_;
+
+  std::thread send_thread_, commit_thread_, commit_seq_thread_;
+
+  int batch_size_;
+
+  std::mutex mutex_;
+  std::map<int,  std::map<int, std::unique_ptr<Proposal>> > receive_;
+
+  std::mutex learn_mutex_;
+  std::map<int,  std::set<int>> learn_receive_;
+
+  std::mutex n_mutex_;
+  std::map<int, std::unique_ptr<Proposal> > commit_data_;
+  std::condition_variable vote_cv_;
+  int start_seq_;
+};
+
+}  // namespace tusk
+}  // namespace resdb
diff --git 
a/platform/consensus/ordering/multipaxos/algorithm/proposal_manager.cpp 
b/platform/consensus/ordering/multipaxos/algorithm/proposal_manager.cpp
new file mode 100644
index 00000000..1a55f8db
--- /dev/null
+++ b/platform/consensus/ordering/multipaxos/algorithm/proposal_manager.cpp
@@ -0,0 +1,36 @@
+#include "platform/consensus/ordering/multipaxos/algorithm/proposal_manager.h"
+
+#include <glog/logging.h>
+#include "common/utils/utils.h"
+#include "common/crypto/signature_verifier.h"
+
+namespace resdb {
+namespace multipaxos {
+
+ProposalManager::ProposalManager(int32_t id, int limit_count, 
SignatureVerifier* verifier)
+    : id_(id) {
+    round_ = 1;
+    seq_ = 1;
+    assert(verifier_ != nullptr);
+}
+
+std::unique_ptr<Proposal> ProposalManager::GenerateProposal(
+    const std::vector<std::unique_ptr<Transaction>>& txns) {
+  std::unique_ptr<Proposal> proposal = std::make_unique<Proposal>();
+  {
+    for(const auto& txn: txns){
+      *proposal->add_transactions() = *txn;
+    }
+    proposal->mutable_header()->set_proposal_id(seq_++);
+    proposal->mutable_header()->set_proposer(id_);
+    proposal->mutable_header()->set_view(round_++);
+    proposal->set_sender(id_);
+  }
+  return proposal;
+}
+
+int ProposalManager::CurrentView(){
+  return round_;
+}
+}  // namespace tusk
+}  // namespace resdb
diff --git 
a/platform/consensus/ordering/multipaxos/algorithm/proposal_manager.h 
b/platform/consensus/ordering/multipaxos/algorithm/proposal_manager.h
new file mode 100644
index 00000000..a8678b46
--- /dev/null
+++ b/platform/consensus/ordering/multipaxos/algorithm/proposal_manager.h
@@ -0,0 +1,25 @@
+#pragma once
+
+#include "platform/consensus/ordering/multipaxos/proto/proposal.pb.h"
+#include "common/crypto/signature_verifier.h"
+
+namespace resdb {
+namespace multipaxos {
+
+class ProposalManager {
+ public:
+  ProposalManager(int32_t id, int limit_count, SignatureVerifier * verifier);
+
+  std::unique_ptr<Proposal> GenerateProposal(const 
std::vector<std::unique_ptr<Transaction>>& txns);
+  int CurrentView();
+
+ private:
+  int32_t id_;
+  int round_;
+  int seq_;
+
+  SignatureVerifier* verifier_;
+};
+
+}  // namespace tusk
+}  // namespace resdb
diff --git a/platform/consensus/ordering/multipaxos/framework/BUILD 
b/platform/consensus/ordering/multipaxos/framework/BUILD
new file mode 100644
index 00000000..7dcc0ec3
--- /dev/null
+++ b/platform/consensus/ordering/multipaxos/framework/BUILD
@@ -0,0 +1,20 @@
+package(default_visibility = ["//visibility:private"])
+
+cc_library(
+    name = "consensus",
+    srcs = ["consensus.cpp"],
+    hdrs = ["consensus.h"],
+    visibility = [
+        "//visibility:public",
+    ],
+    deps = [
+        "//common/utils",
+        "//executor/common:transaction_manager",
+        "//platform/consensus/ordering/common/framework:consensus",
+        "//platform/consensus/execution:transaction_executor",
+        "//platform/consensus/ordering/multipaxos/algorithm:multipaxos",
+        "//platform/networkstrate:consensus_manager",
+    ],
+)
+
+
diff --git a/platform/consensus/ordering/multipaxos/framework/consensus.cpp 
b/platform/consensus/ordering/multipaxos/framework/consensus.cpp
new file mode 100644
index 00000000..b20d4e5e
--- /dev/null
+++ b/platform/consensus/ordering/multipaxos/framework/consensus.cpp
@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2019-2022 ExpoLab, UC Davis
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "platform/consensus/ordering/multipaxos/framework/consensus.h"
+
+#include <glog/logging.h>
+#include <unistd.h>
+
+#include "common/utils/utils.h"
+
+namespace resdb {
+namespace multipaxos {
+
+Consensus::Consensus(const ResDBConfig& config,
+                     std::unique_ptr<TransactionManager> executor)
+    : common::Consensus(config, std::move(executor)) {
+
+  Init();
+
+  int total_replicas = config_.GetReplicaNum();
+  int f = (total_replicas - 1) / 3;
+
+  if (config_.GetPublicKeyCertificateInfo()
+          .public_key()
+          .public_key_info()
+          .type() != CertificateKeyInfo::CLIENT) {
+    multipaxos_= std::make_unique<MultiPaxos>(config_.GetSelfInfo().id(), f, 
total_replicas, GetSignatureVerifier());
+    InitProtocol(multipaxos_.get());
+  }
+}
+
+int Consensus::ProcessCustomConsensus(std::unique_ptr<Request> request) {
+  //LOG(ERROR)<<"recv request:"<<MessageType_Name(request->user_type());
+  //int64_t current_time = GetCurrentTime();
+
+  if(request->user_type() == MessageType::Propose) {
+    std::unique_ptr<Proposal> p = std::make_unique<Proposal>();
+    if (!p->ParseFromString(request->data())) {
+      LOG(ERROR) << "parse proposal fail";
+      assert(1 == 0);
+      return -1;
+    }
+    multipaxos_->ReceiveProposal(std::move(p));
+  }
+  else if(request->user_type() == MessageType::Query) {
+    std::unique_ptr<Proposal> p = std::make_unique<Proposal>();
+    if (!p->ParseFromString(request->data())) {
+      LOG(ERROR) << "parse proposal fail";
+      assert(1 == 0);
+      return -1;
+    }
+    //multipaxos_->ReceivePropose(std::move(p));
+  }
+  else if(request->user_type() == MessageType::Learn) {
+    std::unique_ptr<Proposal> p = std::make_unique<Proposal>();
+    if (!p->ParseFromString(request->data())) {
+      LOG(ERROR) << "parse proposal fail";
+      assert(1 == 0);
+      return -1;
+    }
+    multipaxos_->ReceiveLearn(std::move(p));
+  }
+  return 0;
+}
+
+int Consensus::ProcessNewTransaction(std::unique_ptr<Request> request) {
+  std::unique_ptr<Transaction> txn = std::make_unique<Transaction>();
+  txn->set_data(request->data());
+  txn->set_hash(request->hash());
+  txn->set_proxy_id(request->proxy_id());
+  txn->set_user_seq(request->user_seq());
+  return multipaxos_->ReceiveTransaction(std::move(txn));
+}
+
+int Consensus::CommitMsg(const google::protobuf::Message& msg) {
+  return CommitMsgInternal(dynamic_cast<const Transaction&>(msg));
+}
+
+int Consensus::CommitMsgInternal(const Transaction& txn) {
+  //LOG(ERROR)<<"commit txn:"<<txn.id()<<" proxy id:"<<txn.proxy_id();
+  std::unique_ptr<Request> request = std::make_unique<Request>();
+  request->set_data(txn.data());
+  request->set_seq(txn.id());
+  request->set_proxy_id(txn.proxy_id());
+  transaction_executor_->AddExecuteMessage(std::move(request));
+  return 0;
+}
+
+
+}  // namespace fairdag
+}  // namespace resdb
diff --git a/platform/consensus/ordering/multipaxos/framework/consensus.h 
b/platform/consensus/ordering/multipaxos/framework/consensus.h
new file mode 100644
index 00000000..4d08c3ac
--- /dev/null
+++ b/platform/consensus/ordering/multipaxos/framework/consensus.h
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2019-2022 ExpoLab, UC Davis
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#pragma once
+
+#include "executor/common/transaction_manager.h"
+#include "platform/consensus/execution/transaction_executor.h"
+#include "platform/consensus/ordering/multipaxos/algorithm/multipaxos.h"
+#include "platform/consensus/ordering/common/framework/consensus.h"
+#include "platform/networkstrate/consensus_manager.h"
+
+namespace resdb {
+namespace multipaxos {
+
+class Consensus : public common::Consensus{
+ public:
+  Consensus(const ResDBConfig& config,
+            std::unique_ptr<TransactionManager> transaction_manager);
+
+  protected:
+  int ProcessCustomConsensus(std::unique_ptr<Request> request) override;
+  int ProcessNewTransaction(std::unique_ptr<Request> request) override;
+  int CommitMsg(const google::protobuf::Message& msg) override;
+  int CommitMsgInternal(const Transaction& txn);
+
+  private:
+    std::unique_ptr<MultiPaxos> multipaxos_;
+};
+
+}  // namespace tusk
+}  // namespace resdb
diff --git a/platform/consensus/ordering/multipaxos/proto/BUILD 
b/platform/consensus/ordering/multipaxos/proto/BUILD
new file mode 100644
index 00000000..1fe488c3
--- /dev/null
+++ b/platform/consensus/ordering/multipaxos/proto/BUILD
@@ -0,0 +1,16 @@
+package(default_visibility = 
["//platform/consensus/ordering/multipaxos:__subpackages__"])
+
+load("@rules_cc//cc:defs.bzl", "cc_proto_library")
+load("@rules_proto//proto:defs.bzl", "proto_library")
+load("@rules_proto_grpc//python:defs.bzl", "python_proto_library")
+
+proto_library(
+    name = "proposal_proto",
+    srcs = ["proposal.proto"],
+    deps = ["//common/proto:signature_info_proto"],
+)
+
+cc_proto_library(
+    name = "proposal_cc_proto",
+    deps = [":proposal_proto"],
+)
diff --git a/platform/consensus/ordering/multipaxos/proto/proposal.proto 
b/platform/consensus/ordering/multipaxos/proto/proposal.proto
new file mode 100644
index 00000000..83875513
--- /dev/null
+++ b/platform/consensus/ordering/multipaxos/proto/proposal.proto
@@ -0,0 +1,35 @@
+syntax = "proto3";
+import "common/proto/signature_info.proto";
+
+package resdb.multipaxos;
+
+message Transaction{
+  int32 id = 1;
+  bytes data = 2;
+  bytes hash = 3;
+  int32 proxy_id = 4;
+  int32 proposer = 5;
+  uint64 timestamp = 6;
+  int64 user_seq = 7;
+}
+
+message Header {
+  int32 proposer = 1;
+  int32 proposal_id = 2;
+  int32 view = 3;
+}
+
+message Proposal {
+  Header header = 1;
+  repeated Transaction transactions = 2;
+  int32 sender = 3;
+  bytes hash = 4;
+};
+
+enum MessageType {
+  NewProposal = 0;
+  Propose = 1;
+  Learn = 2;
+  Query = 3;
+};
+
diff --git a/platform/consensus/ordering/tusk/algorithm/tusk.cpp 
b/platform/consensus/ordering/tusk/algorithm/tusk.cpp
index 044e1e3d..0156e38f 100644
--- a/platform/consensus/ordering/tusk/algorithm/tusk.cpp
+++ b/platform/consensus/ordering/tusk/algorithm/tusk.cpp
@@ -300,9 +300,21 @@ bool Tusk::ReceiveTransaction(std::unique_ptr<Transaction> 
txn) {
 }
 
 bool Tusk::ReceiveBlock(std::unique_ptr<Proposal> proposal) {
-  //LOG(ERROR) << "recv block from " << proposal->header().proposer_id()
-  //           << " round:" << proposal->header().round();
 
+             /*
+  LOG(ERROR) << "recv block from " << proposal->header().proposer_id()
+             << " round:" << proposal->header().round();
+  */
+  if(proposal->header().round() == 200 && id_ >= 23 && 
proposal->header().proposer_id() < 23){
+    sleep(10);
+  }
+
+  if(proposal->header().round() == 200 && id_ <= 23 && 
proposal->header().proposer_id() > 23){
+  //LOG(ERROR) << "recv block from " << proposal->header().proposer_id()
+   //          << " round:" << proposal->header().round();
+    sleep(10);
+  }
+  
       std::unique_lock<std::mutex> lk(check_block_mutex_);
   {
    proposal->set_queuing_time(GetCurrentTime());
diff --git a/scripts/deploy/config/cassandra.config 
b/scripts/deploy/config/cassandra.config
index 193b4830..2d91c531 100644
--- a/scripts/deploy/config/cassandra.config
+++ b/scripts/deploy/config/cassandra.config
@@ -1,9 +1,9 @@
 {
-  "clientBatchNum": 800,
+  "clientBatchNum": 400,
   "enable_viewchange": false,
   "recovery_enabled": false,
   "max_client_complaint_num":10,
-  "max_process_txn": 256,
+  "max_process_txn": 512,
   "worker_num": 10,
   "input_worker_num": 1,
   "output_worker_num": 5
diff --git a/scripts/deploy/config/kv_performance_server.conf 
b/scripts/deploy/config/kv_performance_server.conf
index 498f5a98..98253c0a 100644
--- a/scripts/deploy/config/kv_performance_server.conf
+++ b/scripts/deploy/config/kv_performance_server.conf
@@ -1,8 +1,8 @@
 iplist=(
-172.31.23.110
-172.31.31.183
-172.31.22.246
-172.31.26.117
-172.31.21.196
+172.31.25.232
+172.31.17.37
+172.31.30.17
+172.31.27.41
 )
 
+client_num=1
diff --git a/scripts/deploy/config/kv_performance_server_16.conf 
b/scripts/deploy/config/kv_performance_server_16.conf
index f5b61302..981ff4eb 100644
--- a/scripts/deploy/config/kv_performance_server_16.conf
+++ b/scripts/deploy/config/kv_performance_server_16.conf
@@ -1,36 +1,36 @@
 iplist=(
-172.31.40.76
-172.31.37.77
-172.31.45.72
-172.31.41.9
-172.31.42.143
-172.31.38.17
-172.31.47.205
-172.31.41.14
-172.31.45.115
-172.31.45.118
-172.31.40.240
-172.31.45.242
-172.31.42.5
-172.31.39.133
-172.31.47.250
-172.31.46.126
-172.31.47.231
-172.31.44.39
-172.31.47.38
-172.31.38.102
-172.31.34.108
-172.31.42.173
-172.31.42.40
-172.31.43.42
-172.31.42.225
-172.31.39.35
-172.31.37.83
-172.31.40.218
-172.31.38.82
-172.31.34.146
-172.31.34.157
-172.31.35.222
+172.31.29.163
+172.31.22.39
+172.31.16.32
+172.31.27.98
+172.31.23.49
+172.31.18.178
+172.31.25.109
+172.31.17.109
+172.31.18.116
+172.31.25.117
+172.31.28.50
+172.31.24.115
+172.31.17.251
+172.31.17.252
+172.31.27.122
+172.31.26.123
+172.31.20.132
+172.31.20.70
+172.31.24.255
+172.31.25.132
+172.31.24.15
+172.31.20.16
+172.31.26.76
+172.31.18.15
+172.31.29.23
+172.31.19.215
+172.31.19.17
+172.31.31.149
+172.31.17.92
+172.31.31.159
+172.31.26.155
+172.31.24.155
 )
 
 key=~/.ssh/junchao.pem
diff --git a/scripts/deploy/config/kv_performance_server_32.conf 
b/scripts/deploy/config/kv_performance_server_32.conf
index bb245dda..aab16fae 100644
--- a/scripts/deploy/config/kv_performance_server_32.conf
+++ b/scripts/deploy/config/kv_performance_server_32.conf
@@ -1,68 +1,68 @@
 iplist=(
-172.31.35.170
-172.31.45.211
-172.31.40.83
-172.31.43.214
-172.31.40.20
-172.31.44.90
-172.31.36.216
-172.31.42.92
-172.31.34.198
-172.31.43.197
-172.31.36.136
-172.31.36.7
-172.31.39.203
-172.31.47.75
-172.31.36.14
-172.31.46.11
-172.31.42.185
-172.31.35.247
-172.31.32.252
-172.31.45.252
-172.31.45.189
-172.31.44.125
-172.31.39.129
-172.31.34.126
-172.31.40.111
-172.31.32.170
-172.31.42.178
-172.31.39.48
-172.31.47.53
-172.31.41.245
-172.31.41.246
-172.31.37.118
-172.31.40.76
-172.31.37.77
-172.31.45.72
-172.31.41.9
-172.31.42.143
-172.31.38.17
-172.31.47.205
-172.31.41.14
-172.31.45.115
-172.31.45.118
-172.31.40.240
-172.31.45.242
-172.31.42.5
-172.31.39.133
-172.31.47.250
-172.31.46.126
-172.31.47.231
-172.31.44.39
-172.31.47.38
-172.31.38.102
-172.31.34.108
-172.31.42.173
-172.31.42.40
-172.31.43.42
-172.31.42.225
-172.31.39.35
-172.31.37.83
-172.31.40.218
-172.31.38.82
-172.31.34.146
-172.31.34.157
-172.31.35.222
+172.31.31.59
+172.31.19.187
+172.31.28.180
+172.31.27.51
+172.31.20.3
+172.31.24.2
+172.31.29.63
+172.31.18.59
+172.31.26.174
+172.31.28.43
+172.31.27.40
+172.31.27.36
+172.31.20.179
+172.31.17.176
+172.31.27.48
+172.31.31.174
+172.31.20.20
+172.31.27.19
+172.31.17.17
+172.31.16.16
+172.31.23.31
+172.31.21.158
+172.31.24.152
+172.31.18.23
+172.31.16.11
+172.31.27.137
+172.31.26.6
+172.31.24.133
+172.31.20.14
+172.31.17.13
+172.31.18.141
+172.31.24.11
+172.31.30.254
+172.31.16.252
+172.31.29.120
+172.31.22.248
+172.31.28.200
+172.31.26.71
+172.31.24.197
+172.31.26.255
+172.31.24.233
+172.31.19.227
+172.31.28.98
+172.31.18.224
+172.31.19.119
+172.31.31.113
+172.31.25.111
+172.31.26.233
+172.31.17.218
+172.31.26.89
+172.31.16.87
+172.31.18.87
+172.31.26.34
+172.31.27.160
+172.31.24.93
+172.31.24.92
+172.31.18.83
+172.31.29.210
+172.31.30.76
+172.31.19.73
+172.31.21.87
+172.31.28.213
+172.31.23.211
+172.31.16.211
 )
 
 key=~/.ssh/junchao.pem
diff --git a/scripts/deploy/config/kv_performance_server_64.conf 
b/scripts/deploy/config/kv_performance_server_64.conf
index b2d40276..7e8be748 100644
--- a/scripts/deploy/config/kv_performance_server_64.conf
+++ b/scripts/deploy/config/kv_performance_server_64.conf
@@ -1,132 +1,132 @@
 iplist=(
-172.31.35.170
-172.31.45.211
-172.31.40.83
-172.31.43.214
-172.31.40.20
-172.31.44.90
-172.31.36.216
-172.31.42.92
-172.31.34.198
-172.31.43.197
-172.31.36.136
-172.31.36.7
-172.31.39.203
-172.31.47.75
-172.31.36.14
-172.31.46.11
-172.31.42.185
-172.31.35.247
-172.31.32.252
-172.31.45.252
-172.31.45.189
-172.31.44.125
-172.31.39.129
-172.31.34.126
-172.31.40.111
-172.31.32.170
-172.31.42.178
-172.31.39.48
-172.31.47.53
-172.31.41.245
-172.31.41.246
-172.31.37.118
-172.31.40.76
-172.31.37.77
-172.31.45.72
-172.31.41.9
-172.31.42.143
-172.31.38.17
-172.31.47.205
-172.31.41.14
-172.31.45.115
-172.31.45.118
-172.31.40.240
-172.31.45.242
-172.31.42.5
-172.31.39.133
-172.31.47.250
-172.31.46.126
-172.31.47.231
-172.31.44.39
-172.31.47.38
-172.31.38.102
-172.31.34.108
-172.31.42.173
-172.31.42.40
-172.31.43.42
-172.31.42.225
-172.31.39.35
-172.31.37.83
-172.31.40.218
-172.31.38.82
-172.31.34.146
-172.31.34.157
-172.31.35.222
-172.31.34.145
-172.31.35.17
-172.31.42.15
-172.31.40.145
-172.31.36.13
-172.31.47.142
-172.31.36.11
-172.31.43.140
-172.31.43.133
-172.31.41.137
-172.31.37.3
-172.31.43.132
-172.31.34.62
-172.31.38.0
-172.31.43.190
-172.31.35.190
-172.31.32.188
-172.31.43.61
-172.31.45.185
-172.31.47.186
-172.31.34.184
-172.31.40.184
-172.31.32.183
-172.31.38.55
-172.31.32.181
-172.31.41.181
-172.31.46.50
-172.31.35.179
-172.31.41.45
-172.31.34.178
-172.31.43.234
-172.31.34.107
-172.31.40.43
-172.31.41.172
-172.31.32.169
-172.31.41.170
-172.31.44.37
-172.31.39.38
-172.31.43.92
-172.31.34.35
-172.31.32.88
-172.31.47.91
-172.31.37.213
-172.31.41.86
-172.31.40.211
-172.31.40.212
-172.31.46.210
-172.31.43.82
-172.31.33.209
-172.31.40.82
-172.31.36.78
-172.31.39.207
-172.31.43.200
-172.31.46.206
-172.31.38.69
-172.31.43.198
-172.31.34.124
-172.31.45.65
-172.31.36.123
-172.31.35.124
-172.31.37.246
-172.31.41.247
-172.31.38.112
-172.31.44.240
+172.31.29.160
+172.31.22.239
+172.31.23.112
+172.31.21.177
+172.31.23.242
+172.31.26.233
+172.31.23.235
+172.31.16.108
+172.31.22.174
+172.31.23.56
+172.31.30.57
+172.31.22.58
+172.31.16.127
+172.31.16.245
+172.31.17.246
+172.31.17.118
+172.31.24.247
+172.31.24.204
+172.31.23.12
+172.31.29.140
+172.31.28.141
+172.31.31.3
+172.31.30.199
+172.31.29.73
+172.31.27.75
+172.31.19.87
+172.31.25.31
+172.31.18.95
+172.31.23.208
+172.31.22.83
+172.31.20.214
+172.31.16.87
+172.31.26.116
+172.31.27.244
+172.31.18.245
+172.31.25.121
+172.31.29.121
+172.31.27.254
+172.31.21.255
+172.31.27.255
+172.31.28.96
+172.31.21.231
+172.31.21.236
+172.31.24.237
+172.31.30.240
+172.31.22.242
+172.31.18.49
+172.31.18.178
+172.31.17.179
+172.31.26.181
+172.31.31.54
+172.31.29.55
+172.31.30.188
+172.31.27.190
+172.31.17.162
+172.31.31.168
+172.31.20.169
+172.31.22.171
+172.31.28.172
+172.31.24.173
+172.31.28.45
+172.31.19.174
+172.31.31.215
+172.31.22.216
+172.31.19.217
+172.31.28.217
+172.31.19.220
+172.31.18.221
+172.31.21.32
+172.31.17.34
+172.31.29.69
+172.31.23.202
+172.31.16.205
+172.31.30.208
+172.31.20.82
+172.31.28.211
+172.31.30.83
+172.31.30.84
+172.31.25.28
+172.31.18.30
+172.31.28.14
+172.31.19.14
+172.31.30.144
+172.31.31.149
+172.31.28.22
+172.31.31.22
+172.31.20.23
+172.31.22.155
+172.31.26.191
+172.31.21.1
+172.31.17.130
+172.31.17.6
+172.31.29.9
+172.31.29.12
+172.31.18.142
+172.31.26.142
+172.31.29.220
+172.31.24.157
+172.31.27.64
+172.31.30.5
+172.31.30.139
+172.31.27.13
+172.31.22.60
+172.31.22.189
+172.31.31.62
+172.31.24.192
+172.31.23.82
+172.31.23.83
+172.31.29.25
+172.31.23.154
+172.31.23.143
+172.31.29.16
+172.31.29.144
+172.31.22.144
+172.31.20.35
+172.31.25.228
+172.31.24.164
+172.31.16.165
+172.31.22.33
+172.31.29.161
+172.31.26.168
+172.31.31.173
+172.31.31.53
+172.31.20.54
+172.31.24.165
+172.31.26.230
+172.31.18.230
+172.31.22.104
 )
 
 key=~/.ssh/junchao.pem
diff --git a/scripts/deploy/config/rcc.config 
b/scripts/deploy/config/multipaxos.config
similarity index 88%
copy from scripts/deploy/config/rcc.config
copy to scripts/deploy/config/multipaxos.config
index 70ccdcc7..1ac9fe46 100644
--- a/scripts/deploy/config/rcc.config
+++ b/scripts/deploy/config/multipaxos.config
@@ -1,5 +1,5 @@
 {
-  "clientBatchNum": 200,
+  "clientBatchNum": 100,
   "enable_viewchange": false,
   "recovery_enabled": false,
   "max_client_complaint_num":10,
diff --git a/scripts/deploy/config/rcc.config b/scripts/deploy/config/rcc.config
index 70ccdcc7..985c7f80 100644
--- a/scripts/deploy/config/rcc.config
+++ b/scripts/deploy/config/rcc.config
@@ -1,5 +1,5 @@
 {
-  "clientBatchNum": 200,
+  "clientBatchNum": 400,
   "enable_viewchange": false,
   "recovery_enabled": false,
   "max_client_complaint_num":10,
diff --git a/scripts/deploy/config/tusk.config 
b/scripts/deploy/config/tusk.config
index 424cad47..80213d34 100644
--- a/scripts/deploy/config/tusk.config
+++ b/scripts/deploy/config/tusk.config
@@ -1,10 +1,10 @@
 {
-  "clientBatchNum": 200,
+  "clientBatchNum": 400,
   "enable_viewchange": false,
   "recovery_enabled": false,
   "max_client_complaint_num":10,
-  "max_process_txn": 384,
-  "worker_num": 10,
+  "max_process_txn": 4096,
+  "worker_num": 40,
   "input_worker_num": 1,
   "output_worker_num": 5
 }
diff --git a/scripts/deploy/performance/multipaxos_performance.sh 
b/scripts/deploy/performance/multipaxos_performance.sh
new file mode 100755
index 00000000..d9586847
--- /dev/null
+++ b/scripts/deploy/performance/multipaxos_performance.sh
@@ -0,0 +1,6 @@
+export server=//benchmark/protocols/multipaxos:kv_server_performance
+export TEMPLATE_PATH=$PWD/config/multipaxos.config
+#export COPTS="--define enable_leveldb=True"
+#export COPTS="-pg"
+
+./performance/run_performance.sh $*
diff --git a/scripts/deploy/performance/set_delay.sh 
b/scripts/deploy/performance/set_delay.sh
new file mode 100755
index 00000000..9bfe1048
--- /dev/null
+++ b/scripts/deploy/performance/set_delay.sh
@@ -0,0 +1,91 @@
+. ./script/env.sh
+
+
+KEY_FILE="config/key.conf"
+
+. ${KEY_FILE}
+iplist=(
+172.31.29.163
+172.31.22.39
+172.31.16.32
+172.31.27.98
+172.31.23.49
+172.31.18.178
+172.31.25.109
+172.31.17.109
+172.31.18.116
+172.31.25.117
+172.31.28.50
+172.31.24.115
+172.31.17.251
+172.31.17.252
+172.31.27.122
+172.31.26.123
+172.31.25.31
+#172.31.18.95
+#172.31.23.208
+#172.31.22.83
+#172.31.20.214
+#172.31.16.87
+#172.31.26.116
+#172.31.27.244
+#172.31.18.245
+#172.31.25.121
+#172.31.29.121
+#172.31.27.254
+#172.31.21.255
+#172.31.27.255
+#172.31.28.96
+#172.31.21.231
+#172.31.21.236
+#172.31.24.237
+#172.31.30.240
+#172.31.22.242
+#172.31.18.49
+#172.31.18.178
+#172.31.17.179
+#172.31.26.181
+#172.31.31.54
+#172.31.29.55
+#172.31.30.188
+#172.31.27.190
+#172.31.17.162
+#172.31.31.168
+#172.31.20.169
+#172.31.22.171
+#172.31.28.172
+#172.31.24.173
+#172.31.28.45
+#172.31.19.174
+#172.31.31.215
+#172.31.22.216
+
+)
+
+command1="sudo tc qdisc  add dev ens5 root netem delay 200ms"
+command2="sudo tc qdisc  change dev ens5 root netem delay 200ms"
+command3="tc qdisc show"
+
+echo ${command1}
+
+function run_cmd(){
+  ip=$1
+  cmd=$2
+  ssh -i ${key} -n -o BatchMode=yes -o StrictHostKeyChecking=no ubuntu@${ip} 
"${cmd}"
+  echo "ssh -i ${key} -n -o BatchMode=yes -o StrictHostKeyChecking=no 
ubuntu@${ip} \"${cmd}\""
+}
+
+for ip in ${iplist[@]};
+do
+  run_cmd ${ip} "${command1}" &
+  #run_cmd ${ip} "${command2}" &
+done
+
+wait 
+
+for ip in ${iplist[@]};
+do
+  run_cmd ${ip} "${command3}" 
+done
+
+wait 


Reply via email to