This is an automated email from the ASF dual-hosted git repository.
hulk pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks-controller.git
The following commit(s) were added to refs/heads/unstable by this push:
new fa7e9d1 Introduce etcd's raft to implement the KV store (#222)
fa7e9d1 is described below
commit fa7e9d160e0cd4fbc272d02770401771367bcd82
Author: hulk <[email protected]>
AuthorDate: Fri Dec 6 16:31:09 2024 +0800
Introduce etcd's raft to implement the KV store (#222)
Currently, the controller must depend on external services like
zookeeper/etcd to store its data. And it's not so friendly for users
since they need to set up another cluster before running the controller
server. To simplify this process, we would like to introduce the raft
engine to save and replicate the data.
---
.github/workflows/ci.yaml | 6 +-
config/config.go | 3 +
go.mod | 38 ++-
go.sum | 148 +++++------
server/server.go | 12 +-
store/engine/engine.go | 4 +-
store/engine/raft/config.go | 62 +++++
store/engine/raft/config_test.go | 59 +++++
store/engine/raft/node.go | 526 +++++++++++++++++++++++++++++++++++++++
store/engine/raft/node_test.go | 290 +++++++++++++++++++++
store/engine/raft/store.go | 227 +++++++++++++++++
store/engine/raft/store_test.go | 101 ++++++++
store/store.go | 1 +
13 files changed, 1366 insertions(+), 111 deletions(-)
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 8552770..5845856 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -27,7 +27,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v3
with:
- go-version: 1.19
+ go-version: 1.22
- name: Checkout Code Base
uses: actions/checkout@v3
@@ -50,7 +50,7 @@ jobs:
run: |
export GOPATH=$HOME/go
export PATH=$PATH:$GOPATH/bin
- curl -sSfL
https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh
-s -- -b $(go env GOPATH)/bin v1.52.2
+ curl -sSfL
https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh
-s -- -b $(go env GOPATH)/bin v1.62.0
make lint
build-test:
@@ -58,7 +58,7 @@ jobs:
needs: [lint]
strategy:
matrix:
- go-version: [1.19, oldstable, stable]
+ go-version: [1.22, stable]
os: [ubuntu-latest]
runs-on: ${{ matrix.os }}
steps:
diff --git a/config/config.go b/config/config.go
index 8916e48..80b820d 100644
--- a/config/config.go
+++ b/config/config.go
@@ -26,6 +26,8 @@ import (
"os"
"strings"
+ "github.com/apache/kvrocks-controller/store/engine/raft"
+
"github.com/go-playground/validator/v10"
"github.com/apache/kvrocks-controller/logger"
@@ -53,6 +55,7 @@ type Config struct {
StorageType string `yaml:"storage_type"`
Etcd *etcd.Config `yaml:"etcd"`
Zookeeper *zookeeper.Config `yaml:"zookeeper"`
+ Raft *raft.Config `yaml:"raft"`
Admin AdminConfig `yaml:"admin"`
Controller *ControllerConfig `yaml:"controller"`
}
diff --git a/go.mod b/go.mod
index d03e7a5..77d8437 100644
--- a/go.mod
+++ b/go.mod
@@ -1,8 +1,11 @@
module github.com/apache/kvrocks-controller
-go 1.19
+go 1.22
+
+toolchain go1.22.9
require (
+ github.com/fatih/color v1.16.0
github.com/gin-gonic/gin v1.9.1
github.com/go-playground/validator/v10 v10.14.0
github.com/go-redis/redis/v8 v8.11.5
@@ -11,9 +14,12 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/prometheus/client_golang v1.11.1
github.com/spf13/cobra v1.8.0
- github.com/stretchr/testify v1.8.3
+ github.com/stretchr/testify v1.9.0
go.etcd.io/etcd v3.3.27+incompatible
- go.etcd.io/etcd/client/v3 v3.5.4
+ go.etcd.io/etcd/client/pkg/v3 v3.5.17
+ go.etcd.io/etcd/client/v3 v3.5.17
+ go.etcd.io/etcd/raft/v3 v3.5.17
+ go.etcd.io/etcd/server/v3 v3.5.17
go.uber.org/atomic v1.7.0
go.uber.org/zap v1.21.0
gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0
@@ -22,7 +28,7 @@ require (
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/bytedance/sonic v1.9.1 // indirect
- github.com/cespare/xxhash/v2 v2.1.2 // indirect
+ github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 //
indirect
github.com/coreos/etcd v3.3.27+incompatible // indirect
github.com/coreos/go-semver v0.3.0 // indirect
@@ -31,17 +37,18 @@ require (
github.com/coreos/pkg v0.0.0-20230327231512-ba87abf18a23 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f //
indirect
- github.com/fatih/color v1.16.0 // indirect
+ github.com/dustin/go-humanize v1.0.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
- github.com/golang/protobuf v1.5.2 // indirect
+ github.com/golang/protobuf v1.5.4 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
+ github.com/kr/pretty v0.3.1 // indirect
github.com/leodido/go-urn v1.2.4 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
@@ -54,19 +61,26 @@ require (
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.26.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
+ github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
- go.etcd.io/etcd/api/v3 v3.5.4 // indirect
- go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
+ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 //
indirect
+ go.etcd.io/etcd/api/v3 v3.5.17 // indirect
+ go.etcd.io/etcd/pkg/v3 v3.5.17 // indirect
+ go.uber.org/goleak v1.3.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/crypto v0.21.0 // indirect
- golang.org/x/net v0.22.0 // indirect
+ golang.org/x/net v0.23.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
- google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c //
indirect
- google.golang.org/grpc v1.38.0 // indirect
- google.golang.org/protobuf v1.30.0 // indirect
+ golang.org/x/time v0.5.0 // indirect
+ google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d //
indirect
+ google.golang.org/genproto/googleapis/api
v0.0.0-20230822172742-b8732ec3820d // indirect
+ google.golang.org/genproto/googleapis/rpc
v0.0.0-20230822172742-b8732ec3820d // indirect
+ google.golang.org/grpc v1.59.0 // indirect
+ google.golang.org/protobuf v1.33.0 // indirect
+ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
diff --git a/go.sum b/go.sum
index 89003fb..9420fb6 100644
--- a/go.sum
+++ b/go.sum
@@ -1,12 +1,9 @@
-cloud.google.com/go v0.26.0/go.mod
h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod
h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1/go.mod
h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod
h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod
h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod
h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod
h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod
h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/antihax/optional v1.0.0/go.mod
h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/benbjohnson/clock v1.1.0
h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod
h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod
h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -16,16 +13,14 @@ github.com/beorn7/perks v1.0.1/go.mod
h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
github.com/bytedance/sonic v1.5.0/go.mod
h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
github.com/bytedance/sonic v1.9.1
h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
github.com/bytedance/sonic v1.9.1/go.mod
h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod
h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.1.1/go.mod
h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2
h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
-github.com/cespare/xxhash/v2 v2.1.2/go.mod
h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.2.0
h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod
h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod
h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311
h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod
h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
-github.com/client9/misspell v0.3.4/go.mod
h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod
h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod
h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cockroachdb/datadriven v1.0.2
h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA=
+github.com/cockroachdb/datadriven v1.0.2/go.mod
h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
github.com/coreos/etcd v3.3.27+incompatible
h1:QIudLb9KeBsE5zyYxd1mjzRSkzLg9Wf9QlRwFgd6oTA=
github.com/coreos/etcd v3.3.27+incompatible/go.mod
h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.3.0
h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
@@ -37,23 +32,20 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod
h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
github.com/coreos/pkg v0.0.0-20230327231512-ba87abf18a23
h1:SrdboTJZnOqc2r4cT4wQCzQJjGYwkclLwx2sPrDsx7g=
github.com/coreos/pkg v0.0.0-20230327231512-ba87abf18a23/go.mod
h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod
h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod
h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod
h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1
h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod
h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f
h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod
h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/dustin/go-humanize v1.0.0
h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod
h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod
h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane
v0.9.1-0.20191026205805-5f8ba28d4473/go.mod
h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod
h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane
v0.9.9-0.20210217033140-668b12f5399d/go.mod
h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod
h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod
h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/fsnotify/fsnotify v1.4.9
h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod
h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/gabriel-vasile/mimetype v1.4.2
h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
github.com/gabriel-vasile/mimetype v1.4.2/go.mod
h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
-github.com/ghodss/yaml v1.0.0/go.mod
h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gin-contrib/sse v0.1.0
h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
github.com/gin-contrib/sse v0.1.0/go.mod
h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
@@ -65,6 +57,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod
h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
github.com/go-logfmt/logfmt v0.4.0/go.mod
h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logfmt/logfmt v0.5.0/go.mod
h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-playground/assert/v2 v2.2.0
h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod
h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1
h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod
h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1
h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
@@ -84,35 +77,26 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod
h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x
github.com/gogo/protobuf v1.1.1/go.mod
h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod
h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod
h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/mock v1.1.1/go.mod
h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod
h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod
h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod
h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod
h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.4.0-rc.1/go.mod
h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod
h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod
h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod
h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod
h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod
h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2/go.mod
h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod
h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.0/go.mod
h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2
h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/golang/protobuf v1.5.2/go.mod
h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/go-cmp v0.2.0/go.mod
h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/golang/protobuf v1.5.4
h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod
h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.3.0/go.mod
h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod
h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod
h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod
h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod
h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod
h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod
h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod
h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/uuid v1.1.2/go.mod
h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod
h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
-github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod
h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/inconshreveable/mousetrap v1.1.0
h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod
h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jpillora/backoff v1.0.0/go.mod
h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
@@ -131,18 +115,19 @@ github.com/klauspost/cpuid/v2 v2.2.4/go.mod
h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8t
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod
h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod
h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod
h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod
h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1/go.mod
h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod
h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod
h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod
h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.2.4
h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
github.com/leodido/go-urn v1.2.4/go.mod
h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
github.com/mattn/go-colorable v0.1.13
h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod
h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod
h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.19
h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
-github.com/mattn/go-isatty v0.0.19/go.mod
h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.20
h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod
h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.9
h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
@@ -159,12 +144,16 @@ github.com/modern-go/reflect2 v1.0.2/go.mod
h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod
h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod
h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod
h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/olekukonko/tablewriter v0.0.5
h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod
h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
+github.com/onsi/ginkgo v1.16.5/go.mod
h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
+github.com/onsi/gomega v1.18.1/go.mod
h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
github.com/pelletier/go-toml/v2 v2.0.8
h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
github.com/pelletier/go-toml/v2 v2.0.8/go.mod
h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod
h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod
h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod
h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -178,7 +167,6 @@ github.com/prometheus/client_golang v1.11.1
h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs
github.com/prometheus/client_golang v1.11.1/go.mod
h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod
h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod
h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod
h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0
h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
github.com/prometheus/client_model v0.2.0/go.mod
h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.4.1/go.mod
h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
@@ -190,7 +178,9 @@ github.com/prometheus/procfs v0.0.2/go.mod
h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
github.com/prometheus/procfs v0.1.3/go.mod
h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0
h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
github.com/prometheus/procfs v0.6.0/go.mod
h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/rogpeppe/fastuuid v1.2.0/go.mod
h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-internal v1.9.0/go.mod
h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.10.0
h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod
h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/russross/blackfriday/v2 v2.1.0/go.mod
h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sirupsen/logrus v1.2.0/go.mod
h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod
h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -206,37 +196,45 @@ github.com/stretchr/objx v0.5.0/go.mod
h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE
github.com/stretchr/testify v1.2.2/go.mod
h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod
h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod
h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1/go.mod
h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod
h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod
h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod
h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod
h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2/go.mod
h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.3
h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
github.com/stretchr/testify v1.8.3/go.mod
h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0
h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod
h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/twitchyliquid64/golang-asm v0.15.1
h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod
h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go/codec v1.2.11
h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
github.com/ugorji/go/codec v1.2.11/go.mod
h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2
h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod
h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yuin/goldmark v1.1.27/go.mod
h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod
h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod
h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod
h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.etcd.io/etcd v3.3.27+incompatible
h1:5hMrpf6REqTHV2LW2OclNpRtxI0k9ZplMemJsMSWju0=
go.etcd.io/etcd v3.3.27+incompatible/go.mod
h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
-go.etcd.io/etcd/api/v3 v3.5.4 h1:OHVyt3TopwtUQ2GKdd5wu3PmmipR4FTwCqoEjSyRdIc=
-go.etcd.io/etcd/api/v3 v3.5.4/go.mod
h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
-go.etcd.io/etcd/client/pkg/v3 v3.5.4
h1:lrneYvz923dvC14R54XcA7FXoZ3mlGZAgmwhfm7HqOg=
-go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod
h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
-go.etcd.io/etcd/client/v3 v3.5.4
h1:p83BUL3tAYS0OT/r0qglgc3M1JjhM0diV8DSWAhVXv4=
-go.etcd.io/etcd/client/v3 v3.5.4/go.mod
h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY=
+go.etcd.io/etcd/api/v3 v3.5.17 h1:cQB8eb8bxwuxOilBpMJAEo8fAONyrdXTHUNcMd8yT1w=
+go.etcd.io/etcd/api/v3 v3.5.17/go.mod
h1:d1hvkRuXkts6PmaYk2Vrgqbv7H4ADfAKhyJqHNLJCB4=
+go.etcd.io/etcd/client/pkg/v3 v3.5.17
h1:XxnDXAWq2pnxqx76ljWwiQ9jylbpC4rvkAeRVOUKKVw=
+go.etcd.io/etcd/client/pkg/v3 v3.5.17/go.mod
h1:4DqK1TKacp/86nJk4FLQqo6Mn2vvQFBmruW3pP14H/w=
+go.etcd.io/etcd/client/v3 v3.5.17
h1:o48sINNeWz5+pjy/Z0+HKpj/xSnBkuVhVvXkjEXbqZY=
+go.etcd.io/etcd/client/v3 v3.5.17/go.mod
h1:j2d4eXTHWkT2ClBgnnEPm/Wuu7jsqku41v9DZ3OtjQo=
+go.etcd.io/etcd/pkg/v3 v3.5.17 h1:1k2wZ+oDp41jrk3F9o15o8o7K3/qliBo0mXqxo1PKaE=
+go.etcd.io/etcd/pkg/v3 v3.5.17/go.mod
h1:FrztuSuaJG0c7RXCOzT08w+PCugh2kCQXmruNYCpCGA=
+go.etcd.io/etcd/raft/v3 v3.5.17 h1:wHPW/b1oFBw/+HjDAQ9vfr17OIInejTIsmwMZpK1dNo=
+go.etcd.io/etcd/raft/v3 v3.5.17/go.mod
h1:uapEfOMPaJ45CqBYIraLO5+fqyIY2d57nFfxzFwy4D4=
+go.etcd.io/etcd/server/v3 v3.5.17
h1:xykBwLZk9IdDsB8z8rMdCCPRvhrG+fwvARaGA0TRiyc=
+go.etcd.io/etcd/server/v3 v3.5.17/go.mod
h1:40sqgtGt6ZJNKm8nk8x6LexZakPu+NDl/DCgZTZ69Cc=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod
h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
go.uber.org/goleak v1.1.11/go.mod
h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod
h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
go.uber.org/multierr v1.6.0/go.mod
h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8=
go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod
h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
@@ -250,30 +248,21 @@ golang.org/x/crypto
v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.19.0/go.mod
h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
golang.org/x/crypto v0.21.0/go.mod
h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod
h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod
h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod
h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod
h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod
h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod
h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod
h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod
h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod
h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod
h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod
h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod
h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod
h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod
h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod
h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod
h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod
h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod
h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod
h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod
h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod
h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod
h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod
h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
@@ -281,12 +270,10 @@ golang.org/x/net
v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod
h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod
h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod
h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod
h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod
h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod
h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod
h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -296,7 +283,6 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod
h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod
h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod
h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod
h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod
h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod
h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod
h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -310,7 +296,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod
h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod
h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod
h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod
h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod
h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod
h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod
h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod
h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -333,7 +318,6 @@ golang.org/x/term v0.18.0/go.mod
h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
@@ -342,62 +326,47 @@ golang.org/x/text v0.14.0/go.mod
h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod
h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod
h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod
h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod
h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod
h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod
h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod
h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod
h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod
h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.2/go.mod
h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5/go.mod
h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod
h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod
h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod
h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod
h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod
h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod
h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/appengine v1.1.0/go.mod
h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod
h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod
h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod
h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod
h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod
h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c
h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0=
-google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod
h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/grpc v1.19.0/go.mod
h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.23.0/go.mod
h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod
h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.27.0/go.mod
h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.33.1/go.mod
h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
-google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0=
-google.golang.org/grpc v1.38.0/go.mod
h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d
h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY=
+google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod
h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4=
+google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d
h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q=
+google.golang.org/genproto/googleapis/api
v0.0.0-20230822172742-b8732ec3820d/go.mod
h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d
h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4=
+google.golang.org/genproto/googleapis/rpc
v0.0.0-20230822172742-b8732ec3820d/go.mod
h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
+google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
+google.golang.org/grpc v1.59.0/go.mod
h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod
h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod
h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod
h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod
h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod
h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod
h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0/go.mod
h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod
h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.25.0/go.mod
h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod
h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod
h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.30.0
h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod
h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.33.0
h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod
h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod
h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod
h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod
h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15
h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod
h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod
h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod
h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0
h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU=
gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod
h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -408,7 +377,4 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod
h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod
h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod
h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod
h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
-sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
diff --git a/server/server.go b/server/server.go
index b4b8a08..fc99e43 100644
--- a/server/server.go
+++ b/server/server.go
@@ -28,6 +28,8 @@ import (
"strings"
"time"
+ "github.com/apache/kvrocks-controller/store/engine/raft"
+
"github.com/gin-gonic/gin"
"github.com/apache/kvrocks-controller/config"
@@ -53,13 +55,17 @@ func NewServer(cfg *config.Config) (*Server, error) {
var err error
sessionID := helper.GenerateSessionID(cfg.Addr)
- switch {
- case strings.EqualFold(cfg.StorageType, "etcd"):
+ storageType := strings.ToLower(cfg.StorageType)
+ switch storageType {
+ case "etcd":
logger.Get().Info("Use Etcd as store")
persist, err = etcd.New(sessionID, cfg.Etcd)
- case strings.EqualFold(cfg.StorageType, "zookeeper"):
+ case "zookeeper":
logger.Get().Info("Use Zookeeper as store")
persist, err = zookeeper.New(sessionID, cfg.Zookeeper)
+ case "raft":
+ logger.Get().Info("Use Raft as store")
+ persist, err = raft.New(cfg.Raft)
default:
logger.Get().Info("Use Etcd as default store")
persist, err = etcd.New(sessionID, cfg.Etcd)
diff --git a/store/engine/engine.go b/store/engine/engine.go
index 9b95594..b25685c 100644
--- a/store/engine/engine.go
+++ b/store/engine/engine.go
@@ -24,8 +24,8 @@ import (
)
type Entry struct {
- Key string
- Value []byte
+ Key string `json:"key"`
+ Value []byte `json:"value"`
}
type Engine interface {
diff --git a/store/engine/raft/config.go b/store/engine/raft/config.go
new file mode 100644
index 0000000..3ddf059
--- /dev/null
+++ b/store/engine/raft/config.go
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package raft
+
+import "errors"
+
+type Config struct {
+ // ID is the identity of the local raft. ID cannot be 0.
+ ID uint64
+ // DataDir is the directory to store the raft data which includes
snapshot and WALs.
+ DataDir string
+ // Join should be set to true if the node is joining an existing
cluster.
+ Join bool
+
+ // Peers is the list of raft peers.
+ Peers []string
+ HeartbeatSeconds int
+ ElectionSeconds int
+}
+
+func (c *Config) validate() error {
+ if c.ID == 0 {
+ return errors.New("ID cannot be 0")
+ }
+ if len(c.Peers) == 0 {
+ return errors.New("peers cannot be empty")
+ }
+ if c.ID > uint64(len(c.Peers)) {
+ return errors.New("ID cannot be greater than the number of
peers")
+ }
+ return nil
+}
+
+func (c *Config) init() {
+ if c.DataDir == "" {
+ c.DataDir = "."
+ }
+ if c.HeartbeatSeconds == 0 {
+ c.HeartbeatSeconds = 2
+ }
+ if c.ElectionSeconds == 0 {
+ c.ElectionSeconds = c.HeartbeatSeconds * 10
+ }
+}
diff --git a/store/engine/raft/config_test.go b/store/engine/raft/config_test.go
new file mode 100644
index 0000000..88ea00d
--- /dev/null
+++ b/store/engine/raft/config_test.go
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package raft
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestConfig_Validate(t *testing.T) {
+ c := &Config{}
+
+ // missing ID
+ require.ErrorContains(t, c.validate(), "ID cannot be 0")
+ // missing peers
+ c.ID = 1
+ require.ErrorContains(t, c.validate(), "peers cannot be empty")
+ // valid
+ c.Peers = []string{"http://127.0.0.1:12345"}
+ require.NoError(t, c.validate())
+ // ID greater than the number of peers
+ c.ID = 2
+ require.ErrorContains(t, c.validate(), "ID cannot be greater than the
number of peers")
+}
+
+func TestConfig_Init(t *testing.T) {
+ c := &Config{}
+ c.init()
+ require.Equal(t, ".", c.DataDir)
+ require.Equal(t, 2, c.HeartbeatSeconds)
+ require.Equal(t, 20, c.ElectionSeconds)
+
+ c.DataDir = "/tmp"
+ c.HeartbeatSeconds = 3
+ c.ElectionSeconds = 30
+ c.init()
+ require.Equal(t, "/tmp", c.DataDir)
+ require.Equal(t, 3, c.HeartbeatSeconds)
+ require.Equal(t, 30, c.ElectionSeconds)
+}
diff --git a/store/engine/raft/node.go b/store/engine/raft/node.go
new file mode 100644
index 0000000..3eed408
--- /dev/null
+++ b/store/engine/raft/node.go
@@ -0,0 +1,526 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package raft
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net/http"
+ "net/url"
+ "os"
+ "sync"
+ "time"
+
+ "github.com/apache/kvrocks-controller/logger"
+ "github.com/apache/kvrocks-controller/store/engine"
+
+ "go.etcd.io/etcd/client/pkg/v3/types"
+ "go.etcd.io/etcd/raft/v3"
+ "go.etcd.io/etcd/raft/v3/raftpb"
+ "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp"
+ stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats"
+
+ "go.uber.org/atomic"
+ "go.uber.org/zap"
+)
+
+const (
+ defaultSnapshotThreshold = 10000
+ defaultCompactThreshold = 1024
+)
+
+const (
+ opGet = iota + 1
+ opSet
+ opDelete
+)
+
+type Event struct {
+ Op int `json:"op"`
+ Key string `json:"key"`
+ Value []byte `json:"value"`
+}
+
+type Node struct {
+ config *Config
+
+ addr string
+ raftNode raft.Node
+ transport *rafthttp.Transport
+ httpServer *http.Server
+ dataStore *DataStore
+ leaderChanged chan bool
+ logger *zap.Logger
+ peers sync.Map
+
+ mu sync.Mutex
+ leader uint64
+ appliedIndex uint64
+ snapshotIndex uint64
+ confState raftpb.ConfState
+ snapshotThreshold uint64
+ compactThreshold uint64
+
+ wg sync.WaitGroup
+ shutdown chan struct{}
+
+ isRunning atomic.Bool
+}
+
+var _ engine.Engine = (*Node)(nil)
+
+func New(config *Config) (*Node, error) {
+ config.init()
+ if err := config.validate(); err != nil {
+ return nil, err
+ }
+
+ logger := logger.Get().With(zap.Uint64("node_id", config.ID))
+ n := &Node{
+ config: config,
+ leader: raft.None,
+ dataStore: NewDataStore(config.DataDir),
+ leaderChanged: make(chan bool),
+ snapshotThreshold: defaultSnapshotThreshold,
+ compactThreshold: defaultCompactThreshold,
+ logger: logger,
+ }
+ if err := n.run(); err != nil {
+ return nil, err
+ }
+ return n, nil
+}
+
+func (n *Node) Addr() string {
+ return n.addr
+}
+
+func (n *Node) Peers() []string {
+ peers := make([]string, 0)
+ n.peers.Range(func(key, value interface{}) bool {
+ peer, _ := value.(string)
+ peers = append(peers, peer)
+ return true
+ })
+ return peers
+}
+
+func (n *Node) SetSnapshotThreshold(threshold uint64) {
+ n.mu.Lock()
+ defer n.mu.Unlock()
+ n.snapshotThreshold = threshold
+}
+
+func (n *Node) run() error {
+ // The node is already running
+ if !n.isRunning.CAS(false, true) {
+ return nil
+ }
+ n.shutdown = make(chan struct{})
+
+ peers := make([]raft.Peer, len(n.config.Peers))
+ for i, peer := range n.config.Peers {
+ peers[i] = raft.Peer{
+ ID: uint64(i + 1),
+ Context: []byte(peer),
+ }
+ }
+ raftConfig := &raft.Config{
+ ID: n.config.ID,
+ HeartbeatTick: n.config.HeartbeatSeconds,
+ ElectionTick: n.config.ElectionSeconds,
+ MaxInflightMsgs: 128,
+ MaxSizePerMsg: 10 * 1024 * 1024, // 10 MiB
+ Storage: n.dataStore.raftStorage,
+ }
+
+ // WAL existing check must be done before replayWAL since it will
create a new WAL if not exists
+ walExists := n.dataStore.walExists()
+ if err := n.dataStore.replayWAL(); err != nil {
+ return err
+ }
+
+ if n.config.Join || walExists {
+ n.raftNode = raft.RestartNode(raftConfig)
+ } else {
+ n.raftNode = raft.StartNode(raftConfig, peers)
+ }
+
+ if err := n.runTransport(); err != nil {
+ return err
+ }
+ return n.runRaftMessages()
+}
+
+func (n *Node) runTransport() error {
+ logger := logger.Get()
+ idString := fmt.Sprintf("%d", n.config.ID)
+ transport := &rafthttp.Transport{
+ ID: types.ID(n.config.ID),
+ Logger: logger,
+ ClusterID: 0x6666,
+ Raft: n,
+ LeaderStats: stats.NewLeaderStats(logger, idString),
+ ServerStats: stats.NewServerStats("raft", idString),
+ ErrorC: make(chan error),
+ }
+ if err := transport.Start(); err != nil {
+ return fmt.Errorf("unable to start transport: %w", err)
+ }
+ for i, peer := range n.config.Peers {
+ // Don't add self to transport
+ if uint64(i+1) != n.config.ID {
+ transport.AddPeer(types.ID(i+1), []string{peer})
+ }
+ n.peers.Store(uint64(i+1), peer)
+ }
+
+ n.addr = n.config.Peers[n.config.ID-1]
+ url, err := url.Parse(n.addr)
+ if err != nil {
+ return err
+ }
+ httpServer := &http.Server{
+ Addr: url.Host,
+ Handler: transport.Handler(),
+ }
+
+ n.wg.Add(1)
+ go func() {
+ defer n.wg.Done()
+ if err := httpServer.ListenAndServe(); err != nil &&
!errors.Is(err, http.ErrServerClosed) {
+ n.logger.Fatal("Unable to start http server",
zap.Error(err))
+ os.Exit(1)
+ }
+ }()
+
+ n.transport = transport
+ n.httpServer = httpServer
+ return nil
+}
+
+func (n *Node) runRaftMessages() error {
+ snapshot, err := n.dataStore.loadSnapshotFromDisk()
+ if err != nil {
+ return err
+ }
+
+ // Load the snapshot into the key-value store.
+ if err := n.dataStore.reloadSnapshot(); err != nil {
+ return err
+ }
+ n.appliedIndex = snapshot.Metadata.Index
+ n.snapshotIndex = snapshot.Metadata.Index
+ n.confState = snapshot.Metadata.ConfState
+
+ n.wg.Add(1)
+ go func() {
+ ticker := time.NewTicker(100 * time.Millisecond)
+ defer func() {
+ ticker.Stop()
+ n.wg.Done()
+ }()
+
+ for {
+ select {
+ case <-ticker.C:
+ n.raftNode.Tick()
+ case rd := <-n.raftNode.Ready():
+ // Save to wal and storage first
+ if !raft.IsEmptySnap(rd.Snapshot) {
+ if err :=
n.dataStore.saveSnapshot(rd.Snapshot); err != nil {
+ n.logger.Error("Failed to save
snapshot", zap.Error(err))
+ }
+ }
+ if err := n.dataStore.wal.Save(rd.HardState,
rd.Entries); err != nil {
+ n.logger.Error("Failed to save to wal",
zap.Error(err))
+ }
+
+ // Replay the entries into the raft storage
+ if err := n.applySnapshot(rd.Snapshot); err !=
nil {
+ n.logger.Error("Failed to apply
snapshot", zap.Error(err))
+ }
+ if len(rd.Entries) > 0 {
+ _ =
n.dataStore.raftStorage.Append(rd.Entries)
+ }
+
+ for _, msg := range rd.Messages {
+ if msg.Type == raftpb.MsgApp {
+ msg.Snapshot.Metadata.ConfState
= n.confState
+ }
+ }
+ n.transport.Send(rd.Messages)
+
+ // Apply the committed entries to the state
machine
+ n.applyEntries(rd.CommittedEntries)
+ if err := n.triggerSnapshotIfNeed(); err != nil
{
+ n.logger.Error("Failed to trigger
snapshot", zap.Error(err))
+ }
+ n.raftNode.Advance()
+ case err := <-n.transport.ErrorC:
+ n.logger.Fatal("Found transport error",
zap.Error(err))
+ return
+ case <-n.shutdown:
+ n.logger.Info("Shutting down raft node")
+ return
+ }
+ }
+ }()
+ return nil
+}
+
+func (n *Node) triggerSnapshotIfNeed() error {
+ if n.appliedIndex-n.snapshotIndex <= n.snapshotThreshold {
+ return nil
+ }
+ snapshotBytes, err := n.dataStore.GetDataStoreSnapshot()
+ if err != nil {
+ return err
+ }
+ snap, err := n.dataStore.raftStorage.CreateSnapshot(n.appliedIndex,
&n.confState, snapshotBytes)
+ if err != nil {
+ return err
+ }
+ if err := n.dataStore.saveSnapshot(snap); err != nil {
+ return err
+ }
+
+ compactIndex := uint64(1)
+ if n.appliedIndex > n.compactThreshold {
+ compactIndex = n.appliedIndex - n.compactThreshold
+ }
+ if err := n.dataStore.raftStorage.Compact(compactIndex); err != nil &&
!errors.Is(err, raft.ErrCompacted) {
+ return err
+ }
+ n.snapshotIndex = n.appliedIndex
+ return nil
+}
+
+func (n *Node) Set(ctx context.Context, key string, value []byte) error {
+ bytes, err := json.Marshal(&Event{
+ Op: opSet,
+ Key: key,
+ Value: value,
+ })
+ if err != nil {
+ return err
+ }
+ return n.raftNode.Propose(ctx, bytes)
+}
+
+func (n *Node) AddPeer(ctx context.Context, nodeID uint64, peer string) error {
+ cc := raftpb.ConfChange{
+ Type: raftpb.ConfChangeAddNode,
+ NodeID: nodeID,
+ Context: []byte(peer),
+ }
+ return n.raftNode.ProposeConfChange(ctx, cc)
+}
+
+func (n *Node) RemovePeer(ctx context.Context, nodeID uint64) error {
+ cc := raftpb.ConfChange{
+ Type: raftpb.ConfChangeRemoveNode,
+ NodeID: nodeID,
+ }
+ return n.raftNode.ProposeConfChange(ctx, cc)
+}
+
+func (n *Node) ID() string {
+ return fmt.Sprintf("%d", n.config.ID)
+}
+
+func (n *Node) Leader() string {
+ return fmt.Sprintf("%d", n.GetRaftLead())
+}
+
+func (n *Node) GetRaftLead() uint64 {
+ return n.raftNode.Status().Lead
+}
+
+func (n *Node) IsReady(_ context.Context) bool {
+ return n.raftNode.Status().Lead != raft.None
+}
+
+func (n *Node) LeaderChange() <-chan bool {
+ return n.leaderChanged
+}
+
+func (n *Node) Get(_ context.Context, key string) ([]byte, error) {
+ return n.dataStore.Get(key)
+}
+
+func (n *Node) Exists(_ context.Context, key string) (bool, error) {
+ _, err := n.dataStore.Get(key)
+ if err != nil {
+ if errors.Is(err, ErrKeyNotFound) {
+ return false, nil
+ }
+ return false, err
+ }
+ return true, nil
+}
+
+func (n *Node) Delete(ctx context.Context, key string) error {
+ bytes, err := json.Marshal(&Event{
+ Op: opDelete,
+ Key: key,
+ })
+ if err != nil {
+ return err
+ }
+ return n.raftNode.Propose(ctx, bytes)
+}
+
+func (n *Node) List(_ context.Context, prefix string) ([]engine.Entry, error) {
+ n.dataStore.List(prefix)
+ return nil, nil
+}
+
+func (n *Node) applySnapshot(snapshot raftpb.Snapshot) error {
+ if raft.IsEmptySnap(snapshot) {
+ return nil
+ }
+
+ _ = n.dataStore.raftStorage.ApplySnapshot(snapshot)
+ if n.appliedIndex >= snapshot.Metadata.Index {
+ return fmt.Errorf("snapshot index [%d] should be greater than
applied index [%d]", snapshot.Metadata.Index, n.appliedIndex)
+ }
+
+ // Load the snapshot into the key-value store.
+ if err := n.dataStore.reloadSnapshot(); err != nil {
+ return err
+ }
+ n.confState = snapshot.Metadata.ConfState
+ n.appliedIndex = snapshot.Metadata.Index
+ n.snapshotIndex = snapshot.Metadata.Index
+ return nil
+}
+
+func (n *Node) applyEntries(entries []raftpb.Entry) {
+ if len(entries) == 0 || entries[0].Index > n.appliedIndex+1 {
+ return
+ }
+
+ firstEntryIndex := entries[0].Index
+ // remove entries that have been applied
+ if n.appliedIndex-firstEntryIndex+1 < uint64(len(entries)) {
+ entries = entries[n.appliedIndex-firstEntryIndex+1:]
+ }
+ for _, entry := range entries {
+ if err := n.applyEntry(entry); err != nil {
+ n.logger.Error("failed to apply entry", zap.Error(err))
+ }
+ }
+ n.appliedIndex = entries[len(entries)-1].Index
+}
+
+func (n *Node) applyEntry(entry raftpb.Entry) error {
+ switch entry.Type {
+ case raftpb.EntryNormal:
+ // apply entry to the state machine
+ if len(entry.Data) == 0 {
+ // empty message, skip it.
+ return nil
+ }
+
+ var e Event
+ if err := json.Unmarshal(entry.Data, &e); err != nil {
+ return err
+ }
+ switch e.Op {
+ case opSet:
+ n.dataStore.Set(e.Key, e.Value)
+ return nil
+ case opDelete:
+ n.dataStore.Delete(e.Key)
+ case opGet:
+ // do nothing
+ default:
+ return fmt.Errorf("unknown operation type: %d", e.Op)
+ }
+ case raftpb.EntryConfChangeV2, raftpb.EntryConfChange:
+ // apply config change to the state machine
+ var cc raftpb.ConfChange
+ if err := cc.Unmarshal(entry.Data); err != nil {
+ return err
+ }
+
+ n.confState = *n.raftNode.ApplyConfChange(cc)
+ switch cc.Type {
+ case raftpb.ConfChangeAddNode:
+ if cc.NodeID != n.config.ID && len(cc.Context) > 0 {
+ n.logger.Info("Add the new peer",
zap.String("context", string(cc.Context)))
+ n.transport.AddPeer(types.ID(cc.NodeID),
[]string{string(cc.Context)})
+ n.peers.Store(cc.NodeID, string(cc.Context))
+ }
+ case raftpb.ConfChangeRemoveNode:
+ n.peers.Delete(cc.NodeID)
+ n.transport.RemovePeer(types.ID(cc.NodeID))
+ if cc.NodeID == n.config.ID {
+ n.Close()
+ n.logger.Info("Node removed from the cluster")
+ return nil
+ }
+ case raftpb.ConfChangeUpdateNode:
+ n.transport.UpdatePeer(types.ID(cc.NodeID),
[]string{string(cc.Context)})
+ if _, ok := n.peers.Load(cc.NodeID); ok {
+ n.peers.Store(cc.NodeID, string(cc.Context))
+ }
+ case raftpb.ConfChangeAddLearnerNode:
+ // TODO: add the learner node
+ }
+ }
+ return nil
+}
+
+func (n *Node) Process(ctx context.Context, m raftpb.Message) error {
+ return n.raftNode.Step(ctx, m)
+}
+
+func (n *Node) IsIDRemoved(_ uint64) bool {
+ return false
+}
+
+func (n *Node) ReportUnreachable(id uint64) {
+ n.raftNode.ReportUnreachable(id)
+}
+
+func (n *Node) ReportSnapshot(id uint64, status raft.SnapshotStatus) {
+ n.raftNode.ReportSnapshot(id, status)
+}
+
+func (n *Node) Close() error {
+ if !n.isRunning.CAS(true, false) {
+ return nil
+ }
+ close(n.shutdown)
+ n.raftNode.Stop()
+ n.transport.Stop()
+ if err := n.httpServer.Close(); err != nil {
+ return err
+ }
+
+ n.dataStore.Close()
+ n.wg.Wait()
+ return nil
+}
diff --git a/store/engine/raft/node_test.go b/store/engine/raft/node_test.go
new file mode 100644
index 0000000..cb60cb0
--- /dev/null
+++ b/store/engine/raft/node_test.go
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package raft
+
+import (
+ "context"
+ "fmt"
+ "math/rand"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "go.etcd.io/etcd/raft/v3"
+)
+
+type TestCluster struct {
+ nodes []*Node
+}
+
+func NewTestCluster(n int) *TestCluster {
+ if n > 16 {
+ n = 16
+ }
+ nodes := make([]*Node, n)
+ randomStartPort := rand.Int31n(1024) + 10000
+ peers := make([]string, n)
+ for i := 0; i < n; i++ {
+ peers[i] = fmt.Sprintf("http://127.0.0.1:%d";,
randomStartPort+int32(i))
+ }
+ for i := 0; i < n; i++ {
+ nodes[i], _ = New(&Config{
+ ID: uint64(i + 1),
+ DataDir: fmt.Sprintf("/tmp/kvrocks/raft/%d",
randomStartPort+int32(i)),
+ Peers: peers,
+ HeartbeatSeconds: 1,
+ ElectionSeconds: 2,
+ })
+ }
+ return &TestCluster{nodes: nodes}
+}
+
+func (c *TestCluster) createNode(peers []string) (*Node, error) {
+ randomPort := rand.Int31n(1024) + 20000
+ addr := fmt.Sprintf("http://127.0.0.1:%d";, randomPort)
+ node, err := New(&Config{
+ ID: uint64(len(peers) + 1),
+ DataDir: fmt.Sprintf("/tmp/kvrocks/raft/%d",
randomPort),
+ Peers: append(peers, addr),
+ HeartbeatSeconds: 1,
+ ElectionSeconds: 2,
+ })
+ return node, err
+}
+
+func (c *TestCluster) AddNode(ctx context.Context, nodeID uint64, peer string)
error {
+ if len(c.nodes) == 0 {
+ return nil
+ }
+ return c.nodes[0].AddPeer(ctx, nodeID, peer)
+}
+
+func (c *TestCluster) RemoveNode(ctx context.Context, nodeID uint64) error {
+ for i, n := range c.nodes {
+ if n.config.ID == nodeID {
+ c.nodes = append(c.nodes[:i], c.nodes[i+1:]...)
+ break
+ }
+ }
+ if len(c.nodes) == 0 {
+ return nil
+ }
+ return c.nodes[0].RemovePeer(ctx, nodeID)
+}
+
+func (c *TestCluster) SetSnapshotThreshold(threshold uint64) {
+ for _, n := range c.nodes {
+ n.SetSnapshotThreshold(threshold)
+ }
+}
+
+func (c *TestCluster) IsReady(ctx context.Context) bool {
+ for _, n := range c.nodes {
+ if !n.IsReady(ctx) {
+ return false
+ }
+ }
+ return true
+}
+
+func (c *TestCluster) GetNode(i int) *Node {
+ if i < 0 || i >= len(c.nodes) {
+ return nil
+ }
+ return c.nodes[i]
+}
+
+func (c *TestCluster) GetLeaderNode() *Node {
+ leaderID := raft.None
+ for _, n := range c.nodes {
+ if n.GetRaftLead() == leaderID {
+ continue
+ }
+ leaderID = n.GetRaftLead()
+ }
+ if leaderID == raft.None {
+ return nil
+ }
+ return c.GetNode(int(leaderID - 1))
+}
+
+func (c *TestCluster) ListNodes() []*Node {
+ return c.nodes
+}
+
+func (c *TestCluster) Restart() error {
+ for _, n := range c.nodes {
+ n.Close()
+ }
+ for _, n := range c.nodes {
+ if err := n.run(); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (c *TestCluster) Close() {
+ for _, n := range c.nodes {
+ n.Close()
+ os.RemoveAll(n.config.DataDir)
+ }
+}
+
+func TestCluster_SingleNode(t *testing.T) {
+ cluster := NewTestCluster(1)
+ defer cluster.Close()
+
+ ctx := context.Background()
+ require.Eventually(t, func() bool {
+ return cluster.IsReady(ctx)
+ }, 10*time.Second, 100*time.Millisecond)
+
+ n := cluster.GetNode(0)
+ require.NotNil(t, n)
+ require.NoError(t, n.Set(ctx, "foo", []byte("bar")))
+
+ require.Eventually(t, func() bool {
+ gotBytes, _ := n.Get(ctx, "foo")
+ return string(gotBytes) == "bar"
+ }, 1*time.Second, 100*time.Millisecond)
+}
+
+func TestCluster_MultiNodes(t *testing.T) {
+ cluster := NewTestCluster(3)
+ defer cluster.Close()
+
+ ctx := context.Background()
+ require.Eventually(t, func() bool {
+ return cluster.IsReady(ctx)
+ }, 10*time.Second, 100*time.Millisecond)
+
+ t.Run("works well with all nodes ready", func(t *testing.T) {
+ n1 := cluster.GetNode(0)
+ n2 := cluster.GetNode(1)
+ require.NoError(t, n1.Set(ctx, "foo", []byte("bar")))
+ require.Eventually(t, func() bool {
+ got, _ := n2.Get(ctx, "foo")
+ return string(got) == "bar"
+ }, 1*time.Second, 100*time.Millisecond)
+ })
+
+ t.Run("works well if 1/3 nodes down", func(t *testing.T) {
+ oldLeaderNode := cluster.GetLeaderNode()
+ require.NotNil(t, oldLeaderNode)
+ oldLeaderNode.Close()
+
+ require.Eventually(t, func() bool {
+ newLeaderNode := cluster.GetLeaderNode()
+ return newLeaderNode != nil && newLeaderNode !=
oldLeaderNode
+ }, 10*time.Second, 200*time.Millisecond)
+
+ leaderNode := cluster.GetLeaderNode()
+ require.NoError(t, leaderNode.Set(ctx, "foo", []byte("bar")))
+ })
+}
+
+func TestCluster_AddRemovePeer(t *testing.T) {
+ cluster := NewTestCluster(3)
+ defer cluster.Close()
+
+ ctx := context.Background()
+ require.Eventually(t, func() bool {
+ return cluster.IsReady(ctx)
+ }, 10*time.Second, 100*time.Millisecond)
+
+ n1 := cluster.GetNode(0)
+ require.NoError(t, n1.Set(ctx, "foo", []byte("bar")))
+ require.Eventually(t, func() bool {
+ got, _ := n1.Get(ctx, "foo")
+ return string(got) == "bar"
+ }, 1*time.Second, 100*time.Millisecond)
+
+ t.Run("add a new peer node", func(t *testing.T) {
+ n4, err := cluster.createNode(n1.config.Peers)
+ require.NoError(t, err)
+ require.NotNil(t, n4)
+
+ require.NoError(t, cluster.AddNode(ctx, n4.config.ID,
n4.Addr()))
+ require.Eventually(t, func() bool {
+ return n4.IsReady(ctx)
+ }, 10*time.Second, 100*time.Millisecond)
+
+ require.NoError(t, n4.Set(ctx, "foo", []byte("bar-1")))
+ require.Eventually(t, func() bool {
+ got, _ := n1.Get(ctx, "foo")
+ return string(got) == "bar-1"
+ }, 1*time.Second, 100*time.Millisecond)
+ require.Len(t, n1.Peers(), 4)
+ })
+
+ t.Run("remove a peer node", func(t *testing.T) {
+ cluster.RemoveNode(ctx, 4)
+ require.Eventually(t, func() bool {
+ return len(n1.Peers()) == 3
+ }, 10*time.Second, 100*time.Millisecond)
+ })
+}
+
+func TestTriggerSnapshot(t *testing.T) {
+ cluster := NewTestCluster(3)
+ defer cluster.Close()
+
+ ctx := context.Background()
+ require.Eventually(t, func() bool {
+ return cluster.IsReady(ctx)
+ }, 10*time.Second, 100*time.Millisecond)
+
+ cnt := 128
+ cluster.SetSnapshotThreshold(uint64(cnt / 5))
+
+ n := cluster.GetNode(0)
+ require.NotNil(t, n)
+ for i := 0; i < cnt; i++ {
+ require.NoError(t, n.Set(ctx, fmt.Sprintf("foo%d", i),
[]byte("bar")))
+ }
+
+ // Use Eventually to wait for snapshot to be triggered
+ require.Eventually(t, func() bool {
+ allNodesHasSnapshot := true
+ for _, n := range cluster.ListNodes() {
+ snapshot, err := n.dataStore.loadSnapshotFromDisk()
+ require.NoError(t, err)
+ if snapshot.Metadata.Index <= 0 {
+ allNodesHasSnapshot = false
+ break
+ }
+ }
+ return allNodesHasSnapshot
+ }, 10*time.Second, 100*time.Millisecond)
+
+ require.NoError(t, cluster.Restart())
+ require.Eventually(t, func() bool {
+ return cluster.IsReady(ctx)
+ }, 10*time.Second, 100*time.Millisecond)
+
+ // Can restore data from snapshot correctly after restart
+ for i := 0; i < cnt; i++ {
+ gotBytes, _ := n.Get(ctx, fmt.Sprintf("foo%d", i))
+ require.Equal(t, "bar", string(gotBytes))
+ }
+}
diff --git a/store/engine/raft/store.go b/store/engine/raft/store.go
new file mode 100644
index 0000000..4834a31
--- /dev/null
+++ b/store/engine/raft/store.go
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package raft
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "strings"
+ "sync"
+
+ "github.com/apache/kvrocks-controller/logger"
+ "github.com/apache/kvrocks-controller/store/engine"
+
+ "go.etcd.io/etcd/pkg/fileutil"
+ "go.etcd.io/etcd/raft/v3"
+ "go.etcd.io/etcd/raft/v3/raftpb"
+ "go.etcd.io/etcd/server/v3/etcdserver/api/snap"
+ "go.etcd.io/etcd/server/v3/wal"
+ "go.etcd.io/etcd/server/v3/wal/walpb"
+)
+
+var ErrKeyNotFound = errors.New("key not found")
+
+type DataStore struct {
+ walDir string
+ snapshotDir string
+
+ snapshotter *snap.Snapshotter
+ wal *wal.WAL
+
+ raftStorage *raft.MemoryStorage
+
+ mu sync.RWMutex
+ kvs map[string][]byte
+}
+
+func NewDataStore(dir string) *DataStore {
+ snapshotDir := fmt.Sprintf("%s/snapshot", dir)
+ snapshotter := snap.New(logger.Get(), snapshotDir)
+ return &DataStore{
+ walDir: fmt.Sprintf("%s/wal", dir),
+ snapshotDir: snapshotDir,
+ snapshotter: snapshotter,
+ raftStorage: raft.NewMemoryStorage(),
+ kvs: make(map[string][]byte),
+ }
+}
+
+func (ds *DataStore) walExists() bool {
+ return wal.Exist(ds.walDir)
+}
+
+func (ds *DataStore) loadSnapshotFromDisk() (*raftpb.Snapshot, error) {
+ if !fileutil.Exist(ds.snapshotDir) {
+ if err := os.MkdirAll(ds.snapshotDir, 0750); err != nil {
+ return nil, err
+ }
+ }
+
+ emptySnapshot := &raftpb.Snapshot{}
+ if !ds.walExists() {
+ return emptySnapshot, nil
+ }
+
+ snapshots, err := wal.ValidSnapshotEntries(logger.Get(), ds.walDir)
+ if err != nil {
+ return nil, err
+ }
+ latestSnapshot, err := ds.snapshotter.LoadNewestAvailable(snapshots)
+ if err != nil {
+ if errors.Is(err, snap.ErrNoSnapshot) {
+ return emptySnapshot, nil
+ }
+ return nil, err
+ }
+ return latestSnapshot, nil
+}
+
+func (ds *DataStore) reloadSnapshot() error {
+ snapshot, err := ds.snapshotter.Load()
+ if errors.Is(err, snap.ErrNoSnapshot) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+
+ var m map[string][]byte
+ if err := json.Unmarshal(snapshot.Data, &m); err != nil {
+ return err
+ }
+
+ ds.mu.Lock()
+ ds.kvs = m
+ ds.mu.Unlock()
+ return nil
+}
+
+func (ds *DataStore) openWAL(snapshot *raftpb.Snapshot) (*wal.WAL, error) {
+ if !ds.walExists() {
+ if err := os.MkdirAll(ds.walDir, 0750); err != nil {
+ return nil, err
+ }
+ w, err := wal.Create(logger.Get(), ds.walDir, nil)
+ if err != nil {
+ return nil, err
+ }
+ w.Close()
+ }
+ walSnapshot := walpb.Snapshot{}
+ if snapshot != nil {
+ walSnapshot.Index = snapshot.Metadata.Index
+ walSnapshot.Term = snapshot.Metadata.Term
+ }
+ return wal.Open(logger.Get(), ds.walDir, walSnapshot)
+}
+
+func (ds *DataStore) replayWAL() error {
+ snapshot, err := ds.loadSnapshotFromDisk()
+ if err != nil {
+ return fmt.Errorf("failed to load newest snapshot: %w", err)
+ }
+
+ w, err := ds.openWAL(snapshot)
+ if err != nil {
+ return fmt.Errorf("failed to open WAL: %w", err)
+ }
+ ds.wal = w
+
+ _, hardState, entries, err := w.ReadAll()
+ if err != nil {
+ return fmt.Errorf("failed to read WAL: %w", err)
+ }
+ if snapshot != nil {
+ _ = ds.raftStorage.ApplySnapshot(*snapshot)
+ }
+ if err := ds.raftStorage.SetHardState(hardState); err != nil {
+ return fmt.Errorf("failed to set hard state: %w", err)
+ }
+ if err := ds.raftStorage.Append(entries); err != nil {
+ return fmt.Errorf("failed to append entries: %w", err)
+ }
+ return nil
+}
+
+func (ds *DataStore) saveSnapshot(snapshot raftpb.Snapshot) error {
+ walSnap := walpb.Snapshot{
+ Index: snapshot.Metadata.Index,
+ Term: snapshot.Metadata.Term,
+ ConfState: &snapshot.Metadata.ConfState,
+ }
+ if err := ds.snapshotter.SaveSnap(snapshot); err != nil {
+ return err
+ }
+ if err := ds.wal.SaveSnapshot(walSnap); err != nil {
+ return err
+ }
+ return ds.wal.ReleaseLockTo(snapshot.Metadata.Index)
+}
+
+func (ds *DataStore) Set(key string, value []byte) {
+ ds.mu.Lock()
+ defer ds.mu.Unlock()
+ ds.kvs[key] = value
+}
+
+func (ds *DataStore) Get(key string) ([]byte, error) {
+ ds.mu.RLock()
+ defer ds.mu.RUnlock()
+ if v, ok := ds.kvs[key]; ok {
+ return v, nil
+ }
+ return nil, ErrKeyNotFound
+}
+
+func (ds *DataStore) Delete(key string) {
+ ds.mu.Lock()
+ defer ds.mu.Unlock()
+ delete(ds.kvs, key)
+}
+
+func (ds *DataStore) List(prefix string) []engine.Entry {
+ ds.mu.RLock()
+ defer ds.mu.RUnlock()
+ entries := make([]engine.Entry, 0)
+ for k := range ds.kvs {
+ if strings.HasPrefix(k, prefix) {
+ entries = append(entries, engine.Entry{
+ Key: strings.TrimPrefix(k, prefix),
+ Value: ds.kvs[k],
+ })
+ }
+ }
+ return entries
+}
+
+func (ds *DataStore) GetDataStoreSnapshot() ([]byte, error) {
+ ds.mu.RLock()
+ defer ds.mu.RUnlock()
+ return json.Marshal(ds.kvs)
+}
+
+func (ds *DataStore) Close() {
+ if ds.wal != nil {
+ ds.wal.Close()
+ }
+}
diff --git a/store/engine/raft/store_test.go b/store/engine/raft/store_test.go
new file mode 100644
index 0000000..e16e2ae
--- /dev/null
+++ b/store/engine/raft/store_test.go
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package raft
+
+import (
+ "os"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+ "go.etcd.io/etcd/raft/v3/raftpb"
+)
+
+func TestDataStore(t *testing.T) {
+ dir := "/tmp/kvrocks/raft/test-datastore"
+ store := NewDataStore(dir)
+ require.NotNil(t, store)
+
+ defer func() {
+ store.Close()
+ os.RemoveAll(dir)
+ }()
+
+ err := store.replayWAL()
+ require.NoError(t, err)
+
+ t.Run("reply WAL from the disk", func(t *testing.T) {
+ require.NoError(t, store.wal.Save(raftpb.HardState{Term: 1,
Vote: 1}, []raftpb.Entry{
+ {Term: 1, Index: 1, Type: raftpb.EntryNormal, Data:
[]byte("test-1")},
+ {Term: 1, Index: 2, Type: raftpb.EntryNormal, Data:
[]byte("test-2")},
+ {Term: 1, Index: 3, Type: raftpb.EntryNormal, Data:
[]byte("test-3")},
+ }))
+ store.Close()
+
+ store = NewDataStore(dir)
+ require.NoError(t, store.replayWAL())
+
+ firstIndex, err := store.raftStorage.FirstIndex()
+ require.NoError(t, err)
+ require.EqualValues(t, 1, firstIndex)
+
+ lastIndex, err := store.raftStorage.LastIndex()
+ require.NoError(t, err)
+ require.EqualValues(t, 3, lastIndex)
+
+ term, err := store.raftStorage.Term(1)
+ require.NoError(t, err)
+ require.EqualValues(t, 1, term)
+ })
+
+ t.Run("Basic GET/SET/DELETE/LIST", func(t *testing.T) {
+ store.Set("bar-1", []byte("v1"))
+ store.Set("bar-2", []byte("v2"))
+ store.Set("baz-3", []byte("v3"))
+ store.Set("ba-4", []byte("v4"))
+ store.Set("foo", []byte("v5"))
+
+ v, err := store.Get("bar-2")
+ require.NoError(t, err)
+ require.Equal(t, []byte("v2"), v)
+
+ entries := store.List("bar")
+ require.Len(t, entries, 2)
+
+ entries = store.List("baz")
+ require.Len(t, entries, 1)
+
+ entries = store.List("ba")
+ require.Len(t, entries, 4)
+
+ entries = store.List("bar-2")
+ require.Len(t, entries, 1)
+
+ entries = store.List("foo")
+ require.Len(t, entries, 1)
+
+ store.Delete("bar-2")
+ _, err = store.Get("bar-2")
+ require.ErrorIs(t, err, ErrKeyNotFound)
+
+ entries = store.List("bar")
+ require.Len(t, entries, 1)
+ })
+}
diff --git a/store/store.go b/store/store.go
index 860df19..ce730b4 100644
--- a/store/store.go
+++ b/store/store.go
@@ -17,6 +17,7 @@
* under the License.
*
*/
+
package store
import (
