Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package etcd for openSUSE:Factory checked in at 2025-03-24 13:32:48 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/etcd (Old) and /work/SRC/openSUSE:Factory/.etcd.new.2696 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "etcd" Mon Mar 24 13:32:48 2025 rev:44 rq:1255598 version:3.5.20 Changes: -------- --- /work/SRC/openSUSE:Factory/etcd/etcd.changes 2025-03-10 18:06:16.300923081 +0100 +++ /work/SRC/openSUSE:Factory/.etcd.new.2696/etcd.changes 2025-03-24 13:32:56.113533440 +0100 @@ -1,0 +2,14 @@ +Mon Mar 24 09:42:26 UTC 2025 - Elisei Roca <er...@suse.com> + +- Update to version 3.5.20: + * Fix the issue that learner promotion command doesn't support + json output + * overwrite the member if already exist + * add verification to check whether membership data is in sync + between v2store and v3store + * fix: grpcproxy can get stuck in and endless loop causing high + cpu usage + * perf(release3.5): use RLock in Demoted method for read-only + access to expiry + +------------------------------------------------------------------- Old: ---- etcd-3.5.19.tar.gz New: ---- etcd-3.5.20.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ etcd.spec ++++++ --- /var/tmp/diff_new_pack.1FEjPe/_old 2025-03-24 13:32:57.701599604 +0100 +++ /var/tmp/diff_new_pack.1FEjPe/_new 2025-03-24 13:32:57.705599770 +0100 @@ -23,7 +23,7 @@ %define _fillupdir %{_localstatedir}/adm/fillup-templates %endif Name: etcd -Version: 3.5.19 +Version: 3.5.20 Release: 0 Summary: Highly-available key value store for configuration and service discovery License: Apache-2.0 ++++++ _service ++++++ --- /var/tmp/diff_new_pack.1FEjPe/_old 2025-03-24 13:32:57.789603270 +0100 +++ /var/tmp/diff_new_pack.1FEjPe/_new 2025-03-24 13:32:57.793603436 +0100 @@ -3,7 +3,7 @@ <param name="url">https://github.com/etcd-io/etcd.git</param> <param name="scm">git</param> <param name="exclude">.git</param> - <param name="revision">v3.5.19</param> + <param name="revision">v3.5.20</param> <param name="versionformat">@PARENT_TAG@</param> <param name="changesgenerate">enable</param> <param name="versionrewrite-pattern">v(.*)</param> ++++++ _servicedata ++++++ --- /var/tmp/diff_new_pack.1FEjPe/_old 2025-03-24 13:32:57.825604769 +0100 +++ /var/tmp/diff_new_pack.1FEjPe/_new 2025-03-24 13:32:57.829604937 +0100 @@ -5,6 +5,6 @@ <param name="url">git://github.com/etcd-io/etcd.git</param> <param name="changesrevision">99018a77bea9a9d29962e5169876c64e02739c52</param></service><service name="tar_scm"> <param name="url">https://github.com/etcd-io/etcd.git</param> - <param name="changesrevision">815eaba08570ab0a123d65c12ef419e5b3f8e250</param></service></servicedata> + <param name="changesrevision">ac31c34d0784b6a50a59bc125ccfcbf0ccbe5540</param></service></servicedata> (No newline at EOF) ++++++ etcd-3.5.19.tar.gz -> etcd-3.5.20.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/api/version/version.go new/etcd-3.5.20/api/version/version.go --- old/etcd-3.5.19/api/version/version.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/api/version/version.go 2025-03-21 21:04:31.000000000 +0100 @@ -26,7 +26,7 @@ var ( // MinClusterVersion is the min cluster version this etcd binary is compatible with. MinClusterVersion = "3.0.0" - Version = "3.5.19" + Version = "3.5.20" APIVersion = "unknown" // Git SHA Value will be set during build diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/client/v2/go.mod new/etcd-3.5.20/client/v2/go.mod --- old/etcd-3.5.19/client/v2/go.mod 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/client/v2/go.mod 2025-03-21 21:04:31.000000000 +0100 @@ -7,8 +7,8 @@ require ( github.com/json-iterator/go v1.1.11 github.com/modern-go/reflect2 v1.0.1 - go.etcd.io/etcd/api/v3 v3.5.19 - go.etcd.io/etcd/client/pkg/v3 v3.5.19 + go.etcd.io/etcd/api/v3 v3.5.20 + go.etcd.io/etcd/client/pkg/v3 v3.5.20 ) require ( diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/client/v3/go.mod new/etcd-3.5.20/client/v3/go.mod --- old/etcd-3.5.19/client/v3/go.mod 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/client/v3/go.mod 2025-03-21 21:04:31.000000000 +0100 @@ -8,8 +8,8 @@ github.com/dustin/go-humanize v1.0.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/prometheus/client_golang v1.11.1 - go.etcd.io/etcd/api/v3 v3.5.19 - go.etcd.io/etcd/client/pkg/v3 v3.5.19 + go.etcd.io/etcd/api/v3 v3.5.20 + go.etcd.io/etcd/client/pkg/v3 v3.5.20 go.uber.org/zap v1.17.0 google.golang.org/grpc v1.59.0 sigs.k8s.io/yaml v1.2.0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/etcdctl/ctlv3/command/printer.go new/etcd-3.5.20/etcdctl/ctlv3/command/printer.go --- old/etcd-3.5.19/etcdctl/ctlv3/command/printer.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/etcdctl/ctlv3/command/printer.go 2025-03-21 21:04:31.000000000 +0100 @@ -107,6 +107,9 @@ func (p *printerRPC) MemberRemove(id uint64, r v3.MemberRemoveResponse) { p.p((*pb.MemberRemoveResponse)(&r)) } +func (p *printerRPC) MemberPromote(id uint64, r v3.MemberPromoteResponse) { + p.p((*pb.MemberPromoteResponse)(&r)) +} func (p *printerRPC) MemberUpdate(id uint64, r v3.MemberUpdateResponse) { p.p((*pb.MemberUpdateResponse)(&r)) } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/etcdctl/go.mod new/etcd-3.5.20/etcdctl/go.mod --- old/etcd-3.5.19/etcdctl/go.mod 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/etcdctl/go.mod 2025-03-21 21:04:31.000000000 +0100 @@ -11,12 +11,12 @@ github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 github.com/urfave/cli v1.22.4 - go.etcd.io/etcd/api/v3 v3.5.19 - go.etcd.io/etcd/client/pkg/v3 v3.5.19 - go.etcd.io/etcd/client/v2 v2.305.19 - go.etcd.io/etcd/client/v3 v3.5.19 - go.etcd.io/etcd/etcdutl/v3 v3.5.19 - go.etcd.io/etcd/pkg/v3 v3.5.19 + go.etcd.io/etcd/api/v3 v3.5.20 + go.etcd.io/etcd/client/pkg/v3 v3.5.20 + go.etcd.io/etcd/client/v2 v2.305.20 + go.etcd.io/etcd/client/v3 v3.5.20 + go.etcd.io/etcd/etcdutl/v3 v3.5.20 + go.etcd.io/etcd/pkg/v3 v3.5.20 go.uber.org/zap v1.17.0 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba google.golang.org/grpc v1.59.0 @@ -50,8 +50,8 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.etcd.io/bbolt v1.3.11 // indirect - go.etcd.io/etcd/raft/v3 v3.5.19 // indirect - go.etcd.io/etcd/server/v3 v3.5.19 // indirect + go.etcd.io/etcd/raft/v3 v3.5.20 // indirect + go.etcd.io/etcd/server/v3 v3.5.20 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect go.opentelemetry.io/otel v1.20.0 // indirect go.opentelemetry.io/otel/metric v1.20.0 // indirect diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/etcdutl/go.mod new/etcd-3.5.20/etcdutl/go.mod --- old/etcd-3.5.19/etcdutl/go.mod 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/etcdutl/go.mod 2025-03-21 21:04:31.000000000 +0100 @@ -27,12 +27,12 @@ github.com/olekukonko/tablewriter v0.0.5 github.com/spf13/cobra v1.1.3 go.etcd.io/bbolt v1.3.11 - go.etcd.io/etcd/api/v3 v3.5.19 - go.etcd.io/etcd/client/pkg/v3 v3.5.19 - go.etcd.io/etcd/client/v3 v3.5.19 - go.etcd.io/etcd/pkg/v3 v3.5.19 - go.etcd.io/etcd/raft/v3 v3.5.19 - go.etcd.io/etcd/server/v3 v3.5.19 + go.etcd.io/etcd/api/v3 v3.5.20 + go.etcd.io/etcd/client/pkg/v3 v3.5.20 + go.etcd.io/etcd/client/v3 v3.5.20 + go.etcd.io/etcd/pkg/v3 v3.5.20 + go.etcd.io/etcd/raft/v3 v3.5.20 + go.etcd.io/etcd/server/v3 v3.5.20 go.uber.org/zap v1.17.0 ) @@ -60,7 +60,7 @@ github.com/prometheus/procfs v0.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect - go.etcd.io/etcd/client/v2 v2.305.19 // indirect + go.etcd.io/etcd/client/v2 v2.305.20 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect go.opentelemetry.io/otel v1.20.0 // indirect go.opentelemetry.io/otel/metric v1.20.0 // indirect diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/go.mod new/etcd-3.5.20/go.mod --- old/etcd-3.5.19/go.mod 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/go.mod 2025-03-21 21:04:31.000000000 +0100 @@ -22,16 +22,16 @@ github.com/dustin/go-humanize v1.0.0 github.com/spf13/cobra v1.1.3 go.etcd.io/bbolt v1.3.11 - go.etcd.io/etcd/api/v3 v3.5.19 - go.etcd.io/etcd/client/pkg/v3 v3.5.19 - go.etcd.io/etcd/client/v2 v2.305.19 - go.etcd.io/etcd/client/v3 v3.5.19 - go.etcd.io/etcd/etcdctl/v3 v3.5.19 - go.etcd.io/etcd/etcdutl/v3 v3.5.19 - go.etcd.io/etcd/pkg/v3 v3.5.19 - go.etcd.io/etcd/raft/v3 v3.5.19 - go.etcd.io/etcd/server/v3 v3.5.19 - go.etcd.io/etcd/tests/v3 v3.5.19 + go.etcd.io/etcd/api/v3 v3.5.20 + go.etcd.io/etcd/client/pkg/v3 v3.5.20 + go.etcd.io/etcd/client/v2 v2.305.20 + go.etcd.io/etcd/client/v3 v3.5.20 + go.etcd.io/etcd/etcdctl/v3 v3.5.20 + go.etcd.io/etcd/etcdutl/v3 v3.5.20 + go.etcd.io/etcd/pkg/v3 v3.5.20 + go.etcd.io/etcd/raft/v3 v3.5.20 + go.etcd.io/etcd/server/v3 v3.5.20 + go.etcd.io/etcd/tests/v3 v3.5.20 go.uber.org/zap v1.17.0 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba google.golang.org/grpc v1.59.0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/pkg/go.mod new/etcd-3.5.20/pkg/go.mod --- old/etcd-3.5.19/pkg/go.mod 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/pkg/go.mod 2025-03-21 21:04:31.000000000 +0100 @@ -10,7 +10,7 @@ github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 - go.etcd.io/etcd/client/pkg/v3 v3.5.19 + go.etcd.io/etcd/client/pkg/v3 v3.5.20 go.uber.org/zap v1.17.0 google.golang.org/grpc v1.59.0 ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/raft/go.mod new/etcd-3.5.20/raft/go.mod --- old/etcd-3.5.19/raft/go.mod 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/raft/go.mod 2025-03-21 21:04:31.000000000 +0100 @@ -8,7 +8,7 @@ github.com/cockroachdb/datadriven v1.0.2 github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.5.4 - go.etcd.io/etcd/client/pkg/v3 v3.5.19 + go.etcd.io/etcd/client/pkg/v3 v3.5.20 ) require ( diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/server/etcdserver/api/membership/cluster.go new/etcd-3.5.20/server/etcdserver/api/membership/cluster.go --- old/etcd-3.5.19/server/etcdserver/api/membership/cluster.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/server/etcdserver/api/membership/cluster.go 2025-03-21 21:04:31.000000000 +0100 @@ -696,6 +696,10 @@ return true } +func (c *RaftCluster) MembersFromStore() (map[types.ID]*Member, map[types.ID]bool) { + return membersFromStore(c.lg, c.v2store) +} + func membersFromStore(lg *zap.Logger, st v2store.Store) (map[types.ID]*Member, map[types.ID]bool) { members := make(map[types.ID]*Member) removed := make(map[types.ID]bool) @@ -732,6 +736,10 @@ return members, removed } +func (c *RaftCluster) MembersFromBackend() (map[types.ID]*Member, map[types.ID]bool) { + return membersFromBackend(c.lg, c.be) +} + func membersFromBackend(lg *zap.Logger, be backend.Backend) (map[types.ID]*Member, map[types.ID]bool) { return mustReadMembersFromBackend(lg, be) } @@ -903,6 +911,7 @@ c.Lock() defer c.Unlock() _, ok := c.members[id] + // gofail: var afterIsMemberExist struct{} return ok } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/server/etcdserver/api/membership/cluster_test.go new/etcd-3.5.20/server/etcdserver/api/membership/cluster_test.go --- old/etcd-3.5.19/server/etcdserver/api/membership/cluster_test.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/server/etcdserver/api/membership/cluster_test.go 2025-03-21 21:04:31.000000000 +0100 @@ -1049,10 +1049,9 @@ backendMembers: []*Member{alice}, }, { - name: "Adding member should fail if it exists in both", + name: "Adding member should success if it exists in both", storeV2Members: []*Member{alice}, backendMembers: []*Member{alice}, - expectPanics: true, }, { name: "Adding member should fail if it exists in storeV2 and backend is nil", diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/server/etcdserver/api/membership/store.go new/etcd-3.5.20/server/etcdserver/api/membership/store.go --- old/etcd-3.5.19/server/etcdserver/api/membership/store.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/server/etcdserver/api/membership/store.go 2025-03-21 21:04:31.000000000 +0100 @@ -54,9 +54,6 @@ tx := be.BatchTx() tx.LockInsideApply() defer tx.Unlock() - if unsafeMemberExists(tx, mkey) { - return errMemberAlreadyExist - } tx.UnsafePut(buckets.Members, mkey, mvalue) return nil } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/server/etcdserver/api/v3rpc/watch.go new/etcd-3.5.20/server/etcdserver/api/v3rpc/watch.go --- old/etcd-3.5.19/server/etcdserver/api/v3rpc/watch.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/server/etcdserver/api/v3rpc/watch.go 2025-03-21 21:04:31.000000000 +0100 @@ -453,6 +453,7 @@ sws.mu.RUnlock() var serr error + // gofail: var beforeSendWatchResponse struct{} if !fragmented && !ok { serr = sws.gRPCStream.Send(wr) } else { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/server/etcdserver/raft.go new/etcd-3.5.20/server/etcdserver/raft.go --- old/etcd-3.5.19/server/etcdserver/raft.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/server/etcdserver/raft.go 2025-03-21 21:04:31.000000000 +0100 @@ -334,6 +334,7 @@ notifyc <- struct{}{} } + // gofail: var raftBeforeAdvance struct{} r.Advance() case <-r.stopped: return diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/server/etcdserver/server.go new/etcd-3.5.20/server/etcdserver/server.go --- old/etcd-3.5.19/server/etcdserver/server.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/server/etcdserver/server.go 2025-03-21 21:04:31.000000000 +0100 @@ -24,6 +24,7 @@ "net/http" "os" "path" + "reflect" "regexp" "strconv" "strings" @@ -67,6 +68,7 @@ "go.etcd.io/etcd/server/v3/lease/leasehttp" "go.etcd.io/etcd/server/v3/mvcc" "go.etcd.io/etcd/server/v3/mvcc/backend" + "go.etcd.io/etcd/server/v3/verify" "go.etcd.io/etcd/server/v3/wal" ) @@ -1332,6 +1334,7 @@ // wait for raftNode to persist snapshot onto the disk <-apply.notifyc + // gofail: var applyBeforeOpenSnapshot struct{} newbe, err := openSnapshotBackend(s.Cfg, s.snapshotter, apply.snapshot, s.beHooks) if err != nil { lg.Panic("failed to open snapshot backend", zap.Error(err)) @@ -2242,7 +2245,7 @@ s.consistIndex.SetConsistentApplyingIndex(e.Index, e.Term) shouldApplyV3 = membership.ApplyBoth } - + // gofail: var beforeApplyOneConfChange struct{} var cc raftpb.ConfChange pbutil.MustUnmarshal(&cc, e.Data) removedSelf, err := s.applyConfChange(cc, confState, shouldApplyV3) @@ -2448,9 +2451,51 @@ s.r.transport.UpdatePeer(m.ID, m.PeerURLs) } } + + s.verifyV3StoreInSyncWithV2Store(shouldApplyV3) + return false, nil } +func (s *EtcdServer) verifyV3StoreInSyncWithV2Store(shouldApplyV3 membership.ShouldApplyV3) { + if !verify.VerifyEnabled() { + return + } + + // If shouldApplyV3 == false, then it means v2store hasn't caught up with v3store. + if !shouldApplyV3 { + return + } + + // clean up the Attributes, and we only care about the RaftAttributes + cleanAttributesFunc := func(members map[types.ID]*membership.Member) map[types.ID]*membership.Member { + processedMembers := make(map[types.ID]*membership.Member) + for id, m := range members { + clonedMember := m.Clone() + clonedMember.Attributes = membership.Attributes{} + processedMembers[id] = clonedMember + } + + return processedMembers + } + + v2Members, _ := s.cluster.MembersFromStore() + v3Members, _ := s.cluster.MembersFromBackend() + + processedV2Members := cleanAttributesFunc(v2Members) + processedV3Members := cleanAttributesFunc(v3Members) + + if match := reflect.DeepEqual(processedV2Members, processedV3Members); !match { + v2Data, v2Err := json.Marshal(processedV2Members) + v3Data, v3Err := json.Marshal(processedV3Members) + + if v2Err != nil || v3Err != nil { + panic("members in v2store doesn't match v3store") + } + panic(fmt.Sprintf("members in v2store doesn't match v3store, v2store: %s, v3store: %s", string(v2Data), string(v3Data))) + } +} + // TODO: non-blocking snapshot func (s *EtcdServer) snapshot(snapi uint64, confState raftpb.ConfState) { clone := s.v2store.Clone() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/server/etcdserver/server_test.go new/etcd-3.5.20/server/etcdserver/server_test.go --- old/etcd-3.5.19/server/etcdserver/server_test.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/server/etcdserver/server_test.go 2025-03-21 21:04:31.000000000 +0100 @@ -599,6 +599,11 @@ func TestApplyConfChangeShouldStop(t *testing.T) { cl := membership.NewCluster(zaptest.NewLogger(t)) cl.SetStore(v2store.New()) + + be, _ := betesting.NewDefaultTmpBackend(t) + defer betesting.Close(t, be) + cl.SetBackend(be) + for i := 1; i <= 3; i++ { cl.AddMember(&membership.Member{ID: types.ID(i)}, true) } @@ -647,12 +652,14 @@ cl := membership.NewCluster(zaptest.NewLogger(t)) cl.SetStore(v2store.New()) - cl.AddMember(&membership.Member{ID: types.ID(1)}, true) be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) + cl.SetBackend(be) cindex.CreateMetaBucket(be.BatchTx()) + cl.AddMember(&membership.Member{ID: types.ID(1)}, true) + ci := cindex.NewConsistentIndex(be) srv := &EtcdServer{ lgMu: new(sync.RWMutex), @@ -733,6 +740,11 @@ lg := zaptest.NewLogger(t) cl := membership.NewCluster(lg) cl.SetStore(v2store.New()) + + be, _ := betesting.NewDefaultTmpBackend(t) + defer betesting.Close(t, be) + cl.SetBackend(be) + for i := 1; i <= 5; i++ { cl.AddMember(&membership.Member{ID: types.ID(i)}, true) } @@ -1352,6 +1364,11 @@ cl := newTestCluster(t, nil) st := v2store.New() cl.SetStore(st) + + be, _ := betesting.NewDefaultTmpBackend(t) + defer betesting.Close(t, be) + cl.SetBackend(be) + r := newRaftNode(raftNodeConfig{ lg: lg, Node: n, @@ -1453,6 +1470,11 @@ cl := newTestCluster(t, nil) st := v2store.New() cl.SetStore(v2store.New()) + + be, _ := betesting.NewDefaultTmpBackend(t) + defer betesting.Close(t, be) + cl.SetBackend(be) + cl.AddMember(&membership.Member{ID: 1234}, true) r := newRaftNode(raftNodeConfig{ lg: lg, @@ -1499,6 +1521,11 @@ cl := newTestCluster(t, nil) st := v2store.New() cl.SetStore(st) + + be, _ := betesting.NewDefaultTmpBackend(t) + defer betesting.Close(t, be) + cl.SetBackend(be) + cl.AddMember(&membership.Member{ID: 1234}, true) r := newRaftNode(raftNodeConfig{ lg: lg, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/server/go.mod new/etcd-3.5.20/server/go.mod --- old/etcd-3.5.19/server/go.mod 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/server/go.mod 2025-03-21 21:04:31.000000000 +0100 @@ -26,12 +26,12 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go.etcd.io/bbolt v1.3.11 - go.etcd.io/etcd/api/v3 v3.5.19 - go.etcd.io/etcd/client/pkg/v3 v3.5.19 - go.etcd.io/etcd/client/v2 v2.305.19 - go.etcd.io/etcd/client/v3 v3.5.19 - go.etcd.io/etcd/pkg/v3 v3.5.19 - go.etcd.io/etcd/raft/v3 v3.5.19 + go.etcd.io/etcd/api/v3 v3.5.20 + go.etcd.io/etcd/client/pkg/v3 v3.5.20 + go.etcd.io/etcd/client/v2 v2.305.20 + go.etcd.io/etcd/client/v3 v3.5.20 + go.etcd.io/etcd/pkg/v3 v3.5.20 + go.etcd.io/etcd/raft/v3 v3.5.20 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 go.opentelemetry.io/otel v1.20.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/server/lease/lessor.go new/etcd-3.5.20/server/lease/lessor.go --- old/etcd-3.5.19/server/lease/lessor.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/server/lease/lessor.go 2025-03-21 21:04:31.000000000 +0100 @@ -26,11 +26,12 @@ "time" "github.com/coreos/go-semver/semver" + "go.uber.org/zap" + pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/server/v3/lease/leasepb" "go.etcd.io/etcd/server/v3/mvcc/backend" "go.etcd.io/etcd/server/v3/mvcc/buckets" - "go.uber.org/zap" ) // NoLease is a special LeaseID representing the absence of a lease. @@ -912,8 +913,8 @@ // Demoted returns true if the lease's expiry has been reset to forever. func (l *Lease) Demoted() bool { - l.expiryMu.Lock() - defer l.expiryMu.Unlock() + l.expiryMu.RLock() + defer l.expiryMu.RUnlock() return l.expiry == forever } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/server/mvcc/backend/backend.go new/etcd-3.5.20/server/mvcc/backend/backend.go --- old/etcd-3.5.19/server/mvcc/backend/backend.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/server/mvcc/backend/backend.go 2025-03-21 21:04:31.000000000 +0100 @@ -669,7 +669,9 @@ } func (b *backend) unsafeBegin(write bool) *bolt.Tx { + // gofail: var beforeStartDBTxn struct{} tx, err := b.db.Begin(write) + // gofail: var afterStartDBTxn struct{} if err != nil { b.lg.Fatal("failed to begin tx", zap.Error(err)) } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/server/mvcc/backend/batch_tx.go new/etcd-3.5.20/server/mvcc/backend/batch_tx.go --- old/etcd-3.5.19/server/mvcc/backend/batch_tx.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/server/mvcc/backend/batch_tx.go 2025-03-21 21:04:31.000000000 +0100 @@ -310,6 +310,7 @@ t.backend.readTx.Lock() // blocks txReadBuffer for writing. // gofail: var beforeWritebackBuf struct{} t.buf.writeback(&t.backend.readTx.buf) + // gofail: var afterWritebackBuf struct{} t.backend.readTx.Unlock() // We commit the transaction when the number of pending operations // reaches the configured limit(batchLimit) to prevent it from @@ -359,7 +360,9 @@ func (t *batchTxBuffered) unsafeCommit(stop bool) { if t.backend.hooks != nil { + // gofail: var commitBeforePreCommitHook struct{} t.backend.hooks.OnPreCommitUnsafe(t) + // gofail: var commitAfterPreCommitHook struct{} } if t.backend.readTx.tx != nil { // wait all store read transactions using the current boltdb tx to finish, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/server/mvcc/kvstore.go new/etcd-3.5.20/server/mvcc/kvstore.go --- old/etcd-3.5.19/server/mvcc/kvstore.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/server/mvcc/kvstore.go 2025-03-21 21:04:31.000000000 +0100 @@ -225,7 +225,9 @@ tx.UnsafePut(buckets.Meta, scheduledCompactKeyName, rbytes) tx.Unlock() // ensure that desired compaction is persisted + // gofail: var compactBeforeCommitScheduledCompact struct{} s.b.ForceCommit() + // gofail: var compactAfterCommitScheduledCompact struct{} s.revMu.Unlock() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/server/mvcc/kvstore_compaction.go new/etcd-3.5.20/server/mvcc/kvstore_compaction.go --- old/etcd-3.5.19/server/mvcc/kvstore_compaction.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/server/mvcc/kvstore_compaction.go 2025-03-21 21:04:31.000000000 +0100 @@ -65,6 +65,7 @@ revToBytes(revision{main: compactMainRev}, rbytes) tx.UnsafePut(buckets.Meta, finishedCompactKeyName, rbytes) tx.Unlock() + // gofail: var compactAfterSetFinishedCompact struct{} hash := h.Hash() size, sizeInUse := s.b.Size(), s.b.SizeInUse() s.lg.Info( @@ -84,7 +85,9 @@ revToBytes(revision{main: rev.main, sub: rev.sub + 1}, last) tx.Unlock() // Immediately commit the compaction deletes instead of letting them accumulate in the write buffer + // gofail: var compactBeforeCommitBatch struct{} s.b.ForceCommit() + // gofail: var compactAfterCommitBatch struct{} dbCompactionPauseMs.Observe(float64(time.Since(start) / time.Millisecond)) select { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/server/proxy/grpcproxy/cluster.go new/etcd-3.5.20/server/proxy/grpcproxy/cluster.go --- old/etcd-3.5.19/server/proxy/grpcproxy/cluster.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/server/proxy/grpcproxy/cluster.go 2025-03-21 21:04:31.000000000 +0100 @@ -107,7 +107,11 @@ case <-cp.ctx.Done(): cp.lg.Info("watching endpoints interrupted", zap.Error(cp.ctx.Err())) return - case updates := <-wa: + case updates, ok := <-wa: + if !ok { + cp.lg.Info("endpoints watch channel closed") + return + } cp.umu.Lock() for _, up := range updates { switch up.Op { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/server/verify/verify.go new/etcd-3.5.20/server/verify/verify.go --- old/etcd-3.5.19/server/verify/verify.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/server/verify/verify.go 2025-03-21 21:04:31.000000000 +0100 @@ -90,12 +90,37 @@ // VerifyIfEnabled performs verification according to ETCD_VERIFY env settings. // See Verify for more information. func VerifyIfEnabled(cfg Config) error { - if os.Getenv(ENV_VERIFY) == ENV_VERIFY_ALL_VALUE { + if VerifyEnabled() { return Verify(cfg) } return nil } +// VerifyEnabled returns `true` if verification is enabled. +func VerifyEnabled() bool { + return os.Getenv(ENV_VERIFY) == ENV_VERIFY_ALL_VALUE +} + +// EnableVerification enables the verification and returns a function that +// can be used to bring the original settings. +func EnableVerification() func() { + previousEnv := os.Getenv(ENV_VERIFY) + os.Setenv(ENV_VERIFY, ENV_VERIFY_ALL_VALUE) + return func() { + os.Setenv(ENV_VERIFY, previousEnv) + } +} + +// DisableVerification disables the verification and returns a function that +// can be used to bring the original settings. +func DisableVerification() func() { + previousEnv := os.Getenv(ENV_VERIFY) + os.Unsetenv(ENV_VERIFY) + return func() { + os.Setenv(ENV_VERIFY, previousEnv) + } +} + // MustVerifyIfEnabled performs verification according to ETCD_VERIFY env settings // and exits in case of found problems. // See Verify for more information. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/tests/e2e/ctl_v2_test.go new/etcd-3.5.20/tests/e2e/ctl_v2_test.go --- old/etcd-3.5.19/tests/e2e/ctl_v2_test.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/tests/e2e/ctl_v2_test.go 2025-03-21 21:04:31.000000000 +0100 @@ -22,6 +22,7 @@ "testing" "time" + "go.etcd.io/etcd/server/v3/verify" "go.etcd.io/etcd/tests/v3/framework/e2e" ) @@ -229,6 +230,10 @@ func testUtlCtlV2Backup(t *testing.T, snapCount int, v3 bool, utl bool) { BeforeTestV2(t) + // disable the verification because this case updated the db offline + revertFunc := verify.DisableVerification() + defer revertFunc() + backupDir, err := ioutil.TempDir(t.TempDir(), "testbackup0.etcd") if err != nil { t.Fatal(err) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/tests/e2e/ctl_v3_member_test.go new/etcd-3.5.20/tests/e2e/ctl_v3_member_test.go --- old/etcd-3.5.19/tests/e2e/ctl_v3_member_test.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/tests/e2e/ctl_v3_member_test.go 2025-03-21 21:04:31.000000000 +0100 @@ -22,7 +22,14 @@ "strings" "testing" + "github.com/stretchr/testify/require" + + "go.etcd.io/bbolt" "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/client/pkg/v3/types" + "go.etcd.io/etcd/server/v3/datadir" + "go.etcd.io/etcd/server/v3/etcdserver/api/membership" + "go.etcd.io/etcd/server/v3/mvcc/buckets" "go.etcd.io/etcd/tests/v3/framework/e2e" ) @@ -227,3 +234,186 @@ cmdArgs := append(cx.PrefixArgs(), "member", "update", memberID, fmt.Sprintf("--peer-urls=%s", peerURL)) return e2e.SpawnWithExpectWithEnv(cmdArgs, cx.envMap, " updated in cluster ") } + +// TestCtlV3PromotingLearner tests whether etcd can automatically fix the +// issue caused by https://github.com/etcd-io/etcd/issues/19557. +func TestCtlV3PromotingLearner(t *testing.T) { + testCases := []struct { + name string + snapshotCount int + writeToV3StoreSuccess bool + }{ + { + name: "create snapshot after learner promotion which is not saved to v3store", + snapshotCount: 10, + }, + { + name: "not create snapshot and learner promotion is not saved to v3store", + snapshotCount: 0, + }, + { + name: "not create snapshot and learner promotion is saved to v3store", + snapshotCount: 0, + writeToV3StoreSuccess: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Log("Create a single node etcd cluster") + cfg := e2e.NewConfigNoTLS() + cfg.BasePeerScheme = "unix" + cfg.ClusterSize = 1 + cfg.InitialCorruptCheck = true + if tc.snapshotCount != 0 { + cfg.SnapshotCount = tc.snapshotCount + } + + epc, err := e2e.NewEtcdProcessCluster(t, cfg) + require.NoError(t, err, "failed to start etcd cluster: %v", err) + defer func() { + derr := epc.Close() + require.NoError(t, derr, "failed to close etcd cluster: %v", derr) + }() + + t.Log("Add and start a learner") + learnerID, err := epc.StartNewProc(nil, true, t) + require.NoError(t, err) + + t.Log("Write a key to ensure the cluster is healthy so far") + etcdctl := epc.Procs[0].Etcdctl(e2e.ClientNonTLS, false, false) + err = etcdctl.Put("foo", "bar") + require.NoError(t, err) + + t.Logf("Promoting the learner %x", learnerID) + resp, err := etcdctl.MemberPromote(learnerID) + require.NoError(t, err) + + var promotedMember *etcdserverpb.Member + for _, m := range resp.Members { + if m.ID == learnerID { + promotedMember = m + break + } + } + require.NotNil(t, promotedMember) + t.Logf("The promoted member: %+v", promotedMember) + + t.Log("Ensure all members are voting members from user perspective") + ensureAllMembersAreVotingMembers(t, etcdctl) + + if tc.snapshotCount != 0 { + t.Logf("Write %d keys to trigger a snapshot", tc.snapshotCount) + for i := 0; i < tc.snapshotCount; i++ { + err = etcdctl.Put(fmt.Sprintf("key_%d", i), fmt.Sprintf("value_%d", i)) + require.NoError(t, err) + } + } + + if tc.writeToV3StoreSuccess { + t.Log("Skip manually changing the already promoted learner to a learner in v3store") + } else { + t.Logf("Stopping the already promoted member") + require.NoError(t, epc.Procs[1].Stop()) + + t.Log("Manually changing the already promoted member to a learner again in v3store") + promotedMember.IsLearner = true + mustSaveMemberIntoBbolt(t, epc.Procs[1].Config().DataDirPath, promotedMember) + + t.Log("Starting the member again") + require.NoError(t, epc.Procs[1].Start()) + } + + t.Log("Checking all members are ready to serve client requests") + for i := 0; i < len(epc.Procs); i++ { + e2e.AssertProcessLogs(t, epc.Procs[i], e2e.EtcdServerReadyLines[0]) + } + + // Wait for the learner published attribute to be applied by all members in the cluster + t.Log("Write a key to ensure the the learner published attribute has been applied by all members") + for i := 0; i < len(epc.Procs); i++ { + cli := epc.Procs[i].Etcdctl(e2e.ClientNonTLS, false, false) + err = cli.Put("foo", "bar") + require.NoError(t, err) + } + + t.Log("Ensure all members in v3store are voting members again") + for i := 0; i < len(epc.Procs); i++ { + t.Logf("Stopping the member: %d", i) + require.NoError(t, epc.Procs[i].Stop()) + + t.Logf("Checking all members in member's backend store: %d", i) + ensureAllMembersFromV3StoreAreVotingMembers(t, epc.Procs[i].Config().DataDirPath) + + t.Logf("Starting the member again: %d", i) + require.NoError(t, epc.Procs[i].Start()) + } + }) + } +} + +func mustSaveMemberIntoBbolt(t *testing.T, dataDir string, protoMember *etcdserverpb.Member) { + dbPath := datadir.ToBackendFileName(dataDir) + db, err := bbolt.Open(dbPath, 0666, nil) + require.NoError(t, err) + defer func() { + require.NoError(t, db.Close()) + }() + + m := &membership.Member{ + ID: types.ID(protoMember.ID), + RaftAttributes: membership.RaftAttributes{ + PeerURLs: protoMember.PeerURLs, + IsLearner: protoMember.IsLearner, + }, + Attributes: membership.Attributes{ + Name: protoMember.Name, + ClientURLs: protoMember.ClientURLs, + }, + } + + err = db.Update(func(tx *bbolt.Tx) error { + b := tx.Bucket(buckets.Members.Name()) + + mkey := []byte(m.ID.String()) + mvalue, err := json.Marshal(m) + require.NoError(t, err) + + return b.Put(mkey, mvalue) + }) + require.NoError(t, err) +} + +func ensureAllMembersAreVotingMembers(t *testing.T, etcdctl *e2e.Etcdctl) { + memberListResp, err := etcdctl.MemberList() + require.NoError(t, err) + for _, m := range memberListResp.Members { + require.False(t, m.IsLearner) + } +} + +func ensureAllMembersFromV3StoreAreVotingMembers(t *testing.T, dataDir string) { + dbPath := datadir.ToBackendFileName(dataDir) + db, err := bbolt.Open(dbPath, 0400, &bbolt.Options{ReadOnly: true}) + require.NoError(t, err) + defer func() { + require.NoError(t, db.Close()) + }() + + var members []membership.Member + _ = db.View(func(tx *bbolt.Tx) error { + b := tx.Bucket(buckets.Members.Name()) + _ = b.ForEach(func(k, v []byte) error { + m := membership.Member{} + err := json.Unmarshal(v, &m) + require.NoError(t, err) + members = append(members, m) + return nil + }) + return nil + }) + + for _, m := range members { + require.Falsef(t, m.IsLearner, "member is still learner: %+v", m) + } +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/tests/e2e/etcd_mix_versions_test.go new/etcd-3.5.20/tests/e2e/etcd_mix_versions_test.go --- old/etcd-3.5.19/tests/e2e/etcd_mix_versions_test.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/tests/e2e/etcd_mix_versions_test.go 2025-03-21 21:04:31.000000000 +0100 @@ -101,7 +101,7 @@ newCfg := *epc.Cfg newCfg.Version = newInstanceVersion t.Log("Starting a new etcd instance") - _, err = epc.StartNewProc(&newCfg, t) + _, err = epc.StartNewProc(&newCfg, false, t) require.NoError(t, err, "failed to start the new etcd instance: %v", err) defer epc.Close() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/tests/framework/e2e/cluster.go new/etcd-3.5.20/tests/framework/e2e/cluster.go --- old/etcd-3.5.19/tests/framework/e2e/cluster.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/tests/framework/e2e/cluster.go 2025-03-21 21:04:31.000000000 +0100 @@ -24,6 +24,7 @@ "testing" "time" + clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/proxy" "go.etcd.io/etcd/server/v3/etcdserver" "go.uber.org/zap" @@ -628,8 +629,8 @@ // StartNewProc grows cluster size by one with two phases // Phase 1 - Inform cluster of new configuration // Phase 2 - Start new member -func (epc *EtcdProcessCluster) StartNewProc(cfg *EtcdProcessClusterConfig, tb testing.TB) (memberID uint64, err error) { - memberID, serverCfg, err := epc.AddMember(cfg, tb) +func (epc *EtcdProcessCluster) StartNewProc(cfg *EtcdProcessClusterConfig, isLearner bool, tb testing.TB) (memberID uint64, err error) { + memberID, serverCfg, err := epc.AddMember(cfg, isLearner, tb) if err != nil { return 0, err } @@ -643,12 +644,11 @@ } // AddMember adds a new member to the cluster without starting it. -func (epc *EtcdProcessCluster) AddMember(cfg *EtcdProcessClusterConfig, tb testing.TB) (memberID uint64, serverCfg *EtcdServerProcessConfig, err error) { - if cfg != nil { - serverCfg = cfg.EtcdServerProcessConfig(tb, epc.nextSeq) - } else { - serverCfg = epc.Cfg.EtcdServerProcessConfig(tb, epc.nextSeq) +func (epc *EtcdProcessCluster) AddMember(cfg *EtcdProcessClusterConfig, isLearner bool, tb testing.TB) (memberID uint64, serverCfg *EtcdServerProcessConfig, err error) { + if cfg == nil { + cfg = epc.Cfg } + serverCfg = cfg.EtcdServerProcessConfig(tb, epc.nextSeq) epc.nextSeq++ @@ -664,9 +664,17 @@ // First add new member to cluster tb.Logf("add new member to cluster; member-name %s, member-peer-url %s", serverCfg.Name, serverCfg.Purl.String()) memberCtl := NewEtcdctl(epc.Procs[0].EndpointsV3(), cfg.ClientTLS, cfg.IsClientAutoTLS, false) - resp, err := memberCtl.MemberAdd(serverCfg.Name, []string{serverCfg.Purl.String()}) - if err != nil { - return 0, nil, fmt.Errorf("failed to add new member: %w", err) + var ( + resp *clientv3.MemberAddResponse + mErr error + ) + if isLearner { + resp, mErr = memberCtl.MemberAddAsLearner(serverCfg.Name, []string{serverCfg.Purl.String()}) + } else { + resp, mErr = memberCtl.MemberAdd(serverCfg.Name, []string{serverCfg.Purl.String()}) + } + if mErr != nil { + return 0, nil, fmt.Errorf("failed to add new member: %w", mErr) } return resp.Member.ID, serverCfg, nil diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/tests/framework/e2e/etcdctl.go new/etcd-3.5.20/tests/framework/e2e/etcdctl.go --- old/etcd-3.5.19/tests/framework/e2e/etcdctl.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/tests/framework/e2e/etcdctl.go 2025-03-21 21:04:31.000000000 +0100 @@ -174,6 +174,15 @@ return &resp, err } +func (ctl *Etcdctl) MemberPromote(id uint64) (*clientv3.MemberPromoteResponse, error) { + if ctl.v2 { + panic("Unsupported method for v2") + } + var resp clientv3.MemberPromoteResponse + err := ctl.spawnJsonCmd(&resp, "member", "promote", fmt.Sprintf("%x", id)) + return &resp, err +} + func (ctl *Etcdctl) Compact(rev int64) (*clientv3.CompactResponse, error) { if ctl.v2 { panic("Unsupported method for v2") diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/tests/go.mod new/etcd-3.5.20/tests/go.mod --- old/etcd-3.5.19/tests/go.mod 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/tests/go.mod 2025-03-21 21:04:31.000000000 +0100 @@ -30,14 +30,15 @@ github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 - go.etcd.io/etcd/api/v3 v3.5.19 - go.etcd.io/etcd/client/pkg/v3 v3.5.19 - go.etcd.io/etcd/client/v2 v2.305.19 - go.etcd.io/etcd/client/v3 v3.5.19 - go.etcd.io/etcd/etcdutl/v3 v3.5.19 - go.etcd.io/etcd/pkg/v3 v3.5.19 - go.etcd.io/etcd/raft/v3 v3.5.19 - go.etcd.io/etcd/server/v3 v3.5.19 + go.etcd.io/bbolt v1.3.11 + go.etcd.io/etcd/api/v3 v3.5.20 + go.etcd.io/etcd/client/pkg/v3 v3.5.20 + go.etcd.io/etcd/client/v2 v2.305.20 + go.etcd.io/etcd/client/v3 v3.5.20 + go.etcd.io/etcd/etcdutl/v3 v3.5.20 + go.etcd.io/etcd/pkg/v3 v3.5.20 + go.etcd.io/etcd/raft/v3 v3.5.20 + go.etcd.io/etcd/server/v3 v3.5.20 go.etcd.io/gofail v0.2.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 go.opentelemetry.io/otel v1.20.0 @@ -78,7 +79,6 @@ github.com/sirupsen/logrus v1.9.3 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect - go.etcd.io/bbolt v1.3.11 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect go.opentelemetry.io/otel/metric v1.20.0 // indirect diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/etcd-3.5.19/tests/integration/hashkv_test.go new/etcd-3.5.20/tests/integration/hashkv_test.go --- old/etcd-3.5.19/tests/integration/hashkv_test.go 2025-03-05 20:33:28.000000000 +0100 +++ new/etcd-3.5.20/tests/integration/hashkv_test.go 2025-03-21 21:04:31.000000000 +0100 @@ -19,7 +19,6 @@ "net" "net/http" "testing" - "time" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver" @@ -83,8 +82,6 @@ } func (tc hashTestCase) Compact(ctx context.Context, rev int64) error { - _, err := tc.Client.Compact(ctx, rev) - // Wait for compaction to be compacted - time.Sleep(50 * time.Millisecond) + _, err := tc.Client.Compact(ctx, rev, clientv3.WithCompactPhysical()) return err } ++++++ vendor.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdctl/vendor/go.etcd.io/etcd/api/v3/version/version.go new/vendor/etcdctl/vendor/go.etcd.io/etcd/api/v3/version/version.go --- old/vendor/etcdctl/vendor/go.etcd.io/etcd/api/v3/version/version.go 2025-03-06 14:17:07.377058826 +0100 +++ new/vendor/etcdctl/vendor/go.etcd.io/etcd/api/v3/version/version.go 2025-03-24 10:40:27.729342397 +0100 @@ -26,7 +26,7 @@ var ( // MinClusterVersion is the min cluster version this etcd binary is compatible with. MinClusterVersion = "3.0.0" - Version = "3.5.19" + Version = "3.5.20" APIVersion = "unknown" // Git SHA Value will be set during build diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/etcdserver/api/membership/cluster.go new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/etcdserver/api/membership/cluster.go --- old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/etcdserver/api/membership/cluster.go 2025-03-06 14:17:07.570319539 +0100 +++ new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/etcdserver/api/membership/cluster.go 2025-03-24 10:40:28.363940485 +0100 @@ -696,6 +696,10 @@ return true } +func (c *RaftCluster) MembersFromStore() (map[types.ID]*Member, map[types.ID]bool) { + return membersFromStore(c.lg, c.v2store) +} + func membersFromStore(lg *zap.Logger, st v2store.Store) (map[types.ID]*Member, map[types.ID]bool) { members := make(map[types.ID]*Member) removed := make(map[types.ID]bool) @@ -732,6 +736,10 @@ return members, removed } +func (c *RaftCluster) MembersFromBackend() (map[types.ID]*Member, map[types.ID]bool) { + return membersFromBackend(c.lg, c.be) +} + func membersFromBackend(lg *zap.Logger, be backend.Backend) (map[types.ID]*Member, map[types.ID]bool) { return mustReadMembersFromBackend(lg, be) } @@ -903,6 +911,7 @@ c.Lock() defer c.Unlock() _, ok := c.members[id] + // gofail: var afterIsMemberExist struct{} return ok } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/etcdserver/api/membership/store.go new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/etcdserver/api/membership/store.go --- old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/etcdserver/api/membership/store.go 2025-03-06 14:17:07.570319539 +0100 +++ new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/etcdserver/api/membership/store.go 2025-03-24 10:40:28.363940485 +0100 @@ -54,9 +54,6 @@ tx := be.BatchTx() tx.LockInsideApply() defer tx.Unlock() - if unsafeMemberExists(tx, mkey) { - return errMemberAlreadyExist - } tx.UnsafePut(buckets.Members, mkey, mvalue) return nil } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/etcdserver/raft.go new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/etcdserver/raft.go --- old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/etcdserver/raft.go 2025-03-06 14:17:07.568986030 +0100 +++ new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/etcdserver/raft.go 2025-03-24 10:40:28.362093130 +0100 @@ -334,6 +334,7 @@ notifyc <- struct{}{} } + // gofail: var raftBeforeAdvance struct{} r.Advance() case <-r.stopped: return diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/etcdserver/server.go new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/etcdserver/server.go --- old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/etcdserver/server.go 2025-03-06 14:17:07.568986030 +0100 +++ new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/etcdserver/server.go 2025-03-24 10:40:28.362093130 +0100 @@ -24,6 +24,7 @@ "net/http" "os" "path" + "reflect" "regexp" "strconv" "strings" @@ -67,6 +68,7 @@ "go.etcd.io/etcd/server/v3/lease/leasehttp" "go.etcd.io/etcd/server/v3/mvcc" "go.etcd.io/etcd/server/v3/mvcc/backend" + "go.etcd.io/etcd/server/v3/verify" "go.etcd.io/etcd/server/v3/wal" ) @@ -1332,6 +1334,7 @@ // wait for raftNode to persist snapshot onto the disk <-apply.notifyc + // gofail: var applyBeforeOpenSnapshot struct{} newbe, err := openSnapshotBackend(s.Cfg, s.snapshotter, apply.snapshot, s.beHooks) if err != nil { lg.Panic("failed to open snapshot backend", zap.Error(err)) @@ -2242,7 +2245,7 @@ s.consistIndex.SetConsistentApplyingIndex(e.Index, e.Term) shouldApplyV3 = membership.ApplyBoth } - + // gofail: var beforeApplyOneConfChange struct{} var cc raftpb.ConfChange pbutil.MustUnmarshal(&cc, e.Data) removedSelf, err := s.applyConfChange(cc, confState, shouldApplyV3) @@ -2448,9 +2451,51 @@ s.r.transport.UpdatePeer(m.ID, m.PeerURLs) } } + + s.verifyV3StoreInSyncWithV2Store(shouldApplyV3) + return false, nil } +func (s *EtcdServer) verifyV3StoreInSyncWithV2Store(shouldApplyV3 membership.ShouldApplyV3) { + if !verify.VerifyEnabled() { + return + } + + // If shouldApplyV3 == false, then it means v2store hasn't caught up with v3store. + if !shouldApplyV3 { + return + } + + // clean up the Attributes, and we only care about the RaftAttributes + cleanAttributesFunc := func(members map[types.ID]*membership.Member) map[types.ID]*membership.Member { + processedMembers := make(map[types.ID]*membership.Member) + for id, m := range members { + clonedMember := m.Clone() + clonedMember.Attributes = membership.Attributes{} + processedMembers[id] = clonedMember + } + + return processedMembers + } + + v2Members, _ := s.cluster.MembersFromStore() + v3Members, _ := s.cluster.MembersFromBackend() + + processedV2Members := cleanAttributesFunc(v2Members) + processedV3Members := cleanAttributesFunc(v3Members) + + if match := reflect.DeepEqual(processedV2Members, processedV3Members); !match { + v2Data, v2Err := json.Marshal(processedV2Members) + v3Data, v3Err := json.Marshal(processedV3Members) + + if v2Err != nil || v3Err != nil { + panic("members in v2store doesn't match v3store") + } + panic(fmt.Sprintf("members in v2store doesn't match v3store, v2store: %s, v3store: %s", string(v2Data), string(v3Data))) + } +} + // TODO: non-blocking snapshot func (s *EtcdServer) snapshot(snapi uint64, confState raftpb.ConfState) { clone := s.v2store.Clone() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/lease/lessor.go new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/lease/lessor.go --- old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/lease/lessor.go 2025-03-06 14:17:07.574206969 +0100 +++ new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/lease/lessor.go 2025-03-24 10:40:28.369940485 +0100 @@ -26,11 +26,12 @@ "time" "github.com/coreos/go-semver/semver" + "go.uber.org/zap" + pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/server/v3/lease/leasepb" "go.etcd.io/etcd/server/v3/mvcc/backend" "go.etcd.io/etcd/server/v3/mvcc/buckets" - "go.uber.org/zap" ) // NoLease is a special LeaseID representing the absence of a lease. @@ -912,8 +913,8 @@ // Demoted returns true if the lease's expiry has been reset to forever. func (l *Lease) Demoted() bool { - l.expiryMu.Lock() - defer l.expiryMu.Unlock() + l.expiryMu.RLock() + defer l.expiryMu.RUnlock() return l.expiry == forever } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/mvcc/backend/backend.go new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/mvcc/backend/backend.go --- old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/mvcc/backend/backend.go 2025-03-06 14:17:07.576698211 +0100 +++ new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/mvcc/backend/backend.go 2025-03-24 10:40:28.371940485 +0100 @@ -669,7 +669,9 @@ } func (b *backend) unsafeBegin(write bool) *bolt.Tx { + // gofail: var beforeStartDBTxn struct{} tx, err := b.db.Begin(write) + // gofail: var afterStartDBTxn struct{} if err != nil { b.lg.Fatal("failed to begin tx", zap.Error(err)) } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/mvcc/backend/batch_tx.go new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/mvcc/backend/batch_tx.go --- old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/mvcc/backend/batch_tx.go 2025-03-06 14:17:07.576698211 +0100 +++ new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/mvcc/backend/batch_tx.go 2025-03-24 10:40:28.371940485 +0100 @@ -310,6 +310,7 @@ t.backend.readTx.Lock() // blocks txReadBuffer for writing. // gofail: var beforeWritebackBuf struct{} t.buf.writeback(&t.backend.readTx.buf) + // gofail: var afterWritebackBuf struct{} t.backend.readTx.Unlock() // We commit the transaction when the number of pending operations // reaches the configured limit(batchLimit) to prevent it from @@ -359,7 +360,9 @@ func (t *batchTxBuffered) unsafeCommit(stop bool) { if t.backend.hooks != nil { + // gofail: var commitBeforePreCommitHook struct{} t.backend.hooks.OnPreCommitUnsafe(t) + // gofail: var commitAfterPreCommitHook struct{} } if t.backend.readTx.tx != nil { // wait all store read transactions using the current boltdb tx to finish, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/mvcc/kvstore.go new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/mvcc/kvstore.go --- old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/mvcc/kvstore.go 2025-03-06 14:17:07.575040469 +0100 +++ new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/mvcc/kvstore.go 2025-03-24 10:40:28.371103244 +0100 @@ -225,7 +225,9 @@ tx.UnsafePut(buckets.Meta, scheduledCompactKeyName, rbytes) tx.Unlock() // ensure that desired compaction is persisted + // gofail: var compactBeforeCommitScheduledCompact struct{} s.b.ForceCommit() + // gofail: var compactAfterCommitScheduledCompact struct{} s.revMu.Unlock() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/mvcc/kvstore_compaction.go new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/mvcc/kvstore_compaction.go --- old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/mvcc/kvstore_compaction.go 2025-03-06 14:17:07.575040469 +0100 +++ new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/mvcc/kvstore_compaction.go 2025-03-24 10:40:28.371103244 +0100 @@ -65,6 +65,7 @@ revToBytes(revision{main: compactMainRev}, rbytes) tx.UnsafePut(buckets.Meta, finishedCompactKeyName, rbytes) tx.Unlock() + // gofail: var compactAfterSetFinishedCompact struct{} hash := h.Hash() size, sizeInUse := s.b.Size(), s.b.SizeInUse() s.lg.Info( @@ -84,7 +85,9 @@ revToBytes(revision{main: rev.main, sub: rev.sub + 1}, last) tx.Unlock() // Immediately commit the compaction deletes instead of letting them accumulate in the write buffer + // gofail: var compactBeforeCommitBatch struct{} s.b.ForceCommit() + // gofail: var compactAfterCommitBatch struct{} dbCompactionPauseMs.Observe(float64(time.Since(start) / time.Millisecond)) select { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/verify/verify.go new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/verify/verify.go --- old/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/verify/verify.go 2025-03-06 14:17:07.577698232 +0100 +++ new/vendor/etcdctl/vendor/go.etcd.io/etcd/server/v3/verify/verify.go 2025-03-24 10:40:28.373940485 +0100 @@ -90,12 +90,37 @@ // VerifyIfEnabled performs verification according to ETCD_VERIFY env settings. // See Verify for more information. func VerifyIfEnabled(cfg Config) error { - if os.Getenv(ENV_VERIFY) == ENV_VERIFY_ALL_VALUE { + if VerifyEnabled() { return Verify(cfg) } return nil } +// VerifyEnabled returns `true` if verification is enabled. +func VerifyEnabled() bool { + return os.Getenv(ENV_VERIFY) == ENV_VERIFY_ALL_VALUE +} + +// EnableVerification enables the verification and returns a function that +// can be used to bring the original settings. +func EnableVerification() func() { + previousEnv := os.Getenv(ENV_VERIFY) + os.Setenv(ENV_VERIFY, ENV_VERIFY_ALL_VALUE) + return func() { + os.Setenv(ENV_VERIFY, previousEnv) + } +} + +// DisableVerification disables the verification and returns a function that +// can be used to bring the original settings. +func DisableVerification() func() { + previousEnv := os.Getenv(ENV_VERIFY) + os.Unsetenv(ENV_VERIFY) + return func() { + os.Setenv(ENV_VERIFY, previousEnv) + } +} + // MustVerifyIfEnabled performs verification according to ETCD_VERIFY env settings // and exits in case of found problems. // See Verify for more information. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdctl/vendor/modules.txt new/vendor/etcdctl/vendor/modules.txt --- old/vendor/etcdctl/vendor/modules.txt 2025-03-06 14:17:07.654699841 +0100 +++ new/vendor/etcdctl/vendor/modules.txt 2025-03-24 10:40:28.470940484 +0100 @@ -107,7 +107,7 @@ # go.etcd.io/bbolt v1.3.11 ## explicit; go 1.22 go.etcd.io/bbolt -# go.etcd.io/etcd/api/v3 v3.5.19 => ../api +# go.etcd.io/etcd/api/v3 v3.5.20 => ../api ## explicit; go 1.23.0 go.etcd.io/etcd/api/v3/authpb go.etcd.io/etcd/api/v3/etcdserverpb @@ -115,7 +115,7 @@ go.etcd.io/etcd/api/v3/mvccpb go.etcd.io/etcd/api/v3/v3rpc/rpctypes go.etcd.io/etcd/api/v3/version -# go.etcd.io/etcd/client/pkg/v3 v3.5.19 => ../client/pkg +# go.etcd.io/etcd/client/pkg/v3 v3.5.20 => ../client/pkg ## explicit; go 1.23.0 go.etcd.io/etcd/client/pkg/v3/fileutil go.etcd.io/etcd/client/pkg/v3/logutil @@ -125,10 +125,10 @@ go.etcd.io/etcd/client/pkg/v3/tlsutil go.etcd.io/etcd/client/pkg/v3/transport go.etcd.io/etcd/client/pkg/v3/types -# go.etcd.io/etcd/client/v2 v2.305.19 => ../client/v2 +# go.etcd.io/etcd/client/v2 v2.305.20 => ../client/v2 ## explicit; go 1.23.0 go.etcd.io/etcd/client/v2 -# go.etcd.io/etcd/client/v3 v3.5.19 => ../client/v3 +# go.etcd.io/etcd/client/v3 v3.5.20 => ../client/v3 ## explicit; go 1.23.0 go.etcd.io/etcd/client/v3 go.etcd.io/etcd/client/v3/concurrency @@ -137,11 +137,11 @@ go.etcd.io/etcd/client/v3/internal/resolver go.etcd.io/etcd/client/v3/mirror go.etcd.io/etcd/client/v3/snapshot -# go.etcd.io/etcd/etcdutl/v3 v3.5.19 => ../etcdutl +# go.etcd.io/etcd/etcdutl/v3 v3.5.20 => ../etcdutl ## explicit; go 1.23.0 go.etcd.io/etcd/etcdutl/v3/etcdutl go.etcd.io/etcd/etcdutl/v3/snapshot -# go.etcd.io/etcd/pkg/v3 v3.5.19 => ../pkg +# go.etcd.io/etcd/pkg/v3 v3.5.20 => ../pkg ## explicit; go 1.23.0 go.etcd.io/etcd/pkg/v3/adt go.etcd.io/etcd/pkg/v3/cobrautl @@ -159,14 +159,14 @@ go.etcd.io/etcd/pkg/v3/schedule go.etcd.io/etcd/pkg/v3/traceutil go.etcd.io/etcd/pkg/v3/wait -# go.etcd.io/etcd/raft/v3 v3.5.19 => ../raft +# go.etcd.io/etcd/raft/v3 v3.5.20 => ../raft ## explicit; go 1.23.0 go.etcd.io/etcd/raft/v3 go.etcd.io/etcd/raft/v3/confchange go.etcd.io/etcd/raft/v3/quorum go.etcd.io/etcd/raft/v3/raftpb go.etcd.io/etcd/raft/v3/tracker -# go.etcd.io/etcd/server/v3 v3.5.19 => ../server +# go.etcd.io/etcd/server/v3 v3.5.20 => ../server ## explicit; go 1.23.0 go.etcd.io/etcd/server/v3/auth go.etcd.io/etcd/server/v3/config diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdutl/vendor/go.etcd.io/etcd/api/v3/version/version.go new/vendor/etcdutl/vendor/go.etcd.io/etcd/api/v3/version/version.go --- old/vendor/etcdutl/vendor/go.etcd.io/etcd/api/v3/version/version.go 2025-03-06 14:17:07.377058826 +0100 +++ new/vendor/etcdutl/vendor/go.etcd.io/etcd/api/v3/version/version.go 2025-03-24 10:40:27.729342397 +0100 @@ -26,7 +26,7 @@ var ( // MinClusterVersion is the min cluster version this etcd binary is compatible with. MinClusterVersion = "3.0.0" - Version = "3.5.19" + Version = "3.5.20" APIVersion = "unknown" // Git SHA Value will be set during build diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/etcdserver/api/membership/cluster.go new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/etcdserver/api/membership/cluster.go --- old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/etcdserver/api/membership/cluster.go 2025-03-06 14:17:07.570319539 +0100 +++ new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/etcdserver/api/membership/cluster.go 2025-03-24 10:40:28.363940485 +0100 @@ -696,6 +696,10 @@ return true } +func (c *RaftCluster) MembersFromStore() (map[types.ID]*Member, map[types.ID]bool) { + return membersFromStore(c.lg, c.v2store) +} + func membersFromStore(lg *zap.Logger, st v2store.Store) (map[types.ID]*Member, map[types.ID]bool) { members := make(map[types.ID]*Member) removed := make(map[types.ID]bool) @@ -732,6 +736,10 @@ return members, removed } +func (c *RaftCluster) MembersFromBackend() (map[types.ID]*Member, map[types.ID]bool) { + return membersFromBackend(c.lg, c.be) +} + func membersFromBackend(lg *zap.Logger, be backend.Backend) (map[types.ID]*Member, map[types.ID]bool) { return mustReadMembersFromBackend(lg, be) } @@ -903,6 +911,7 @@ c.Lock() defer c.Unlock() _, ok := c.members[id] + // gofail: var afterIsMemberExist struct{} return ok } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/etcdserver/api/membership/store.go new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/etcdserver/api/membership/store.go --- old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/etcdserver/api/membership/store.go 2025-03-06 14:17:07.570319539 +0100 +++ new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/etcdserver/api/membership/store.go 2025-03-24 10:40:28.363940485 +0100 @@ -54,9 +54,6 @@ tx := be.BatchTx() tx.LockInsideApply() defer tx.Unlock() - if unsafeMemberExists(tx, mkey) { - return errMemberAlreadyExist - } tx.UnsafePut(buckets.Members, mkey, mvalue) return nil } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/etcdserver/raft.go new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/etcdserver/raft.go --- old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/etcdserver/raft.go 2025-03-06 14:17:07.568986030 +0100 +++ new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/etcdserver/raft.go 2025-03-24 10:40:28.362093130 +0100 @@ -334,6 +334,7 @@ notifyc <- struct{}{} } + // gofail: var raftBeforeAdvance struct{} r.Advance() case <-r.stopped: return diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/etcdserver/server.go new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/etcdserver/server.go --- old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/etcdserver/server.go 2025-03-06 14:17:07.568986030 +0100 +++ new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/etcdserver/server.go 2025-03-24 10:40:28.362093130 +0100 @@ -24,6 +24,7 @@ "net/http" "os" "path" + "reflect" "regexp" "strconv" "strings" @@ -67,6 +68,7 @@ "go.etcd.io/etcd/server/v3/lease/leasehttp" "go.etcd.io/etcd/server/v3/mvcc" "go.etcd.io/etcd/server/v3/mvcc/backend" + "go.etcd.io/etcd/server/v3/verify" "go.etcd.io/etcd/server/v3/wal" ) @@ -1332,6 +1334,7 @@ // wait for raftNode to persist snapshot onto the disk <-apply.notifyc + // gofail: var applyBeforeOpenSnapshot struct{} newbe, err := openSnapshotBackend(s.Cfg, s.snapshotter, apply.snapshot, s.beHooks) if err != nil { lg.Panic("failed to open snapshot backend", zap.Error(err)) @@ -2242,7 +2245,7 @@ s.consistIndex.SetConsistentApplyingIndex(e.Index, e.Term) shouldApplyV3 = membership.ApplyBoth } - + // gofail: var beforeApplyOneConfChange struct{} var cc raftpb.ConfChange pbutil.MustUnmarshal(&cc, e.Data) removedSelf, err := s.applyConfChange(cc, confState, shouldApplyV3) @@ -2448,9 +2451,51 @@ s.r.transport.UpdatePeer(m.ID, m.PeerURLs) } } + + s.verifyV3StoreInSyncWithV2Store(shouldApplyV3) + return false, nil } +func (s *EtcdServer) verifyV3StoreInSyncWithV2Store(shouldApplyV3 membership.ShouldApplyV3) { + if !verify.VerifyEnabled() { + return + } + + // If shouldApplyV3 == false, then it means v2store hasn't caught up with v3store. + if !shouldApplyV3 { + return + } + + // clean up the Attributes, and we only care about the RaftAttributes + cleanAttributesFunc := func(members map[types.ID]*membership.Member) map[types.ID]*membership.Member { + processedMembers := make(map[types.ID]*membership.Member) + for id, m := range members { + clonedMember := m.Clone() + clonedMember.Attributes = membership.Attributes{} + processedMembers[id] = clonedMember + } + + return processedMembers + } + + v2Members, _ := s.cluster.MembersFromStore() + v3Members, _ := s.cluster.MembersFromBackend() + + processedV2Members := cleanAttributesFunc(v2Members) + processedV3Members := cleanAttributesFunc(v3Members) + + if match := reflect.DeepEqual(processedV2Members, processedV3Members); !match { + v2Data, v2Err := json.Marshal(processedV2Members) + v3Data, v3Err := json.Marshal(processedV3Members) + + if v2Err != nil || v3Err != nil { + panic("members in v2store doesn't match v3store") + } + panic(fmt.Sprintf("members in v2store doesn't match v3store, v2store: %s, v3store: %s", string(v2Data), string(v3Data))) + } +} + // TODO: non-blocking snapshot func (s *EtcdServer) snapshot(snapi uint64, confState raftpb.ConfState) { clone := s.v2store.Clone() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/lease/lessor.go new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/lease/lessor.go --- old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/lease/lessor.go 2025-03-06 14:17:07.574206969 +0100 +++ new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/lease/lessor.go 2025-03-24 10:40:28.369940485 +0100 @@ -26,11 +26,12 @@ "time" "github.com/coreos/go-semver/semver" + "go.uber.org/zap" + pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/server/v3/lease/leasepb" "go.etcd.io/etcd/server/v3/mvcc/backend" "go.etcd.io/etcd/server/v3/mvcc/buckets" - "go.uber.org/zap" ) // NoLease is a special LeaseID representing the absence of a lease. @@ -912,8 +913,8 @@ // Demoted returns true if the lease's expiry has been reset to forever. func (l *Lease) Demoted() bool { - l.expiryMu.Lock() - defer l.expiryMu.Unlock() + l.expiryMu.RLock() + defer l.expiryMu.RUnlock() return l.expiry == forever } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/mvcc/backend/backend.go new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/mvcc/backend/backend.go --- old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/mvcc/backend/backend.go 2025-03-06 14:17:07.576698211 +0100 +++ new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/mvcc/backend/backend.go 2025-03-24 10:40:28.371940485 +0100 @@ -669,7 +669,9 @@ } func (b *backend) unsafeBegin(write bool) *bolt.Tx { + // gofail: var beforeStartDBTxn struct{} tx, err := b.db.Begin(write) + // gofail: var afterStartDBTxn struct{} if err != nil { b.lg.Fatal("failed to begin tx", zap.Error(err)) } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/mvcc/backend/batch_tx.go new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/mvcc/backend/batch_tx.go --- old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/mvcc/backend/batch_tx.go 2025-03-06 14:17:07.576698211 +0100 +++ new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/mvcc/backend/batch_tx.go 2025-03-24 10:40:28.371940485 +0100 @@ -310,6 +310,7 @@ t.backend.readTx.Lock() // blocks txReadBuffer for writing. // gofail: var beforeWritebackBuf struct{} t.buf.writeback(&t.backend.readTx.buf) + // gofail: var afterWritebackBuf struct{} t.backend.readTx.Unlock() // We commit the transaction when the number of pending operations // reaches the configured limit(batchLimit) to prevent it from @@ -359,7 +360,9 @@ func (t *batchTxBuffered) unsafeCommit(stop bool) { if t.backend.hooks != nil { + // gofail: var commitBeforePreCommitHook struct{} t.backend.hooks.OnPreCommitUnsafe(t) + // gofail: var commitAfterPreCommitHook struct{} } if t.backend.readTx.tx != nil { // wait all store read transactions using the current boltdb tx to finish, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/mvcc/kvstore.go new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/mvcc/kvstore.go --- old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/mvcc/kvstore.go 2025-03-06 14:17:07.575040469 +0100 +++ new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/mvcc/kvstore.go 2025-03-24 10:40:28.371103244 +0100 @@ -225,7 +225,9 @@ tx.UnsafePut(buckets.Meta, scheduledCompactKeyName, rbytes) tx.Unlock() // ensure that desired compaction is persisted + // gofail: var compactBeforeCommitScheduledCompact struct{} s.b.ForceCommit() + // gofail: var compactAfterCommitScheduledCompact struct{} s.revMu.Unlock() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/mvcc/kvstore_compaction.go new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/mvcc/kvstore_compaction.go --- old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/mvcc/kvstore_compaction.go 2025-03-06 14:17:07.575040469 +0100 +++ new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/mvcc/kvstore_compaction.go 2025-03-24 10:40:28.371103244 +0100 @@ -65,6 +65,7 @@ revToBytes(revision{main: compactMainRev}, rbytes) tx.UnsafePut(buckets.Meta, finishedCompactKeyName, rbytes) tx.Unlock() + // gofail: var compactAfterSetFinishedCompact struct{} hash := h.Hash() size, sizeInUse := s.b.Size(), s.b.SizeInUse() s.lg.Info( @@ -84,7 +85,9 @@ revToBytes(revision{main: rev.main, sub: rev.sub + 1}, last) tx.Unlock() // Immediately commit the compaction deletes instead of letting them accumulate in the write buffer + // gofail: var compactBeforeCommitBatch struct{} s.b.ForceCommit() + // gofail: var compactAfterCommitBatch struct{} dbCompactionPauseMs.Observe(float64(time.Since(start) / time.Millisecond)) select { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/verify/verify.go new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/verify/verify.go --- old/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/verify/verify.go 2025-03-06 14:17:07.577698232 +0100 +++ new/vendor/etcdutl/vendor/go.etcd.io/etcd/server/v3/verify/verify.go 2025-03-24 10:40:28.373940485 +0100 @@ -90,12 +90,37 @@ // VerifyIfEnabled performs verification according to ETCD_VERIFY env settings. // See Verify for more information. func VerifyIfEnabled(cfg Config) error { - if os.Getenv(ENV_VERIFY) == ENV_VERIFY_ALL_VALUE { + if VerifyEnabled() { return Verify(cfg) } return nil } +// VerifyEnabled returns `true` if verification is enabled. +func VerifyEnabled() bool { + return os.Getenv(ENV_VERIFY) == ENV_VERIFY_ALL_VALUE +} + +// EnableVerification enables the verification and returns a function that +// can be used to bring the original settings. +func EnableVerification() func() { + previousEnv := os.Getenv(ENV_VERIFY) + os.Setenv(ENV_VERIFY, ENV_VERIFY_ALL_VALUE) + return func() { + os.Setenv(ENV_VERIFY, previousEnv) + } +} + +// DisableVerification disables the verification and returns a function that +// can be used to bring the original settings. +func DisableVerification() func() { + previousEnv := os.Getenv(ENV_VERIFY) + os.Unsetenv(ENV_VERIFY) + return func() { + os.Setenv(ENV_VERIFY, previousEnv) + } +} + // MustVerifyIfEnabled performs verification according to ETCD_VERIFY env settings // and exits in case of found problems. // See Verify for more information. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/etcdutl/vendor/modules.txt new/vendor/etcdutl/vendor/modules.txt --- old/vendor/etcdutl/vendor/modules.txt 2025-03-06 14:17:07.824337276 +0100 +++ new/vendor/etcdutl/vendor/modules.txt 2025-03-24 10:40:28.684940481 +0100 @@ -92,7 +92,7 @@ # go.etcd.io/bbolt v1.3.11 ## explicit; go 1.22 go.etcd.io/bbolt -# go.etcd.io/etcd/api/v3 v3.5.19 => ../api +# go.etcd.io/etcd/api/v3 v3.5.20 => ../api ## explicit; go 1.23.0 go.etcd.io/etcd/api/v3/authpb go.etcd.io/etcd/api/v3/etcdserverpb @@ -100,7 +100,7 @@ go.etcd.io/etcd/api/v3/mvccpb go.etcd.io/etcd/api/v3/v3rpc/rpctypes go.etcd.io/etcd/api/v3/version -# go.etcd.io/etcd/client/pkg/v3 v3.5.19 => ../client/pkg +# go.etcd.io/etcd/client/pkg/v3 v3.5.20 => ../client/pkg ## explicit; go 1.23.0 go.etcd.io/etcd/client/pkg/v3/fileutil go.etcd.io/etcd/client/pkg/v3/logutil @@ -110,17 +110,17 @@ go.etcd.io/etcd/client/pkg/v3/tlsutil go.etcd.io/etcd/client/pkg/v3/transport go.etcd.io/etcd/client/pkg/v3/types -# go.etcd.io/etcd/client/v2 v2.305.19 => ../client/v2 +# go.etcd.io/etcd/client/v2 v2.305.20 => ../client/v2 ## explicit; go 1.23.0 go.etcd.io/etcd/client/v2 -# go.etcd.io/etcd/client/v3 v3.5.19 => ../client/v3 +# go.etcd.io/etcd/client/v3 v3.5.20 => ../client/v3 ## explicit; go 1.23.0 go.etcd.io/etcd/client/v3 go.etcd.io/etcd/client/v3/credentials go.etcd.io/etcd/client/v3/internal/endpoint go.etcd.io/etcd/client/v3/internal/resolver go.etcd.io/etcd/client/v3/snapshot -# go.etcd.io/etcd/pkg/v3 v3.5.19 => ../pkg +# go.etcd.io/etcd/pkg/v3 v3.5.20 => ../pkg ## explicit; go 1.23.0 go.etcd.io/etcd/pkg/v3/adt go.etcd.io/etcd/pkg/v3/cobrautl @@ -136,14 +136,14 @@ go.etcd.io/etcd/pkg/v3/schedule go.etcd.io/etcd/pkg/v3/traceutil go.etcd.io/etcd/pkg/v3/wait -# go.etcd.io/etcd/raft/v3 v3.5.19 => ../raft +# go.etcd.io/etcd/raft/v3 v3.5.20 => ../raft ## explicit; go 1.23.0 go.etcd.io/etcd/raft/v3 go.etcd.io/etcd/raft/v3/confchange go.etcd.io/etcd/raft/v3/quorum go.etcd.io/etcd/raft/v3/raftpb go.etcd.io/etcd/raft/v3/tracker -# go.etcd.io/etcd/server/v3 v3.5.19 => ../server +# go.etcd.io/etcd/server/v3 v3.5.20 => ../server ## explicit; go 1.23.0 go.etcd.io/etcd/server/v3/auth go.etcd.io/etcd/server/v3/config diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/server/vendor/go.etcd.io/etcd/api/v3/version/version.go new/vendor/server/vendor/go.etcd.io/etcd/api/v3/version/version.go --- old/vendor/server/vendor/go.etcd.io/etcd/api/v3/version/version.go 2025-03-06 14:17:07.377058826 +0100 +++ new/vendor/server/vendor/go.etcd.io/etcd/api/v3/version/version.go 2025-03-24 10:40:27.729342397 +0100 @@ -26,7 +26,7 @@ var ( // MinClusterVersion is the min cluster version this etcd binary is compatible with. MinClusterVersion = "3.0.0" - Version = "3.5.19" + Version = "3.5.20" APIVersion = "unknown" // Git SHA Value will be set during build diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/server/vendor/modules.txt new/vendor/server/vendor/modules.txt --- old/vendor/server/vendor/modules.txt 2025-03-06 14:17:07.486582002 +0100 +++ new/vendor/server/vendor/modules.txt 2025-03-24 10:40:27.892940491 +0100 @@ -142,7 +142,7 @@ # go.etcd.io/bbolt v1.3.11 ## explicit; go 1.22 go.etcd.io/bbolt -# go.etcd.io/etcd/api/v3 v3.5.19 => ../api +# go.etcd.io/etcd/api/v3 v3.5.20 => ../api ## explicit; go 1.23.0 go.etcd.io/etcd/api/v3/authpb go.etcd.io/etcd/api/v3/etcdserverpb @@ -151,7 +151,7 @@ go.etcd.io/etcd/api/v3/mvccpb go.etcd.io/etcd/api/v3/v3rpc/rpctypes go.etcd.io/etcd/api/v3/version -# go.etcd.io/etcd/client/pkg/v3 v3.5.19 => ../client/pkg +# go.etcd.io/etcd/client/pkg/v3 v3.5.20 => ../client/pkg ## explicit; go 1.23.0 go.etcd.io/etcd/client/pkg/v3/fileutil go.etcd.io/etcd/client/pkg/v3/logutil @@ -162,10 +162,10 @@ go.etcd.io/etcd/client/pkg/v3/tlsutil go.etcd.io/etcd/client/pkg/v3/transport go.etcd.io/etcd/client/pkg/v3/types -# go.etcd.io/etcd/client/v2 v2.305.19 => ../client/v2 +# go.etcd.io/etcd/client/v2 v2.305.20 => ../client/v2 ## explicit; go 1.23.0 go.etcd.io/etcd/client/v2 -# go.etcd.io/etcd/client/v3 v3.5.19 => ../client/v3 +# go.etcd.io/etcd/client/v3 v3.5.20 => ../client/v3 ## explicit; go 1.23.0 go.etcd.io/etcd/client/v3 go.etcd.io/etcd/client/v3/concurrency @@ -177,7 +177,7 @@ go.etcd.io/etcd/client/v3/naming/endpoints go.etcd.io/etcd/client/v3/naming/endpoints/internal go.etcd.io/etcd/client/v3/ordering -# go.etcd.io/etcd/pkg/v3 v3.5.19 => ../pkg +# go.etcd.io/etcd/pkg/v3 v3.5.20 => ../pkg ## explicit; go 1.23.0 go.etcd.io/etcd/pkg/v3/adt go.etcd.io/etcd/pkg/v3/contention @@ -195,7 +195,7 @@ go.etcd.io/etcd/pkg/v3/schedule go.etcd.io/etcd/pkg/v3/traceutil go.etcd.io/etcd/pkg/v3/wait -# go.etcd.io/etcd/raft/v3 v3.5.19 => ../raft +# go.etcd.io/etcd/raft/v3 v3.5.20 => ../raft ## explicit; go 1.23.0 go.etcd.io/etcd/raft/v3 go.etcd.io/etcd/raft/v3/confchange