Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package rqlite for openSUSE:Factory checked in at 2026-01-06 17:45:30 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/rqlite (Old) and /work/SRC/openSUSE:Factory/.rqlite.new.1928 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "rqlite" Tue Jan 6 17:45:30 2026 rev:38 rq:1325487 version:9.3.10 Changes: -------- --- /work/SRC/openSUSE:Factory/rqlite/rqlite.changes 2026-01-05 14:52:18.544152372 +0100 +++ /work/SRC/openSUSE:Factory/.rqlite.new.1928/rqlite.changes 2026-01-06 17:46:59.095551933 +0100 @@ -1,0 +2,9 @@ +Mon Jan 05 21:29:41 UTC 2026 - Andreas Stieger <[email protected]> + +- Update to version 9.3.10: + * Remove any temporary WAL files if persisting a Snapshot fails + or is not even invoked + * Improve FSM logging + * Order alphabetically output of shell command .tables + +------------------------------------------------------------------- Old: ---- rqlite-9.3.9.tar.xz New: ---- rqlite-9.3.10.tar.xz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ rqlite.spec ++++++ --- /var/tmp/diff_new_pack.ivciF8/_old 2026-01-06 17:46:59.667575465 +0100 +++ /var/tmp/diff_new_pack.ivciF8/_new 2026-01-06 17:46:59.667575465 +0100 @@ -17,7 +17,7 @@ Name: rqlite -Version: 9.3.9 +Version: 9.3.10 Release: 0 Summary: Distributed relational database built on SQLite License: MIT ++++++ _service ++++++ --- /var/tmp/diff_new_pack.ivciF8/_old 2026-01-06 17:46:59.703576946 +0100 +++ /var/tmp/diff_new_pack.ivciF8/_new 2026-01-06 17:46:59.707577111 +0100 @@ -3,7 +3,7 @@ <param name="url">https://github.com/rqlite/rqlite.git</param> <param name="scm">git</param> <param name="exclude">.git</param> - <param name="revision">v9.3.9</param> + <param name="revision">v9.3.10</param> <param name="versionformat">@PARENT_TAG@</param> <param name="changesgenerate">enable</param> <param name="versionrewrite-pattern">v(.*)</param> ++++++ _servicedata ++++++ --- /var/tmp/diff_new_pack.ivciF8/_old 2026-01-06 17:46:59.735578263 +0100 +++ /var/tmp/diff_new_pack.ivciF8/_new 2026-01-06 17:46:59.739578427 +0100 @@ -1,7 +1,7 @@ <servicedata> <service name="tar_scm"> <param name="url">https://github.com/rqlite/rqlite.git</param> - <param name="changesrevision">3c0f61cc18e55f070246a451084a9fa49563c777</param> + <param name="changesrevision">fdd7ac15bf191ab7c850e60e5cd928484553c57a</param> </service> </servicedata> (No newline at EOF) ++++++ rqlite-9.3.9.tar.xz -> rqlite-9.3.10.tar.xz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rqlite-9.3.9/CHANGELOG.md new/rqlite-9.3.10/CHANGELOG.md --- old/rqlite-9.3.9/CHANGELOG.md 2026-01-04 08:09:34.000000000 +0100 +++ new/rqlite-9.3.10/CHANGELOG.md 2026-01-05 14:44:28.000000000 +0100 @@ -1,3 +1,10 @@ +## v9.3.10 (January 5th 2026) +### Implementation changes and bug fixes +- [PR #2427](https://github.com/rqlite/rqlite/pull/2427): Remove any temporary WAL files if persisting a Snapshot fails or is not even invoked. +- [PR #2428](https://github.com/rqlite/rqlite/pull/2428): Move Store-Snapshot unit testing to own source file. +- [PR #2429](https://github.com/rqlite/rqlite/pull/2429): Improve FSM logging. +- [PR #2432](https://github.com/rqlite/rqlite/pull/2432): Order alphabetically output of shell command `.tables`. Fixes issue [#2431](https://github.com/rqlite/rqlite/issues/2431). + ## v9.3.9 (January 4th 2026) ### Implementation changes and bug fixes - [PR #2423](https://github.com/rqlite/rqlite/pull/2423): Handle possible WAL checkpoint failure. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rqlite-9.3.9/cmd/rqlite/main.go new/rqlite-9.3.10/cmd/rqlite/main.go --- old/rqlite-9.3.9/cmd/rqlite/main.go 2026-01-04 08:09:34.000000000 +0100 +++ new/rqlite-9.3.10/cmd/rqlite/main.go 2026-01-05 14:44:28.000000000 +0100 @@ -213,7 +213,7 @@ } err = toggleFlag(input[index+1:], &forceWrites) case ".TABLES": - err = queryWithClient(ctx, client, timer, blobArray, consistency, `SELECT name FROM sqlite_master WHERE type="table"`) + err = queryWithClient(ctx, client, timer, blobArray, consistency, `SELECT name FROM sqlite_master WHERE type="table" ORDER BY name ASC`) case ".INDEXES": err = queryWithClient(ctx, client, timer, blobArray, consistency, `SELECT sql FROM sqlite_master WHERE type="index"`) case ".SCHEMA": diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rqlite-9.3.9/db/db.go new/rqlite-9.3.10/db/db.go --- old/rqlite-9.3.9/db/db.go 2026-01-04 08:09:34.000000000 +0100 +++ new/rqlite-9.3.10/db/db.go 2026-01-05 14:44:28.000000000 +0100 @@ -659,7 +659,7 @@ // Checkpoint checkpoints the WAL file. If the WAL file is not enabled, this // function is a no-op. func (db *DB) Checkpoint(mode CheckpointMode) (*CheckpointMeta, error) { - return db.CheckpointWithTimeout(mode, 100) + return db.CheckpointWithTimeout(mode, 1000) } // CheckpointWithTimeout performs a WAL checkpoint. If the checkpoint does not diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rqlite-9.3.9/store/fsm.go new/rqlite-9.3.10/store/fsm.go --- old/rqlite-9.3.9/store/fsm.go 2026-01-04 08:09:34.000000000 +0100 +++ new/rqlite-9.3.10/store/fsm.go 2026-01-05 14:44:28.000000000 +0100 @@ -40,16 +40,21 @@ // FSMSnapshot is a wrapper around raft.FSMSnapshot which adds an optional // Finalizer, instrumentation, and logging. type FSMSnapshot struct { + Full bool Finalizer func() error - OnFailure func() + OnRelease func(invoked, succeeded bool) raft.FSMSnapshot + persistInvoked bool persistSucceeded bool logger *log.Logger } // Persist writes the snapshot to the given sink. func (f *FSMSnapshot) Persist(sink raft.SnapshotSink) (retError error) { + fpLog := fullPretty(f.Full) + f.persistInvoked = true + startT := time.Now() defer func() { if retError == nil { @@ -58,14 +63,14 @@ stats.Add(numSnapshotPersists, 1) stats.Get(snapshotPersistDuration).(*expvar.Int).Set(dur.Milliseconds()) if f.logger != nil { - f.logger.Printf("persisted snapshot %s in %s", sink.ID(), dur) + f.logger.Printf("persisted %s snapshot %s in %s", fpLog, sink.ID(), dur) } } else { stats.Add(numSnapshotPersistsFailed, 1) } }() if err := f.FSMSnapshot.Persist(sink); err != nil { - fsmSnapshotErrLogger.Printf("failed to persist snapshot %s: %v", sink.ID(), err) + fsmSnapshotErrLogger.Printf("failed to persist %s snapshot %s: %v", fpLog, sink.ID(), err) return err } if f.Finalizer != nil { @@ -77,7 +82,7 @@ // Release performs any final cleanup once the Snapshot has been persisted. func (f *FSMSnapshot) Release() { f.FSMSnapshot.Release() - if !f.persistSucceeded && f.OnFailure != nil { - f.OnFailure() + if f.OnRelease != nil { + f.OnRelease(f.persistInvoked, f.persistSucceeded) } } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rqlite-9.3.9/store/fsm_test.go new/rqlite-9.3.10/store/fsm_test.go --- old/rqlite-9.3.9/store/fsm_test.go 2026-01-04 08:09:34.000000000 +0100 +++ new/rqlite-9.3.10/store/fsm_test.go 2026-01-05 14:44:28.000000000 +0100 @@ -1,6 +1,7 @@ package store import ( + "errors" "testing" "github.com/hashicorp/raft" @@ -25,11 +26,16 @@ } } -func Test_FSMSnapshot_OnFailure_NotCalled(t *testing.T) { - onFailureCalled := false +func Test_FSMSnapshot_OnRelease_OK(t *testing.T) { + onReleaseCalled := false + invoked := false + succeeded := false + f := FSMSnapshot{ - OnFailure: func() { - onFailureCalled = true + OnRelease: func(i, s bool) { + onReleaseCalled = true + invoked = i + succeeded = s }, FSMSnapshot: &mockRaftSnapshot{}, logger: nil, @@ -39,29 +45,70 @@ t.Fatalf("unexpected error: %v", err) } f.Release() - if onFailureCalled { - t.Fatalf("OnFailure was called") + if !onReleaseCalled { + t.Fatalf("OnRelease was not called") + } + if !invoked { + t.Fatalf("OnRelease invoked argument incorrect") + } + if !succeeded { + t.Fatalf("OnRelease succeeded argument incorrect") } } -func Test_FSMSnapshot_OnFailure_Called(t *testing.T) { - onFailureCalled := false +func Test_FSMSnapshot_OnRelease_NotInvoked(t *testing.T) { + onReleaseCalled := false + invoked := false + f := FSMSnapshot{ - OnFailure: func() { - onFailureCalled = true + OnRelease: func(i, s bool) { + onReleaseCalled = true + invoked = i }, FSMSnapshot: &mockRaftSnapshot{}, logger: nil, } + f.Release() - if !onFailureCalled { - t.Fatalf("OnFailure was not called") + if !onReleaseCalled { + t.Fatalf("OnRelease was not called") + } + if invoked { + t.Fatalf("OnRelease invoked argument incorrect") } } -type mockSink struct { +func Test_FSMSnapshot_OnRelease_NotSucceeded(t *testing.T) { + onReleaseCalled := false + invoked := false + succeeded := false + + f := FSMSnapshot{ + OnRelease: func(i, s bool) { + onReleaseCalled = true + invoked = i + succeeded = s + }, + FSMSnapshot: &mockRaftSnapshot{forceErr: true}, + logger: nil, + } + + f.Persist(&mockSink{}) + + f.Release() + if !onReleaseCalled { + t.Fatalf("OnRelease was not called") + } + if !invoked { + t.Fatalf("OnRelease invoked argument incorrect") + } + if succeeded { + t.Fatalf("OnRelease succeeded argument incorrect") + } } +type mockSink struct{} + func (m *mockSink) ID() string { return "" } @@ -79,9 +126,13 @@ } type mockRaftSnapshot struct { + forceErr bool } func (m *mockRaftSnapshot) Persist(sink raft.SnapshotSink) error { + if m.forceErr { + return errors.New("forced error") + } return nil } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rqlite-9.3.9/store/store.go new/rqlite-9.3.10/store/store.go --- old/rqlite-9.3.9/store/store.go 2026-01-04 08:09:34.000000000 +0100 +++ new/rqlite-9.3.10/store/store.go 2026-01-05 14:44:28.000000000 +0100 @@ -2526,6 +2526,7 @@ }() var fsmSnapshot raft.FSMSnapshot + var walTmpFD *os.File if fullNeeded { chkStartTime := time.Now() meta, err := s.db.Checkpoint(sql.CheckpointTruncate) @@ -2553,7 +2554,6 @@ stats.Add(numSnapshotsFull, 1) s.numFullSnapshots++ } else { - var walTmpFD *os.File if pathExistsWithData(s.walPath) { // Using files is about protecting against large WAL files, even // post-compaction. Large files, if processed entirely in memory, could @@ -2625,6 +2625,10 @@ if walTmpFD != nil { name = walTmpFD.Name() } + // When it comes to incremental snapshotting of WAL files, we pass the data to the Snapshot + // Store and Sink indirectly. We wrap its filepath in an io.Reader, not the file data. The + // Snapshotting system knows to check for this. If it finds a filepath in the io.Reader (as + // opposed to a reader returning actual file data, it will move the file from here to it. fsmSnapshot = snapshot.NewSnapshot(io.NopCloser(bytes.NewBufferString(name))) stats.Add(numSnapshotsIncremental, 1) } @@ -2633,15 +2637,35 @@ dur := time.Since(startT) stats.Get(snapshotCreateDuration).(*expvar.Int).Set(dur.Milliseconds()) fs := FSMSnapshot{ + Full: fullNeeded, FSMSnapshot: fsmSnapshot, Finalizer: func() error { return s.createSnapshotFingerprint() }, - OnFailure: func() { - s.logger.Printf("Persisting snapshot did not succeed, full snapshot needed") - if err := s.snapshotStore.SetFullNeeded(); err != nil { - // If this happens, only recourse is to shut down the node. - s.logger.Fatalf("failed to set full snapshot needed: %s", err) + OnRelease: func(invoked, succeeded bool) { + if !invoked { + s.logger.Printf("persisting %s snapshot was not invoked on node ID %s", fPLog, s.raftID) + } else if !succeeded { + s.logger.Printf("persisting %s snapshot did not succeed on node ID %s", fPLog, s.raftID) + } + + // We treat any snapshot that is not successfully persisted for whatever reason as + // requiring a full snapshot next time. Truncation could run to complete, the WAL + // deleted, but if the snapshot processing doesn't run to completion, the snapshot + // store hasn't been updated with the WAL data. It's gone. + if !invoked || !succeeded { + s.logger.Printf("setting full snapshot needed") + if err := s.snapshotStore.SetFullNeeded(); err != nil { + // If this happens, only recourse is to shut down the node. + s.logger.Fatalf("failed to set full snapshot needed: %s", err) + } + } + + if walTmpFD != nil { + // Incremental snapshotting active, clean up any temp WAL files. + // There may be none around, but that's OK. + walTmpFD.Close() + os.Remove(walTmpFD.Name()) } }, } @@ -2926,7 +2950,7 @@ return } case <-time.After(mustWALCheckpointTimeout): - panic("timed out trying to truncate checkpointed WAL") + s.logger.Fatal("timed out trying to truncate checkpointed WAL - aborting") } } } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rqlite-9.3.9/store/store_multi_test.go new/rqlite-9.3.10/store/store_multi_test.go --- old/rqlite-9.3.9/store/store_multi_test.go 2026-01-04 08:09:34.000000000 +0100 +++ new/rqlite-9.3.10/store/store_multi_test.go 2026-01-05 14:44:28.000000000 +0100 @@ -472,6 +472,7 @@ func Test_MultiNodeSnapshot_BlockedSnapshot(t *testing.T) { // Fire up first node and write one record. s0, ln := mustNewStore(t) + s0.NoSnapshotOnClose = true defer ln.Close() if err := s0.Open(); err != nil { t.Fatalf("failed to open single-node store: %s", err.Error()) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rqlite-9.3.9/store/store_snapshot_test.go new/rqlite-9.3.10/store/store_snapshot_test.go --- old/rqlite-9.3.9/store/store_snapshot_test.go 1970-01-01 01:00:00.000000000 +0100 +++ new/rqlite-9.3.10/store/store_snapshot_test.go 2026-01-05 14:44:28.000000000 +0100 @@ -0,0 +1,633 @@ +package store + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/rqlite/rqlite/v9/command/proto" + "github.com/rqlite/rqlite/v9/db" + "github.com/rqlite/rqlite/v9/internal/random" +) + +// Test_SingleNodeSnapshot tests that the Store correctly takes a snapshot +// and recovers from it. +func Test_SingleNodeSnapshot(t *testing.T) { + s, ln := mustNewStore(t) + defer ln.Close() + + if err := s.Open(); err != nil { + t.Fatalf("failed to open single-node store: %s", err.Error()) + } + defer s.Close(true) + if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { + t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) + } + if _, err := s.WaitForLeader(10 * time.Second); err != nil { + t.Fatalf("Error waiting for leader: %s", err) + } + + queries := []string{ + `CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`, + `INSERT INTO foo(id, name) VALUES(1, "fiona")`, + } + _, _, err := s.Execute(executeRequestFromStrings(queries, false, false)) + if err != nil { + t.Fatalf("failed to execute on single node: %s", err.Error()) + } + _, _, _, err = s.Query(queryRequestFromString("SELECT * FROM foo", false, false)) + if err != nil { + t.Fatalf("failed to query single node: %s", err.Error()) + } + + // Snap the node and write to disk. + fsm := NewFSM(s) + f, err := fsm.Snapshot() + if err != nil { + t.Fatalf("failed to snapshot node: %s", err.Error()) + } + + snapDir := t.TempDir() + snapFile, err := os.Create(filepath.Join(snapDir, "snapshot")) + if err != nil { + t.Fatalf("failed to create snapshot file: %s", err.Error()) + } + defer snapFile.Close() + sink := &mockSnapshotSink{snapFile} + if err := f.Persist(sink); err != nil { + t.Fatalf("failed to persist snapshot to disk: %s", err.Error()) + } + + // Check restoration. + snapFile, err = os.Open(filepath.Join(snapDir, "snapshot")) + if err != nil { + t.Fatalf("failed to open snapshot file: %s", err.Error()) + } + defer snapFile.Close() + if err := fsm.Restore(snapFile); err != nil { + t.Fatalf("failed to restore snapshot from disk: %s", err.Error()) + } + + // Ensure database is back in the correct state. + r, _, _, err := s.Query(queryRequestFromString("SELECT * FROM foo", false, false)) + if err != nil { + t.Fatalf("failed to query single node: %s", err.Error()) + } + if exp, got := `["id","name"]`, asJSON(r[0].Columns); exp != got { + t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got) + } + if exp, got := `[[1,"fiona"]]`, asJSON(r[0].Values); exp != got { + t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got) + } +} + +func Test_SingleNodeUserSnapshot_CAS(t *testing.T) { + s, ln := mustNewStore(t) + defer ln.Close() + if err := s.Open(); err != nil { + t.Fatalf("failed to open single-node store: %s", err.Error()) + } + defer s.Close(true) + if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { + t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) + } + if _, err := s.WaitForLeader(10 * time.Second); err != nil { + t.Fatalf("Error waiting for leader: %s", err) + } + + mustNoop(s, "1") + if err := s.Snapshot(0); err != nil { + t.Fatalf("failed to snapshot single-node store: %s", err.Error()) + } + + if err := s.snapshotCAS.Begin("snapshot-test"); err != nil { + t.Fatalf("failed to begin snapshot CAS: %s", err.Error()) + } + mustNoop(s, "2") + if err := s.Snapshot(0); err == nil { + t.Fatalf("expected error snapshotting single-node store with CAS") + } + s.snapshotCAS.End() + mustNoop(s, "3") + if err := s.Snapshot(0); err != nil { + t.Fatalf("failed to snapshot single-node store: %s", err.Error()) + } +} + +func Test_SingleNodeUserSnapshot_Sync(t *testing.T) { + s, ln := mustNewStore(t) + defer ln.Close() + if err := s.Open(); err != nil { + t.Fatalf("failed to open single-node store: %s", err.Error()) + } + defer s.Close(true) + if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { + t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) + } + if _, err := s.WaitForLeader(10 * time.Second); err != nil { + t.Fatalf("Error waiting for leader: %s", err) + } + + mustNoop(s, "1") + if err := s.Snapshot(0); err != nil { + t.Fatalf("failed to snapshot single-node store: %s", err.Error()) + } + + // Register a channel, and close it, allowing snapshotting to proceed. + ch := make(chan chan struct{}) + s.RegisterSnapshotSync(ch) + called := false + go func() { + c := <-ch + called = true + close(c) + }() + mustNoop(s, "2") + if err := s.Snapshot(0); err != nil { + t.Fatalf("failed to snapshot single-node store with sync: %s", err.Error()) + } + if !called { + t.Fatalf("expected sync function to be called") + } + + // Register a channel, but don't close it, which should cause a timeout. + mustNoop(s, "3") + if err := s.Snapshot(0); err == nil { + t.Fatalf("snapshotting succeeded, expected failure due to sync timeout") + } +} + +func Test_SingleNode_WALTriggeredSnapshot(t *testing.T) { + s, ln := mustNewStore(t) + defer ln.Close() + s.SnapshotThreshold = 8192 + s.SnapshotInterval = 500 * time.Millisecond + s.SnapshotThresholdWALSize = 4096 + + if err := s.Open(); err != nil { + t.Fatalf("failed to open single-node store: %s", err.Error()) + } + defer s.Close(true) + if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { + t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) + } + if _, err := s.WaitForLeader(10 * time.Second); err != nil { + t.Fatalf("Error waiting for leader: %s", err) + } + er := executeRequestFromString(`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`, + false, false) + _, _, err := s.Execute(er) + if err != nil { + t.Fatalf("failed to execute on single node: %s", err.Error()) + } + nSnaps := stats.Get(numWALSnapshots).String() + + for i := 0; i < 100; i++ { + _, _, err := s.Execute(executeRequestFromString(`INSERT INTO foo(name) VALUES("fiona")`, false, false)) + if err != nil { + t.Fatalf("failed to execute INSERT on single node: %s", err.Error()) + } + } + + // Ensure WAL-triggered snapshots take place. + f := func() bool { + return stats.Get(numWALSnapshots).String() != nSnaps + } + testPoll(t, f, 100*time.Millisecond, 2*time.Second) + + // Sanity-check the contents of the Store. There should be two + // files -- a SQLite database file, and a directory named after + // the most recent snapshot. This basically checks that reaping + // is working, as it can be tricky on Windows due to stricter + // file deletion rules. + time.Sleep(5 * time.Second) // Tricky to know when all snapshots are done. Just wait. + snaps, err := s.snapshotStore.List() + if err != nil { + t.Fatalf("failed to list snapshots: %s", err.Error()) + } + if len(snaps) != 1 { + t.Fatalf("wrong number of snapshots: %d", len(snaps)) + } + snapshotDir := filepath.Join(s.raftDir, snapshotsDirName) + files, err := os.ReadDir(snapshotDir) + if err != nil { + t.Fatalf("failed to read snapshot store dir: %s", err.Error()) + } + if len(files) != 2 { + t.Fatalf("wrong number of snapshot store files: %d", len(files)) + } + for _, f := range files { + if !strings.Contains(f.Name(), snaps[0].ID) { + t.Fatalf("wrong snapshot store file: %s", f.Name()) + } + } +} + +func Test_SingleNode_SnapshotFail_Blocked(t *testing.T) { + s, ln := mustNewStore(t) + defer ln.Close() + + s.SnapshotThreshold = 8192 + s.SnapshotInterval = time.Hour + s.NoSnapshotOnClose = true + if err := s.Open(); err != nil { + t.Fatalf("failed to open single-node store: %s", err.Error()) + } + defer s.Close(true) + if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { + t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) + } + if _, err := s.WaitForLeader(10 * time.Second); err != nil { + t.Fatalf("Error waiting for leader: %s", err) + } + er := executeRequestFromString(`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`, + false, false) + _, _, err := s.Execute(er) + if err != nil { + t.Fatalf("failed to execute on single node: %s", err.Error()) + } + + er = executeRequestFromString(`INSERT INTO foo(name) VALUES("fiona")`, false, false) + _, _, err = s.Execute(er) + if err != nil { + t.Fatalf("failed to execute on single node: %s", err.Error()) + } + + ctx, cancelFunc := context.WithCancel(context.Background()) + go func() { + qr := queryRequestFromString("SELECT * FROM foo", false, false) + qr.GetRequest().Statements[0].ForceStall = true + + blockingDB, err := db.Open(s.dbPath, false, true) + if err != nil { + t.Errorf("failed to open blocking DB connection: %s", err.Error()) + } + defer blockingDB.Close() + + _, err = blockingDB.QueryWithContext(ctx, qr.GetRequest(), false) + if err != nil { + t.Errorf("failed to execute stalled query on blocking DB connection: %s", err.Error()) + } + }() + time.Sleep(1 * time.Second) + + er = executeRequestFromString(`INSERT INTO foo(name) VALUES("bob")`, false, false) + _, _, err = s.Execute(er) + if err != nil { + t.Fatalf("failed to execute on single node: %s", err.Error()) + } + + if err := s.Snapshot(0); err == nil { + t.Fatalf("expected error snapshotting single-node store with stalled query") + } + + // Shutdown the blocking query so we can clean up. Windows in particular. + cancelFunc() + <-ctx.Done() +} + +// Test_SingleNode_SnapshotFail_Blocked_Retry tests that a snapshot operation +// that requires a forced checkpoint and truncation does succeed once the +// blocking query unblocks. +func Test_SingleNode_SnapshotFail_Blocked_Retry(t *testing.T) { + s, ln := mustNewStore(t) + defer ln.Close() + + s.SnapshotThreshold = 8192 + s.SnapshotInterval = time.Hour + s.NoSnapshotOnClose = true + if err := s.Open(); err != nil { + t.Fatalf("failed to open single-node store: %s", err.Error()) + } + defer s.Close(true) + if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { + t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) + } + if _, err := s.WaitForLeader(10 * time.Second); err != nil { + t.Fatalf("Error waiting for leader: %s", err) + } + er := executeRequestFromString(`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`, + false, false) + _, _, err := s.Execute(er) + if err != nil { + t.Fatalf("failed to execute on single node: %s", err.Error()) + } + + er = executeRequestFromString(`INSERT INTO foo(name) VALUES("fiona")`, false, false) + _, _, err = s.Execute(er) + if err != nil { + t.Fatalf("failed to execute on single node: %s", err.Error()) + } + + ctx, cancelFunc := context.WithCancel(context.Background()) + go func() { + qr := queryRequestFromString("SELECT * FROM foo", false, false) + qr.GetRequest().Statements[0].ForceStall = true + + blockingDB, err := db.Open(s.dbPath, false, true) + if err != nil { + t.Errorf("failed to open blocking DB connection: %s", err.Error()) + } + defer blockingDB.Close() + + _, err = blockingDB.QueryWithContext(ctx, qr.GetRequest(), false) + if err != nil { + t.Errorf("failed to execute stalled query on blocking DB connection: %s", err.Error()) + } + }() + time.Sleep(1 * time.Second) + + success := false + var wg sync.WaitGroup + wg.Go(func() { + if err := s.Snapshot(0); err != nil { + t.Errorf("failed to snapshot single-node store with released stalled query: %s", err.Error()) + } else { + success = true + } + }) + time.Sleep(1 * time.Second) + cancelFunc() + wg.Wait() + if !success { + t.Fatalf("expected snapshot to succeed after blocking query released") + } + + // Again, this time with a persistent snapshot. + er = executeRequestFromString(`INSERT INTO foo(name) VALUES("fiona")`, false, false) + _, _, err = s.Execute(er) + if err != nil { + t.Fatalf("failed to execute on single node: %s", err.Error()) + } + + ctx, cancelFunc = context.WithCancel(context.Background()) + go func() { + qr := queryRequestFromString("SELECT * FROM foo", false, false) + qr.GetRequest().Statements[0].ForceStall = true + + blockingDB, err := db.Open(s.dbPath, false, true) + if err != nil { + t.Errorf("failed to open blocking DB connection: %s", err.Error()) + } + defer blockingDB.Close() + + _, err = blockingDB.QueryWithContext(ctx, qr.GetRequest(), false) + if err != nil { + t.Errorf("failed to execute stalled query on blocking DB connection: %s", err.Error()) + } + }() + time.Sleep(1 * time.Second) + + success = false + var wg2 sync.WaitGroup + wg2.Go(func() { + if err := s.Snapshot(0); err != nil { + t.Errorf("failed to snapshot single-node store with second released stalled query: %s", err.Error()) + } else { + success = true + } + }) + time.Sleep(1 * time.Second) + cancelFunc() + wg2.Wait() + if !success { + t.Fatalf("expected snapshot to succeed after blocking query released") + } +} + +func Test_SingleNode_SnapshotWithAutoOptimize_Stress(t *testing.T) { + s, ln := mustNewStore(t) + defer ln.Close() + s.SnapshotThreshold = 50 + s.SnapshotInterval = 100 * time.Millisecond + s.AutoOptimizeInterval = 500 * time.Millisecond + + if err := s.Open(); err != nil { + t.Fatalf("failed to open single-node store: %s", err.Error()) + } + defer s.Close(true) + if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { + t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) + } + if _, err := s.WaitForLeader(10 * time.Second); err != nil { + t.Fatalf("Error waiting for leader: %s", err) + } + + // Create a table + er := executeRequestFromString(`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`, + false, false) + _, _, err := s.Execute(er) + if err != nil { + t.Fatalf("failed to execute on single node: %s", err.Error()) + } + + // Create an index on name + er = executeRequestFromString(`CREATE INDEX foo_name ON foo(name)`, false, false) + _, _, err = s.Execute(er) + if err != nil { + t.Fatalf("failed to execute on single node: %s", err.Error()) + } + + // Insert a bunch of data concurrently, putting some load on the Store. + var wg sync.WaitGroup + wg.Add(5) + insertFn := func() { + defer wg.Done() + for i := 0; i < 500; i++ { + _, _, err := s.Execute(executeRequestFromString(fmt.Sprintf(`INSERT INTO foo(name) VALUES("%s")`, random.String()), false, false)) + if err != nil { + t.Errorf("failed to execute INSERT on single node: %s", err.Error()) + } + } + } + for i := 0; i < 5; i++ { + go insertFn() + } + wg.Wait() + + // Query the data, make sure it looks good after all this. + qr := queryRequestFromString("SELECT COUNT(*) FROM foo", false, true) + qr.Level = proto.ConsistencyLevel_STRONG + r, _, _, err := s.Query(qr) + if err != nil { + t.Fatalf("failed to query single node: %s", err.Error()) + } + if exp, got := `[{"columns":["COUNT(*)"],"types":["integer"],"values":[[2500]]}]`, asJSON(r); exp != got { + t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got) + } + + // Restart the Store, make sure all still looks good. + if err := s.Close(true); err != nil { + t.Fatalf("failed to close store: %s", err.Error()) + } + if err := s.Open(); err != nil { + t.Fatalf("failed to open store: %s", err.Error()) + } + defer s.Close(true) + if _, err := s.WaitForLeader(10 * time.Second); err != nil { + t.Fatalf("Error waiting for leader: %s", err) + } + r, _, _, err = s.Query(qr) + if err != nil { + t.Fatalf("failed to query single node: %s", err.Error()) + } + if exp, got := `[{"columns":["COUNT(*)"],"types":["integer"],"values":[[2500]]}]`, asJSON(r); exp != got { + t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got) + } +} + +// Test_SingleNode_DatabaseFileModified tests that a full snapshot is taken +// when the underlying database file is modified by some process external +// to the Store. Such changes are officially unsupported, but if the Store +// detects such a change, it will take a full snapshot to ensure the Snapshot +// remains consistent. +func Test_SingleNode_DatabaseFileModified(t *testing.T) { + s, ln := mustNewStore(t) + defer ln.Close() + if err := s.Open(); err != nil { + t.Fatalf("failed to open single-node store: %s", err.Error()) + } + defer s.Close(true) + + if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { + t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) + } + if _, err := s.WaitForLeader(10 * time.Second); err != nil { + t.Fatalf("Error waiting for leader: %s", err) + } + + // Insert a record and trigger a snapshot to get a full snapshot. + er := executeRequestFromString(`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`, + false, false) + _, _, err := s.Execute(er) + if err != nil { + t.Fatalf("failed to execute on single node: %s", err.Error()) + } + if err := s.Snapshot(0); err != nil { + t.Fatalf("failed to snapshot single-node store: %s", err.Error()) + } + if s.numFullSnapshots != 1 { + t.Fatalf("expected 1 full snapshot, got %d", s.numFullSnapshots) + } + + insertSnap := func() { + t.Helper() + _, _, err := s.Execute(executeRequestFromString(`INSERT INTO foo(name) VALUES("fiona")`, false, false)) + if err != nil { + t.Fatalf("failed to execute INSERT on single node: %s", err.Error()) + } + if err := s.Snapshot(0); err != nil { + t.Fatalf("failed to snapshot single-node store: %s", err.Error()) + } + } + + // Insert a record, trigger a snapshot. It should be an incremental snapshot. + insertSnap() + if s.numFullSnapshots != 1 { + t.Fatalf("expected 1 full snapshot, got %d", s.numFullSnapshots) + } + + // Insert a record, trigger a snapshot. It shouldn't be a full snapshot. + insertSnap() + if s.numFullSnapshots != 1 { + t.Fatalf("expected 1 full snapshot, got %d", s.numFullSnapshots) + } + + lt, err := s.db.DBLastModified() + if err != nil { + t.Fatalf("failed to get last modified time of database: %s", err.Error()) + } + + // Touch the database file to make it newer than Store's record of last + // modified time and then trigger a snapshot. It should be a full snapshot. + if err := os.Chtimes(s.dbPath, time.Time{}, lt.Add(time.Second)); err != nil { + t.Fatalf("failed to change database file times: %s", err.Error()) + } + insertSnap() + if s.numFullSnapshots != 2 { + t.Fatalf("expected 2 full snapshots, got %d", s.numFullSnapshots) + } + + // Insert a record, trigger a snapshot. We should be back to incremental snapshots. + insertSnap() + if s.numFullSnapshots != 2 { + t.Fatalf("expected 2 full snapshots, got %d", s.numFullSnapshots) + } + + // Modify just the access time, and trigger a snapshot. It should still be + // an incremental snapshot. + lt, err = s.db.DBLastModified() + if err != nil { + t.Fatalf("failed to get last modified time of database: %s", err.Error()) + } + if err := os.Chtimes(s.dbPath, lt.Add(time.Second), time.Time{}); err != nil { + t.Fatalf("failed to change database file times: %s", err.Error()) + } + insertSnap() + if s.numFullSnapshots != 2 { + t.Fatalf("expected 2 full snapshots, got %d", s.numFullSnapshots) + } + + // Just a final check... + if s.numSnapshots.Load() != 6 { + t.Fatalf("expected 6 snapshots in total, got %d", s.numSnapshots.Load()) + } +} + +func Test_SingleNodeDBAppliedIndex_SnapshotRestart(t *testing.T) { + s, ln, _ := mustNewStoreSQLitePath(t) + defer ln.Close() + + // Open the store, ensure DBAppliedIndex is at initial value. + if err := s.Open(); err != nil { + t.Fatalf("failed to open single-node store: %s", err.Error()) + } + defer s.Close(true) + if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { + t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) + } + if _, err := s.WaitForLeader(10 * time.Second); err != nil { + t.Fatalf("Error waiting for leader: %s", err) + } + if got, exp := s.DBAppliedIndex(), uint64(0); exp != got { + t.Fatalf("wrong DB applied index, got: %d, exp %d", got, exp) + } + + // Execute a command, and ensure DBAppliedIndex is updated. + er := executeRequestFromStrings([]string{ + `CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`, + }, false, false) + _, _, err := s.Execute(er) + if err != nil { + t.Fatalf("failed to execute on single node: %s", err.Error()) + } + if got, exp := s.DBAppliedIndex(), uint64(3); exp != got { + t.Fatalf("wrong DB applied index, got: %d, exp %d", got, exp) + } + + // Snapshot the Store. + if err := s.Snapshot(0); err != nil { + t.Fatalf("failed to snapshot store: %s", err.Error()) + } + + // Restart the node, and ensure DBAppliedIndex is set to the correct value even + // with a snapshot in place, and no log entries need to be replayed. + if err := s.Close(true); err != nil { + t.Fatalf("failed to close single-node store: %s", err.Error()) + } + if err := s.Open(); err != nil { + t.Fatalf("failed to open single-node store: %s", err.Error()) + } + defer s.Close(true) + if _, err := s.WaitForLeader(10 * time.Second); err != nil { + t.Fatalf("Error waiting for leader: %s", err) + } + if got, exp := s.DBAppliedIndex(), uint64(3); exp != got { + t.Fatalf("wrong DB applied index after restart, got: %d, exp %d", got, exp) + } +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rqlite-9.3.9/store/store_test.go new/rqlite-9.3.10/store/store_test.go --- old/rqlite-9.3.9/store/store_test.go 2026-01-04 08:09:34.000000000 +0100 +++ new/rqlite-9.3.10/store/store_test.go 2026-01-05 14:44:28.000000000 +0100 @@ -2,7 +2,6 @@ import ( "bytes" - "context" "crypto/rand" "errors" "fmt" @@ -10,7 +9,6 @@ "os" "path/filepath" "strings" - "sync" "testing" "time" @@ -281,59 +279,6 @@ }, 100*time.Millisecond, 2*time.Second) } -func Test_SingleNodeDBAppliedIndex_SnapshotRestart(t *testing.T) { - s, ln, _ := mustNewStoreSQLitePath(t) - defer ln.Close() - - // Open the store, ensure DBAppliedIndex is at initial value. - if err := s.Open(); err != nil { - t.Fatalf("failed to open single-node store: %s", err.Error()) - } - defer s.Close(true) - if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { - t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) - } - if _, err := s.WaitForLeader(10 * time.Second); err != nil { - t.Fatalf("Error waiting for leader: %s", err) - } - if got, exp := s.DBAppliedIndex(), uint64(0); exp != got { - t.Fatalf("wrong DB applied index, got: %d, exp %d", got, exp) - } - - // Execute a command, and ensure DBAppliedIndex is updated. - er := executeRequestFromStrings([]string{ - `CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`, - }, false, false) - _, _, err := s.Execute(er) - if err != nil { - t.Fatalf("failed to execute on single node: %s", err.Error()) - } - if got, exp := s.DBAppliedIndex(), uint64(3); exp != got { - t.Fatalf("wrong DB applied index, got: %d, exp %d", got, exp) - } - - // Snapshot the Store. - if err := s.Snapshot(0); err != nil { - t.Fatalf("failed to snapshot store: %s", err.Error()) - } - - // Restart the node, and ensure DBAppliedIndex is set to the correct value even - // with a snapshot in place, and no log entries need to be replayed. - if err := s.Close(true); err != nil { - t.Fatalf("failed to close single-node store: %s", err.Error()) - } - if err := s.Open(); err != nil { - t.Fatalf("failed to open single-node store: %s", err.Error()) - } - defer s.Close(true) - if _, err := s.WaitForLeader(10 * time.Second); err != nil { - t.Fatalf("Error waiting for leader: %s", err) - } - if got, exp := s.DBAppliedIndex(), uint64(3); exp != got { - t.Fatalf("wrong DB applied index after restart, got: %d, exp %d", got, exp) - } -} - func Test_SingleNode_WaitForCommitIndex(t *testing.T) { s, ln := mustNewStore(t) defer ln.Close() @@ -501,77 +446,6 @@ } } -// Test_SingleNodeSnapshot tests that the Store correctly takes a snapshot -// and recovers from it. -func Test_SingleNodeSnapshot(t *testing.T) { - s, ln := mustNewStore(t) - defer ln.Close() - - if err := s.Open(); err != nil { - t.Fatalf("failed to open single-node store: %s", err.Error()) - } - defer s.Close(true) - if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { - t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) - } - if _, err := s.WaitForLeader(10 * time.Second); err != nil { - t.Fatalf("Error waiting for leader: %s", err) - } - - queries := []string{ - `CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`, - `INSERT INTO foo(id, name) VALUES(1, "fiona")`, - } - _, _, err := s.Execute(executeRequestFromStrings(queries, false, false)) - if err != nil { - t.Fatalf("failed to execute on single node: %s", err.Error()) - } - _, _, _, err = s.Query(queryRequestFromString("SELECT * FROM foo", false, false)) - if err != nil { - t.Fatalf("failed to query single node: %s", err.Error()) - } - - // Snap the node and write to disk. - fsm := NewFSM(s) - f, err := fsm.Snapshot() - if err != nil { - t.Fatalf("failed to snapshot node: %s", err.Error()) - } - - snapDir := t.TempDir() - snapFile, err := os.Create(filepath.Join(snapDir, "snapshot")) - if err != nil { - t.Fatalf("failed to create snapshot file: %s", err.Error()) - } - defer snapFile.Close() - sink := &mockSnapshotSink{snapFile} - if err := f.Persist(sink); err != nil { - t.Fatalf("failed to persist snapshot to disk: %s", err.Error()) - } - - // Check restoration. - snapFile, err = os.Open(filepath.Join(snapDir, "snapshot")) - if err != nil { - t.Fatalf("failed to open snapshot file: %s", err.Error()) - } - defer snapFile.Close() - if err := fsm.Restore(snapFile); err != nil { - t.Fatalf("failed to restore snapshot from disk: %s", err.Error()) - } - - // Ensure database is back in the correct state. - r, _, _, err := s.Query(queryRequestFromString("SELECT * FROM foo", false, false)) - if err != nil { - t.Fatalf("failed to query single node: %s", err.Error()) - } - if exp, got := `["id","name"]`, asJSON(r[0].Columns); exp != got { - t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got) - } - if exp, got := `[[1,"fiona"]]`, asJSON(r[0].Values); exp != got { - t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got) - } -} - // Test_StoreSingleNodeNotOpen tests that various methods called on a // closed Store return ErrNotOpen. func Test_StoreSingleNodeNotOpen(t *testing.T) { @@ -2511,305 +2385,6 @@ } } -func Test_SingleNodeUserSnapshot_CAS(t *testing.T) { - s, ln := mustNewStore(t) - defer ln.Close() - if err := s.Open(); err != nil { - t.Fatalf("failed to open single-node store: %s", err.Error()) - } - defer s.Close(true) - if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { - t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) - } - if _, err := s.WaitForLeader(10 * time.Second); err != nil { - t.Fatalf("Error waiting for leader: %s", err) - } - - mustNoop(s, "1") - if err := s.Snapshot(0); err != nil { - t.Fatalf("failed to snapshot single-node store: %s", err.Error()) - } - - if err := s.snapshotCAS.Begin("snapshot-test"); err != nil { - t.Fatalf("failed to begin snapshot CAS: %s", err.Error()) - } - mustNoop(s, "2") - if err := s.Snapshot(0); err == nil { - t.Fatalf("expected error snapshotting single-node store with CAS") - } - s.snapshotCAS.End() - mustNoop(s, "3") - if err := s.Snapshot(0); err != nil { - t.Fatalf("failed to snapshot single-node store: %s", err.Error()) - } -} - -func Test_SingleNodeUserSnapshot_Sync(t *testing.T) { - s, ln := mustNewStore(t) - defer ln.Close() - if err := s.Open(); err != nil { - t.Fatalf("failed to open single-node store: %s", err.Error()) - } - defer s.Close(true) - if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { - t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) - } - if _, err := s.WaitForLeader(10 * time.Second); err != nil { - t.Fatalf("Error waiting for leader: %s", err) - } - - mustNoop(s, "1") - if err := s.Snapshot(0); err != nil { - t.Fatalf("failed to snapshot single-node store: %s", err.Error()) - } - - // Register a channel, and close it, allowing snapshotting to proceed. - ch := make(chan chan struct{}) - s.RegisterSnapshotSync(ch) - called := false - go func() { - c := <-ch - called = true - close(c) - }() - mustNoop(s, "2") - if err := s.Snapshot(0); err != nil { - t.Fatalf("failed to snapshot single-node store with sync: %s", err.Error()) - } - if !called { - t.Fatalf("expected sync function to be called") - } - - // Register a channel, but don't close it, which should cause a timeout. - mustNoop(s, "3") - if err := s.Snapshot(0); err == nil { - t.Fatalf("snapshotting succeeded, expected failure due to sync timeout") - } -} - -func Test_SingleNode_WALTriggeredSnapshot(t *testing.T) { - s, ln := mustNewStore(t) - defer ln.Close() - s.SnapshotThreshold = 8192 - s.SnapshotInterval = 500 * time.Millisecond - s.SnapshotThresholdWALSize = 4096 - - if err := s.Open(); err != nil { - t.Fatalf("failed to open single-node store: %s", err.Error()) - } - defer s.Close(true) - if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { - t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) - } - if _, err := s.WaitForLeader(10 * time.Second); err != nil { - t.Fatalf("Error waiting for leader: %s", err) - } - er := executeRequestFromString(`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`, - false, false) - _, _, err := s.Execute(er) - if err != nil { - t.Fatalf("failed to execute on single node: %s", err.Error()) - } - nSnaps := stats.Get(numWALSnapshots).String() - - for i := 0; i < 100; i++ { - _, _, err := s.Execute(executeRequestFromString(`INSERT INTO foo(name) VALUES("fiona")`, false, false)) - if err != nil { - t.Fatalf("failed to execute INSERT on single node: %s", err.Error()) - } - } - - // Ensure WAL-triggered snapshots take place. - f := func() bool { - return stats.Get(numWALSnapshots).String() != nSnaps - } - testPoll(t, f, 100*time.Millisecond, 2*time.Second) - - // Sanity-check the contents of the Store. There should be two - // files -- a SQLite database file, and a directory named after - // the most recent snapshot. This basically checks that reaping - // is working, as it can be tricky on Windows due to stricter - // file deletion rules. - time.Sleep(5 * time.Second) // Tricky to know when all snapshots are done. Just wait. - snaps, err := s.snapshotStore.List() - if err != nil { - t.Fatalf("failed to list snapshots: %s", err.Error()) - } - if len(snaps) != 1 { - t.Fatalf("wrong number of snapshots: %d", len(snaps)) - } - snapshotDir := filepath.Join(s.raftDir, snapshotsDirName) - files, err := os.ReadDir(snapshotDir) - if err != nil { - t.Fatalf("failed to read snapshot store dir: %s", err.Error()) - } - if len(files) != 2 { - t.Fatalf("wrong number of snapshot store files: %d", len(files)) - } - for _, f := range files { - if !strings.Contains(f.Name(), snaps[0].ID) { - t.Fatalf("wrong snapshot store file: %s", f.Name()) - } - } -} - -func Test_SingleNode_SnapshotFail_Blocked(t *testing.T) { - s, ln := mustNewStore(t) - defer ln.Close() - - s.SnapshotThreshold = 8192 - s.SnapshotInterval = time.Hour - s.NoSnapshotOnClose = true - if err := s.Open(); err != nil { - t.Fatalf("failed to open single-node store: %s", err.Error()) - } - defer s.Close(true) - if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { - t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) - } - if _, err := s.WaitForLeader(10 * time.Second); err != nil { - t.Fatalf("Error waiting for leader: %s", err) - } - er := executeRequestFromString(`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`, - false, false) - _, _, err := s.Execute(er) - if err != nil { - t.Fatalf("failed to execute on single node: %s", err.Error()) - } - - er = executeRequestFromString(`INSERT INTO foo(name) VALUES("fiona")`, false, false) - _, _, err = s.Execute(er) - if err != nil { - t.Fatalf("failed to execute on single node: %s", err.Error()) - } - - go func() { - qr := queryRequestFromString("SELECT * FROM foo", false, false) - qr.GetRequest().Statements[0].ForceStall = true - s.Query(qr) - }() - - time.Sleep(2 * time.Second) - er = executeRequestFromString(`INSERT INTO foo(name) VALUES("bob")`, false, false) - _, _, err = s.Execute(er) - if err != nil { - t.Fatalf("failed to execute on single node: %s", err.Error()) - } - - if err := s.Snapshot(0); err == nil { - t.Fatalf("expected error snapshotting single-node store with stalled query") - } -} - -// Test_SingleNode_SnapshotFail_Blocked_Retry tests that a snapshot operation -// that requires a forced checkpoint and truncation does succeed once the -// blocking query unblocks. -func Test_SingleNode_SnapshotFail_Blocked_Retry(t *testing.T) { - s, ln := mustNewStore(t) - defer ln.Close() - - s.SnapshotThreshold = 8192 - s.SnapshotInterval = time.Hour - s.NoSnapshotOnClose = true - if err := s.Open(); err != nil { - t.Fatalf("failed to open single-node store: %s", err.Error()) - } - defer s.Close(true) - if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { - t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) - } - if _, err := s.WaitForLeader(10 * time.Second); err != nil { - t.Fatalf("Error waiting for leader: %s", err) - } - er := executeRequestFromString(`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`, - false, false) - _, _, err := s.Execute(er) - if err != nil { - t.Fatalf("failed to execute on single node: %s", err.Error()) - } - - er = executeRequestFromString(`INSERT INTO foo(name) VALUES("fiona")`, false, false) - _, _, err = s.Execute(er) - if err != nil { - t.Fatalf("failed to execute on single node: %s", err.Error()) - } - - ctx, cancelFunc := context.WithCancel(context.Background()) - go func() { - qr := queryRequestFromString("SELECT * FROM foo", false, false) - qr.GetRequest().Statements[0].ForceStall = true - - blockingDB, err := db.Open(s.dbPath, false, true) - if err != nil { - t.Errorf("failed to open blocking DB connection: %s", err.Error()) - } - defer blockingDB.Close() - - _, err = blockingDB.QueryWithContext(ctx, qr.GetRequest(), false) - if err != nil { - t.Errorf("failed to execute stalled query on blocking DB connection: %s", err.Error()) - } - }() - time.Sleep(1 * time.Second) - - success := false - var wg sync.WaitGroup - wg.Go(func() { - if err := s.Snapshot(0); err != nil { - t.Errorf("failed to snapshot single-node store with released stalled query: %s", err.Error()) - } else { - success = true - } - }) - time.Sleep(1 * time.Second) - cancelFunc() - wg.Wait() - if !success { - t.Fatalf("expected snapshot to succeed after blocking query released") - } - - // Again, this time with a persistent snapshot. - er = executeRequestFromString(`INSERT INTO foo(name) VALUES("fiona")`, false, false) - _, _, err = s.Execute(er) - if err != nil { - t.Fatalf("failed to execute on single node: %s", err.Error()) - } - - ctx, cancelFunc = context.WithCancel(context.Background()) - go func() { - qr := queryRequestFromString("SELECT * FROM foo", false, false) - qr.GetRequest().Statements[0].ForceStall = true - - blockingDB, err := db.Open(s.dbPath, false, true) - if err != nil { - t.Errorf("failed to open blocking DB connection: %s", err.Error()) - } - defer blockingDB.Close() - - _, err = blockingDB.QueryWithContext(ctx, qr.GetRequest(), false) - if err != nil { - t.Errorf("failed to execute stalled query on blocking DB connection: %s", err.Error()) - } - }() - time.Sleep(1 * time.Second) - - success = false - var wg2 sync.WaitGroup - wg2.Go(func() { - if err := s.Snapshot(0); err != nil { - t.Errorf("failed to snapshot single-node store with second released stalled query: %s", err.Error()) - } else { - success = true - } - }) - time.Sleep(1 * time.Second) - cancelFunc() - wg2.Wait() - if !success { - t.Fatalf("expected snapshot to succeed after blocking query released") - } -} - func Test_OpenStoreSingleNode_OptimizeTimes(t *testing.T) { s0, ln0 := mustNewStore(t) defer s0.Close(true) @@ -2850,185 +2425,6 @@ } } -func Test_SingleNode_SnapshotWithAutoOptimize_Stress(t *testing.T) { - s, ln := mustNewStore(t) - defer ln.Close() - s.SnapshotThreshold = 50 - s.SnapshotInterval = 100 * time.Millisecond - s.AutoOptimizeInterval = 500 * time.Millisecond - - if err := s.Open(); err != nil { - t.Fatalf("failed to open single-node store: %s", err.Error()) - } - defer s.Close(true) - if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { - t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) - } - if _, err := s.WaitForLeader(10 * time.Second); err != nil { - t.Fatalf("Error waiting for leader: %s", err) - } - - // Create a table - er := executeRequestFromString(`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`, - false, false) - _, _, err := s.Execute(er) - if err != nil { - t.Fatalf("failed to execute on single node: %s", err.Error()) - } - - // Create an index on name - er = executeRequestFromString(`CREATE INDEX foo_name ON foo(name)`, false, false) - _, _, err = s.Execute(er) - if err != nil { - t.Fatalf("failed to execute on single node: %s", err.Error()) - } - - // Insert a bunch of data concurrently, putting some load on the Store. - var wg sync.WaitGroup - wg.Add(5) - insertFn := func() { - defer wg.Done() - for i := 0; i < 500; i++ { - _, _, err := s.Execute(executeRequestFromString(fmt.Sprintf(`INSERT INTO foo(name) VALUES("%s")`, random.String()), false, false)) - if err != nil { - t.Errorf("failed to execute INSERT on single node: %s", err.Error()) - } - } - } - for i := 0; i < 5; i++ { - go insertFn() - } - wg.Wait() - - // Query the data, make sure it looks good after all this. - qr := queryRequestFromString("SELECT COUNT(*) FROM foo", false, true) - qr.Level = proto.ConsistencyLevel_STRONG - r, _, _, err := s.Query(qr) - if err != nil { - t.Fatalf("failed to query single node: %s", err.Error()) - } - if exp, got := `[{"columns":["COUNT(*)"],"types":["integer"],"values":[[2500]]}]`, asJSON(r); exp != got { - t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got) - } - - // Restart the Store, make sure all still looks good. - if err := s.Close(true); err != nil { - t.Fatalf("failed to close store: %s", err.Error()) - } - if err := s.Open(); err != nil { - t.Fatalf("failed to open store: %s", err.Error()) - } - defer s.Close(true) - if _, err := s.WaitForLeader(10 * time.Second); err != nil { - t.Fatalf("Error waiting for leader: %s", err) - } - r, _, _, err = s.Query(qr) - if err != nil { - t.Fatalf("failed to query single node: %s", err.Error()) - } - if exp, got := `[{"columns":["COUNT(*)"],"types":["integer"],"values":[[2500]]}]`, asJSON(r); exp != got { - t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got) - } -} - -// Test_SingleNode_DatabaseFileModified tests that a full snapshot is taken -// when the underlying database file is modified by some process external -// to the Store. Such changes are officially unsupported, but if the Store -// detects such a change, it will take a full snapshot to ensure the Snapshot -// remains consistent. -func Test_SingleNode_DatabaseFileModified(t *testing.T) { - s, ln := mustNewStore(t) - defer ln.Close() - if err := s.Open(); err != nil { - t.Fatalf("failed to open single-node store: %s", err.Error()) - } - defer s.Close(true) - - if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil { - t.Fatalf("failed to bootstrap single-node store: %s", err.Error()) - } - if _, err := s.WaitForLeader(10 * time.Second); err != nil { - t.Fatalf("Error waiting for leader: %s", err) - } - - // Insert a record and trigger a snapshot to get a full snapshot. - er := executeRequestFromString(`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`, - false, false) - _, _, err := s.Execute(er) - if err != nil { - t.Fatalf("failed to execute on single node: %s", err.Error()) - } - if err := s.Snapshot(0); err != nil { - t.Fatalf("failed to snapshot single-node store: %s", err.Error()) - } - if s.numFullSnapshots != 1 { - t.Fatalf("expected 1 full snapshot, got %d", s.numFullSnapshots) - } - - insertSnap := func() { - t.Helper() - _, _, err := s.Execute(executeRequestFromString(`INSERT INTO foo(name) VALUES("fiona")`, false, false)) - if err != nil { - t.Fatalf("failed to execute INSERT on single node: %s", err.Error()) - } - if err := s.Snapshot(0); err != nil { - t.Fatalf("failed to snapshot single-node store: %s", err.Error()) - } - } - - // Insert a record, trigger a snapshot. It should be an incremental snapshot. - insertSnap() - if s.numFullSnapshots != 1 { - t.Fatalf("expected 1 full snapshot, got %d", s.numFullSnapshots) - } - - // Insert a record, trigger a snapshot. It shouldn't be a full snapshot. - insertSnap() - if s.numFullSnapshots != 1 { - t.Fatalf("expected 1 full snapshot, got %d", s.numFullSnapshots) - } - - lt, err := s.db.DBLastModified() - if err != nil { - t.Fatalf("failed to get last modified time of database: %s", err.Error()) - } - - // Touch the database file to make it newer than Store's record of last - // modified time and then trigger a snapshot. It should be a full snapshot. - if err := os.Chtimes(s.dbPath, time.Time{}, lt.Add(time.Second)); err != nil { - t.Fatalf("failed to change database file times: %s", err.Error()) - } - insertSnap() - if s.numFullSnapshots != 2 { - t.Fatalf("expected 2 full snapshots, got %d", s.numFullSnapshots) - } - - // Insert a record, trigger a snapshot. We should be back to incremental snapshots. - insertSnap() - if s.numFullSnapshots != 2 { - t.Fatalf("expected 2 full snapshots, got %d", s.numFullSnapshots) - } - - // Modify just the access time, and trigger a snapshot. It should still be - // an incremental snapshot. - lt, err = s.db.DBLastModified() - if err != nil { - t.Fatalf("failed to get last modified time of database: %s", err.Error()) - } - if err := os.Chtimes(s.dbPath, lt.Add(time.Second), time.Time{}); err != nil { - t.Fatalf("failed to change database file times: %s", err.Error()) - } - insertSnap() - if s.numFullSnapshots != 2 { - t.Fatalf("expected 2 full snapshots, got %d", s.numFullSnapshots) - } - - // Just a final check... - if s.numSnapshots.Load() != 6 { - t.Fatalf("expected 6 snapshots in total, got %d", s.numSnapshots.Load()) - } -} - func Test_SingleNodeSelfJoinNoChangeOK(t *testing.T) { s0, ln0 := mustNewStore(t) defer ln0.Close()
