This is an automated email from the ASF dual-hosted git repository.

linkinstar pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/answer.git

commit 6df436c7183fdbaccdc9b7d6659001d51f0f4ef8
Author: Sonui <[email protected]>
AuthorDate: Mon May 26 00:27:06 2025 +0800

    feat(tests): add comprehensive tests for KV storage plugin
---
 plugin/plugin_test/kv_storage_test.go  | 412 +++++++++++++++++++++++++++++++++
 plugin/plugin_test/plugin_main_test.go | 207 +++++++++++++++++
 2 files changed, 619 insertions(+)

diff --git a/plugin/plugin_test/kv_storage_test.go 
b/plugin/plugin_test/kv_storage_test.go
new file mode 100644
index 00000000..0dcd5b46
--- /dev/null
+++ b/plugin/plugin_test/kv_storage_test.go
@@ -0,0 +1,412 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package plugin_test
+
+import (
+       "context"
+       "fmt"
+       "math/rand"
+       "sync"
+       "testing"
+       "time"
+
+       "github.com/apache/answer/plugin"
+       "github.com/segmentfault/pacman/log"
+       _ "modernc.org/sqlite"
+)
+
+var (
+       testPlugin *TestKVStoragePlugin
+)
+
+// Helper functions for testing
+func mustSet(t *testing.T, kv *plugin.KVOperator, ctx context.Context, group, 
key, value string) {
+       if err := kv.Set(ctx, plugin.KVParams{Group: group, Key: key, Value: 
value}); err != nil {
+               t.Fatalf("Failed to set %s/%s: %v", group, key, err)
+       }
+}
+
+func mustGet(t *testing.T, kv *plugin.KVOperator, ctx context.Context, group, 
key, expected string) {
+       val, err := kv.Get(ctx, plugin.KVParams{Group: group, Key: key})
+       if err != nil {
+               t.Fatalf("Failed to get %s/%s: %v", group, key, err)
+       }
+       if val != expected {
+               t.Errorf("Expected '%s' for %s/%s, got '%s'", expected, group, 
key, val)
+       }
+}
+
+func mustDel(t *testing.T, kv *plugin.KVOperator, ctx context.Context, group, 
key string) {
+       if err := kv.Del(ctx, plugin.KVParams{Group: group, Key: key}); err != 
nil {
+               t.Fatalf("Failed to delete %s/%s: %v", group, key, err)
+       }
+}
+
+func assertNotFound(t *testing.T, kv *plugin.KVOperator, ctx context.Context, 
group, key string) {
+       val, err := kv.Get(ctx, plugin.KVParams{Group: group, Key: key})
+       if err != plugin.ErrKVKeyNotFound {
+               t.Errorf("Expected ErrKVKeyNotFound for %s/%s, got: %v", group, 
key, err)
+       }
+       if val != "" {
+               t.Errorf("Expected empty value for %s/%s, got: '%s'", group, 
key, val)
+       }
+}
+
+func assertError(t *testing.T, err error, expected error, msg string) {
+       if err != expected {
+               t.Errorf("%s: expected %v, got %v", msg, expected, err)
+       }
+}
+
+// TestKVStoragePlugin implements KVStorage interface for testing
+type TestKVStoragePlugin struct {
+       operator *plugin.KVOperator
+}
+
+// Info returns plugin information
+func (p *TestKVStoragePlugin) Info() plugin.Info {
+       return plugin.Info{
+               Name:        plugin.MakeTranslator("test_kv_storage_name"),
+               SlugName:    "test_kv_storage",
+               Description: plugin.MakeTranslator("test_kv_storage_desc"),
+               Author:      "Answer Team",
+               Version:     "1.0.0",
+               Link:        "https://github.com/apache/answer";,
+       }
+}
+
+// SetOperator sets KV operator
+func (p *TestKVStoragePlugin) SetOperator(operator *plugin.KVOperator) {
+       p.operator = operator
+}
+
+// setupTestEnvironment sets up test environment
+func setupTestEnvironment() {
+       // Initialize only once
+       if testPlugin != nil {
+               return
+       }
+
+       // Create and register test plugin
+       testPlugin = &TestKVStoragePlugin{}
+       plugin.Register(testPlugin)
+
+       // Enable plugin
+       plugin.StatusManager.Enable("test_kv_storage", true)
+
+       // Initialize plugin data, refer to plugin_common_service.go 
implementation
+       _ = plugin.CallKVStorage(func(k plugin.KVStorage) error {
+               k.SetOperator(plugin.NewKVOperator(
+                       testDataSource.DB,
+                       testDataSource.Cache,
+                       k.Info().SlugName,
+               ))
+               return nil
+       })
+}
+
+// Test basic operations including CRUD and edge cases
+func TestBasicOperations(t *testing.T) {
+       setupTestEnvironment()
+       kv := testPlugin.operator
+       ctx := context.Background()
+
+       t.Run("BasicCRUD", func(t *testing.T) {
+               // Set/Get
+               mustSet(t, kv, ctx, "group1", "key1", "value1")
+               mustGet(t, kv, ctx, "group1", "key1", "value1")
+
+               // Update
+               mustSet(t, kv, ctx, "group1", "key1", "new_value")
+               mustGet(t, kv, ctx, "group1", "key1", "new_value")
+
+               // Delete
+               mustDel(t, kv, ctx, "group1", "key1")
+               assertNotFound(t, kv, ctx, "group1", "key1")
+
+               // Group operation
+               mustSet(t, kv, ctx, "group1", "key2", "value2")
+               mustSet(t, kv, ctx, "group1", "key3", "value3")
+               groupData, err := kv.GetByGroup(ctx, plugin.KVParams{Group: 
"group1", Page: 1, PageSize: 10})
+               if err != nil {
+                       t.Fatalf("Failed to get group data: %v", err)
+               }
+
+               // the groupData should only have key2 and key3 because key1 is 
deleted
+               if len(groupData) != 2 {
+                       t.Errorf("Expected 2 items, got %d", len(groupData))
+               }
+               if groupData["key2"] != "value2" || groupData["key3"] != 
"value3" {
+                       t.Errorf("Unexpected group data: %v", groupData)
+               }
+       })
+
+       t.Run("EdgeCases", func(t *testing.T) {
+               // Empty key
+               err := kv.Set(ctx, plugin.KVParams{Group: "group", Key: "", 
Value: "value"})
+               assertError(t, err, plugin.ErrKVKeyEmpty, "Empty key test")
+
+               // Empty group query
+               _, err = kv.GetByGroup(ctx, plugin.KVParams{Group: "", Page: 1, 
PageSize: 10})
+               assertError(t, err, plugin.ErrKVGroupEmpty, "Empty group test")
+
+               // Non-existent key
+               assertNotFound(t, kv, ctx, "non_exist_group", "non_exist_key")
+
+               // Cache penetration protection
+               key := fmt.Sprintf("non_exist_key_%d", time.Now().UnixNano())
+               assertNotFound(t, kv, ctx, "cache_penetration", key)
+       })
+
+       t.Run("CacheConsistency", func(t *testing.T) {
+               mustSet(t, kv, ctx, "cache_group", "cache_key", "cache_value")
+               mustGet(t, kv, ctx, "cache_group", "cache_key", "cache_value")
+
+               // Update and verify immediate consistency
+               mustSet(t, kv, ctx, "cache_group", "cache_key", "updated_value")
+               mustGet(t, kv, ctx, "cache_group", "cache_key", "updated_value")
+       })
+}
+
+// Test transactions including rollback and nested transactions
+func TestTransactions(t *testing.T) {
+       setupTestEnvironment()
+       kv := testPlugin.operator
+       ctx := context.Background()
+
+       t.Run("SuccessfulTransaction", func(t *testing.T) {
+               err := kv.Tx(ctx, func(ctx context.Context, txKv 
*plugin.KVOperator) error {
+                       if err := txKv.Set(ctx, plugin.KVParams{Group: 
"tx_group", Key: "tx_key1", Value: "tx_value1"}); err != nil {
+                               return err
+                       }
+                       if err := txKv.Set(ctx, plugin.KVParams{Group: 
"tx_group", Key: "tx_key2", Value: "tx_value2"}); err != nil {
+                               return err
+                       }
+                       return nil
+               })
+               if err != nil {
+                       t.Fatalf("Successful transaction failed: %v", err)
+               }
+
+               mustGet(t, kv, ctx, "tx_group", "tx_key1", "tx_value1")
+               mustGet(t, kv, ctx, "tx_group", "tx_key2", "tx_value2")
+       })
+
+       t.Run("TransactionRollback", func(t *testing.T) {
+               err := kv.Tx(ctx, func(ctx context.Context, txKv 
*plugin.KVOperator) error {
+                       if err := txKv.Set(ctx, plugin.KVParams{Group: 
"tx_group", Key: "tx_key3", Value: "tx_value3"}); err != nil {
+                               return err
+                       }
+                       return fmt.Errorf("mock error")
+               })
+               if err == nil {
+                       t.Error("Expected transaction to fail but it succeeded")
+               }
+
+               assertNotFound(t, kv, ctx, "tx_group", "tx_key3")
+       })
+
+       t.Run("NestedTransactions", func(t *testing.T) {
+               err := kv.Tx(ctx, func(ctx context.Context, txKv 
*plugin.KVOperator) error {
+                       if err := txKv.Set(ctx, plugin.KVParams{Group: 
"nested", Key: "key1", Value: "value1"}); err != nil {
+                               return err
+                       }
+
+                       return txKv.Tx(ctx, func(ctx context.Context, nestedKv 
*plugin.KVOperator) error {
+                               if err := nestedKv.Set(ctx, 
plugin.KVParams{Group: "nested", Key: "key2", Value: "value2"}); err != nil {
+                                       return err
+                               }
+                               return fmt.Errorf("mock nested error")
+                       })
+               })
+               if err == nil {
+                       t.Error("Expected nested transaction to fail but it 
succeeded")
+               }
+
+               // Verify outer transaction also rolled back
+               assertNotFound(t, kv, ctx, "nested", "key1")
+               assertNotFound(t, kv, ctx, "nested", "key2")
+       })
+}
+
+// Test pagination in GetByGroup
+func TestPagination(t *testing.T) {
+       setupTestEnvironment()
+       kv := testPlugin.operator
+       ctx := context.Background()
+       totalItems := 25
+
+       for i := range totalItems {
+               mustSet(t, kv, ctx, "pagination", fmt.Sprintf("key%d", i), 
fmt.Sprintf("value%d", i))
+       }
+
+       // Test pagination
+       page1, err := kv.GetByGroup(ctx, plugin.KVParams{Group: "pagination", 
Page: 1, PageSize: 10})
+       if err != nil {
+               t.Fatalf("Failed to get page 1: %v", err)
+       }
+       if len(page1) != 10 {
+               t.Errorf("Page 1: expected 10 items, got %d", len(page1))
+       }
+
+       page2, err := kv.GetByGroup(ctx, plugin.KVParams{Group: "pagination", 
Page: 2, PageSize: 10})
+       if err != nil {
+               t.Fatalf("Failed to get page 2: %v", err)
+       }
+       if len(page2) != 10 {
+               t.Errorf("Page 2: expected 10 items, got %d", len(page2))
+       }
+
+       page3, err := kv.GetByGroup(ctx, plugin.KVParams{Group: "pagination", 
Page: 3, PageSize: 10})
+       if err != nil {
+               t.Fatalf("Failed to get page 3: %v", err)
+       }
+       if len(page3) != 5 {
+               t.Errorf("Page 3: expected 5 items, got %d", len(page3))
+       }
+
+       // Verify different keys on different pages
+       for i := range 10 {
+               key := fmt.Sprintf("key%d", i)
+               if _, ok := page1[key]; !ok {
+                       t.Errorf("Pagination test failed, key %s should be on 
page 1", key)
+               }
+       }
+       for i := range 10 {
+               key := fmt.Sprintf("key%d", i+10)
+               if _, ok := page2[key]; !ok {
+                       t.Errorf("Pagination test failed, key %s should be on 
page 2", key)
+               }
+       }
+}
+
+// Test concurrent operations and performance
+func TestConcurrency(t *testing.T) {
+       setupTestEnvironment()
+       kv := testPlugin.operator
+       ctx := context.Background()
+
+       t.Run("BasicConcurrency", func(t *testing.T) {
+               parallel := 10
+               var wg sync.WaitGroup
+               wg.Add(parallel)
+
+               for i := range parallel {
+                       go func(index int) {
+                               defer wg.Done()
+                               time.Sleep(time.Duration(rand.Intn(200)) * 
time.Millisecond)
+                               mustSet(t, kv, ctx, "concurrent", 
fmt.Sprintf("key%d", index), "value")
+                       }(i)
+               }
+               wg.Wait()
+
+               // Verify results
+               wg.Add(parallel)
+               for i := range parallel {
+                       go func(index int) {
+                               defer wg.Done()
+                               time.Sleep(time.Duration(rand.Intn(200)) * 
time.Millisecond)
+                               mustGet(t, kv, ctx, "concurrent", 
fmt.Sprintf("key%d", index), "value")
+                       }(i)
+               }
+               wg.Wait()
+       })
+
+       t.Run("StressTest", func(t *testing.T) {
+               if testing.Short() {
+                       t.Skip("Skipping stress test in short mode")
+               }
+
+               totalOps := 1000
+               workerCount := 20
+               prefix := "stress_test"
+               opsPerWorker := totalOps / workerCount
+
+               log.Info("Starting KV storage stress test...")
+               startTime := time.Now()
+
+               // Concurrent write test
+               var wg sync.WaitGroup
+               errorCount := int64(0)
+
+               for w := range workerCount {
+                       wg.Add(1)
+                       go func(workerID int) {
+                               defer wg.Done()
+                               startIdx := workerID * opsPerWorker
+
+                               for i := range opsPerWorker {
+                                       i := startIdx + i
+                                       err := kv.Set(ctx, plugin.KVParams{
+                                               Group: prefix,
+                                               Key:   fmt.Sprintf("key%d", i),
+                                               Value: fmt.Sprintf("value%d", 
i),
+                                       })
+                                       if err != nil {
+                                               log.Warnf("Write error: %v", 
err)
+                                               errorCount++
+                                       }
+                               }
+                       }(w)
+               }
+               wg.Wait()
+
+               writeTime := time.Since(startTime)
+
+               // Verify data integrity
+               groupData, err := kv.GetByGroup(ctx, plugin.KVParams{Group: 
prefix, Page: 1, PageSize: totalOps})
+               if err != nil {
+                       t.Fatalf("Failed to verify data: %v", err)
+               }
+               if len(groupData) != totalOps {
+                       t.Errorf("Data loss: expected %d items, got %d", 
totalOps, len(groupData))
+               }
+
+               // Concurrent read test
+               startTime = time.Now()
+               readErrors := int64(0)
+
+               wg.Add(workerCount)
+               for range workerCount {
+                       go func() {
+                               defer wg.Done()
+                               for range opsPerWorker {
+                                       keyIdx := rand.Intn(totalOps)
+                                       key := fmt.Sprintf("key%d", keyIdx)
+                                       expected := fmt.Sprintf("value%d", 
keyIdx)
+
+                                       val, err := kv.Get(ctx, 
plugin.KVParams{Group: prefix, Key: key})
+                                       if err != nil {
+                                               readErrors++
+                                       } else if val != expected {
+                                               t.Errorf("Data inconsistency: 
key=%s, expected=%s, got=%s", key, expected, val)
+                                       }
+                               }
+                       }()
+               }
+               wg.Wait()
+
+               readTime := time.Since(startTime)
+
+               log.Infof("Stress test completed:")
+               log.Infof("  Write: %d ops in %v (%.1f ops/sec), %d errors", 
totalOps, writeTime, float64(totalOps)/writeTime.Seconds(), errorCount)
+               log.Infof("  Read: %d ops in %v (%.1f ops/sec), %d errors", 
totalOps, readTime, float64(totalOps)/readTime.Seconds(), readErrors)
+       })
+}
diff --git a/plugin/plugin_test/plugin_main_test.go 
b/plugin/plugin_test/plugin_main_test.go
new file mode 100644
index 00000000..4f92bebe
--- /dev/null
+++ b/plugin/plugin_test/plugin_main_test.go
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package plugin_test
+
+import (
+       "context"
+       "database/sql"
+       "fmt"
+       "os"
+       "path/filepath"
+       "testing"
+       "time"
+
+       "github.com/apache/answer/internal/base/data"
+       "github.com/apache/answer/internal/migrations"
+       "github.com/ory/dockertest/v3"
+       "github.com/ory/dockertest/v3/docker"
+       "github.com/segmentfault/pacman/cache"
+       "github.com/segmentfault/pacman/log"
+       "xorm.io/xorm"
+       "xorm.io/xorm/schemas"
+)
+
+var (
+       mysqlDBSetting = TestDBSetting{
+               Driver:       string(schemas.MYSQL),
+               ImageName:    "mariadb",
+               ImageVersion: "10.4.7",
+               ENV:          []string{"MYSQL_ROOT_PASSWORD=root", 
"MYSQL_DATABASE=answer", "MYSQL_ROOT_HOST=%"},
+               PortID:       "3306/tcp",
+               Connection:   "root:root@(localhost:%s)/answer?parseTime=true", 
// port is not fixed, it will be got by port id
+       }
+       postgresDBSetting = TestDBSetting{
+               Driver:       string(schemas.POSTGRES),
+               ImageName:    "postgres",
+               ImageVersion: "14",
+               ENV:          []string{"POSTGRES_USER=root", 
"POSTGRES_PASSWORD=root", "POSTGRES_DB=answer", "LISTEN_ADDRESSES='*'"},
+               PortID:       "5432/tcp",
+               Connection:   "host=localhost port=%s user=root password=root 
dbname=answer sslmode=disable",
+       }
+       sqlite3DBSetting = TestDBSetting{
+               Driver:     string(schemas.SQLITE),
+               Connection: filepath.Join(os.TempDir(), "answer-test-data.db"),
+       }
+       dbSettingMapping = map[string]TestDBSetting{
+               mysqlDBSetting.Driver:    mysqlDBSetting,
+               sqlite3DBSetting.Driver:  sqlite3DBSetting,
+               postgresDBSetting.Driver: postgresDBSetting,
+       }
+       // after all test down will execute tearDown function to clean-up
+       tearDown func()
+       // testDataSource used for repo testing
+       testDataSource *data.Data
+       testCache      cache.Cache
+)
+
+func TestMain(t *testing.M) {
+       dbSetting, ok := dbSettingMapping[os.Getenv("TEST_DB_DRIVER")]
+       if !ok {
+               // Use sqlite3 to test.
+               dbSetting = dbSettingMapping[string(schemas.SQLITE)]
+       }
+       if dbSetting.Driver == string(schemas.SQLITE) {
+               os.RemoveAll(dbSetting.Connection)
+       }
+
+       defer func() {
+               if tearDown != nil {
+                       tearDown()
+               }
+       }()
+       if err := initTestDataSource(dbSetting); err != nil {
+               panic(err)
+       }
+       log.Info("init test database successfully")
+
+       if ret := t.Run(); ret != 0 {
+               panic(ret)
+       }
+}
+
+type TestDBSetting struct {
+       Driver       string
+       ImageName    string
+       ImageVersion string
+       ENV          []string
+       PortID       string
+       Connection   string
+}
+
+func initTestDataSource(dbSetting TestDBSetting) error {
+       connection, imageCleanUp, err := initDatabaseImage(dbSetting)
+       if err != nil {
+               return err
+       }
+       dbSetting.Connection = connection
+
+       dbEngine, err := initDatabase(dbSetting)
+       if err != nil {
+               return err
+       }
+
+       newCache, err := initCache()
+       if err != nil {
+               return err
+       }
+
+       newData, dbCleanUp, err := data.NewData(dbEngine, newCache)
+       if err != nil {
+               return err
+       }
+       testDataSource = newData
+       testCache = newCache
+
+       tearDown = func() {
+               dbCleanUp()
+               log.Info("cleanup test database successfully")
+               imageCleanUp()
+               log.Info("cleanup test database image successfully")
+       }
+       return nil
+}
+
+func initDatabaseImage(dbSetting TestDBSetting) (connection string, cleanup 
func(), err error) {
+       // sqlite3 don't need to set up image
+       if dbSetting.Driver == string(schemas.SQLITE) {
+               return dbSetting.Connection, func() {
+                       log.Info("remove database", dbSetting.Connection)
+                       err = os.Remove(dbSetting.Connection)
+                       if err != nil {
+                               log.Error(err)
+                       }
+               }, nil
+       }
+       pool, err := dockertest.NewPool("")
+       pool.MaxWait = time.Minute * 5
+       if err != nil {
+               return "", nil, fmt.Errorf("could not connect to docker: %s", 
err)
+       }
+
+       //resource, err := pool.Run(dbSetting.ImageName, 
dbSetting.ImageVersion, dbSetting.ENV)
+       resource, err := pool.RunWithOptions(&dockertest.RunOptions{
+               Repository: dbSetting.ImageName,
+               Tag:        dbSetting.ImageVersion,
+               Env:        dbSetting.ENV,
+       }, func(config *docker.HostConfig) {
+               config.AutoRemove = true
+               config.RestartPolicy = docker.RestartPolicy{Name: "no"}
+       })
+       if err != nil {
+               return "", nil, fmt.Errorf("could not pull resource: %s", err)
+       }
+
+       connection = fmt.Sprintf(dbSetting.Connection, 
resource.GetPort(dbSetting.PortID))
+       if err := pool.Retry(func() error {
+               db, err := sql.Open(dbSetting.Driver, connection)
+               if err != nil {
+                       return err
+               }
+               return db.Ping()
+       }); err != nil {
+               return "", nil, fmt.Errorf("could not connect to database: %s", 
err)
+       }
+       return connection, func() { _ = pool.Purge(resource) }, nil
+}
+
+func initDatabase(dbSetting TestDBSetting) (dbEngine *xorm.Engine, err error) {
+       dataConf := &data.Database{Driver: dbSetting.Driver, Connection: 
dbSetting.Connection}
+       dbEngine, err = data.NewDB(true, dataConf)
+       if err != nil {
+               return nil, fmt.Errorf("connection to database failed: %s", err)
+       }
+       if err := migrations.NewMentor(context.TODO(), dbEngine, 
&migrations.InitNeedUserInputData{
+               Language:      "en_US",
+               SiteName:      "ANSWER",
+               SiteURL:       "http://127.0.0.1:8080/";,
+               ContactEmail:  "[email protected]",
+               AdminName:     "admin",
+               AdminPassword: "admin",
+               AdminEmail:    "[email protected]",
+       }).InitDB(); err != nil {
+               return nil, fmt.Errorf("migrations init database failed: %s", 
err)
+       }
+       return dbEngine, nil
+}
+
+func initCache() (newCache cache.Cache, err error) {
+       newCache, _, err = data.NewCache(&data.CacheConf{})
+       return newCache, err
+}

Reply via email to