This is an automated email from the ASF dual-hosted git repository. LinkinStars pushed a commit to branch fix/2.0.4/user in repository https://gitbox.apache.org/repos/asf/answer.git
commit 671b58737fc7b7c03b405f72e8869a67fe43c81f Author: LinkinStars <[email protected]> AuthorDate: Tue Oct 20 15:11:08 2026 +0800 fix(security): enforce semantic search visibility --- internal/controller/mcp_controller.go | 76 +++++---- internal/repo/vector_search_sync/syncer.go | 53 ++++++- internal/repo/vector_search_sync/syncer_test.go | 118 ++++++++++++++ internal/service/content/question_service.go | 21 ++- .../service/plugin_common/plugin_common_service.go | 18 ++- internal/service/vector_sync/vector_sync.go | 169 +++++++++++++++++++++ internal/service/vector_sync/vector_sync_test.go | 154 +++++++++++++++++++ 7 files changed, 569 insertions(+), 40 deletions(-) diff --git a/internal/controller/mcp_controller.go b/internal/controller/mcp_controller.go index eebb8baf..f5028ec3 100644 --- a/internal/controller/mcp_controller.go +++ b/internal/controller/mcp_controller.go @@ -37,6 +37,7 @@ import ( "github.com/apache/answer/internal/service/siteinfo_common" tagcommonser "github.com/apache/answer/internal/service/tag_common" usercommon "github.com/apache/answer/internal/service/user_common" + "github.com/apache/answer/pkg/uid" "github.com/apache/answer/plugin" "github.com/mark3labs/mcp-go/mcp" "github.com/segmentfault/pacman/log" @@ -417,7 +418,20 @@ func (c *MCPController) MCPSemanticSearchHandler() func(ctx context.Context, req resp := make([]*schema.MCPSemanticSearchResp, 0, len(results)) for _, r := range results { var meta plugin.VectorSearchMetadata - _ = json.Unmarshal([]byte(r.Metadata), &meta) + if err := json.Unmarshal([]byte(r.Metadata), &meta); err != nil || meta.QuestionID == "" { + // Search metadata is not an authorization source. Ignore malformed + // and legacy entries rather than attempting to return their content. + continue + } + + // The vector store can contain stale documents, and may be managed by a + // third-party plugin. Re-check the authoritative database state before + // exposing every search result. + questionID := uid.DeShortID(meta.QuestionID) + question, qErr := c.questioncommon.Info(ctx, questionID, "") + if qErr != nil || !mcpQuestionIsPublic(question) { + continue + } item := &schema.MCPSemanticSearchResp{ ObjectID: r.ObjectID, @@ -432,20 +446,17 @@ func (c *MCPController) MCPSemanticSearchHandler() func(ctx context.Context, req item.Link = fmt.Sprintf("%s/questions/%s", siteGeneral.SiteUrl, meta.QuestionID) } - // Query content from DB using IDs stored in metadata + // Query content from DB using IDs stored in metadata. All related + // objects must still be available and belong to this public question. if r.ObjectType == "question" { - question, qErr := c.questioncommon.Info(ctx, meta.QuestionID, "") - if qErr != nil { - log.Warnf("get question %s for semantic search failed: %v", meta.QuestionID, qErr) - } else { - item.Title = question.Title - item.Content = question.Content - } + item.Title = question.Title + item.Content = question.Content // Fetch answers by ID from metadata for _, a := range meta.Answers { answerEntity, exist, aErr := c.answerRepo.GetAnswer(ctx, a.AnswerID) - if aErr != nil || !exist { + if aErr != nil || !exist || answerEntity.Status != entity.AnswerStatusAvailable || + uid.DeShortID(answerEntity.QuestionID) != questionID { continue } answerItem := &schema.MCPSemanticSearchAnswer{ @@ -455,7 +466,9 @@ func (c *MCPController) MCPSemanticSearchHandler() func(ctx context.Context, req // Fetch comments on this answer from DB for _, ac := range a.Comments { cmt, cExist, cErr := c.commentRepo.GetComment(ctx, ac.CommentID) - if cErr == nil && cExist { + if cErr == nil && cExist && cmt.Status == entity.CommentStatusAvailable && + uid.DeShortID(cmt.ObjectID) == uid.DeShortID(answerEntity.ID) && + uid.DeShortID(cmt.QuestionID) == questionID { answerItem.Comments = append(answerItem.Comments, &schema.MCPSemanticSearchComment{ CommentID: ac.CommentID, Content: cmt.OriginalText, @@ -468,7 +481,9 @@ func (c *MCPController) MCPSemanticSearchHandler() func(ctx context.Context, req // Fetch question comments from DB for _, qc := range meta.Comments { cmt, cExist, cErr := c.commentRepo.GetComment(ctx, qc.CommentID) - if cErr == nil && cExist { + if cErr == nil && cExist && cmt.Status == entity.CommentStatusAvailable && + uid.DeShortID(cmt.ObjectID) == uid.DeShortID(question.ID) && + uid.DeShortID(cmt.QuestionID) == questionID { item.Comments = append(item.Comments, &schema.MCPSemanticSearchComment{ CommentID: qc.CommentID, Content: cmt.OriginalText, @@ -476,30 +491,29 @@ func (c *MCPController) MCPSemanticSearchHandler() func(ctx context.Context, req } } } else if r.ObjectType == "answer" { - // Fetch question title for context - question, qErr := c.questioncommon.Info(ctx, meta.QuestionID, "") - if qErr == nil { - item.Title = question.Title - } + item.Title = question.Title // Fetch answer content from DB - if meta.AnswerID != "" { - answerEntity, exist, aErr := c.answerRepo.GetAnswer(ctx, meta.AnswerID) - if aErr == nil && exist { - item.Content = answerEntity.OriginalText - } - } else if len(meta.Answers) > 0 { - answerEntity, exist, aErr := c.answerRepo.GetAnswer(ctx, meta.Answers[0].AnswerID) - if aErr == nil && exist { - item.Content = answerEntity.OriginalText - } + if meta.AnswerID == "" { + continue + } + answerEntity, exist, aErr := c.answerRepo.GetAnswer(ctx, meta.AnswerID) + if aErr != nil || !exist || answerEntity.Status != entity.AnswerStatusAvailable || + uid.DeShortID(answerEntity.QuestionID) != questionID { + continue } + item.Content = answerEntity.OriginalText // Fetch answer comments from DB - if len(meta.Answers) > 0 { - for _, ac := range meta.Answers[0].Comments { + for _, a := range meta.Answers { + if a.AnswerID != meta.AnswerID { + continue + } + for _, ac := range a.Comments { cmt, cExist, cErr := c.commentRepo.GetComment(ctx, ac.CommentID) - if cErr == nil && cExist { + if cErr == nil && cExist && cmt.Status == entity.CommentStatusAvailable && + uid.DeShortID(cmt.ObjectID) == uid.DeShortID(answerEntity.ID) && + uid.DeShortID(cmt.QuestionID) == questionID { item.Comments = append(item.Comments, &schema.MCPSemanticSearchComment{ CommentID: ac.CommentID, Content: cmt.OriginalText, @@ -507,6 +521,8 @@ func (c *MCPController) MCPSemanticSearchHandler() func(ctx context.Context, req } } } + } else { + continue } resp = append(resp, item) diff --git a/internal/repo/vector_search_sync/syncer.go b/internal/repo/vector_search_sync/syncer.go index be829e8f..5a3ebd33 100644 --- a/internal/repo/vector_search_sync/syncer.go +++ b/internal/repo/vector_search_sync/syncer.go @@ -49,7 +49,13 @@ func (p *PluginSyncer) GetQuestionsPage(ctx context.Context, page, pageSize int) []*plugin.VectorSearchContent, error) { questions := make([]*entity.Question, 0) startNum := (page - 1) * pageSize - err := p.data.DB.Context(ctx).Limit(pageSize, startNum).Find(&questions) + // Vector indexes are shared retrieval infrastructure. Only index content that + // is public to every caller; authorization is still enforced again when a + // result is read from the primary database. + err := p.data.DB.Context(ctx). + Where("show = ? AND status IN (?, ?)", entity.QuestionShow, + entity.QuestionStatusAvailable, entity.QuestionStatusClosed). + OrderBy("id ASC").Limit(pageSize, startNum).Find(&questions) if err != nil { return nil, err } @@ -62,7 +68,12 @@ func (p *PluginSyncer) GetAnswersPage(ctx context.Context, page, pageSize int) ( []*plugin.VectorSearchContent, error) { answers := make([]*entity.Answer, 0) startNum := (page - 1) * pageSize - err := p.data.DB.Context(ctx).Limit(pageSize, startNum).Find(&answers) + err := p.data.DB.Context(ctx). + Join("INNER", "question", "answer.question_id = question.id"). + Where("answer.status = ? AND question.show = ? AND question.status IN (?, ?)", + entity.AnswerStatusAvailable, entity.QuestionShow, + entity.QuestionStatusAvailable, entity.QuestionStatusClosed). + OrderBy("answer.id ASC").Limit(pageSize, startNum).Find(&answers) if err != nil { return nil, err } @@ -74,6 +85,9 @@ func (p *PluginSyncer) buildQuestionContents(ctx context.Context, questions []*e []*plugin.VectorSearchContent, error) { result := make([]*plugin.VectorSearchContent, 0, len(questions)) for _, q := range questions { + if !isPublicQuestion(q) { + continue + } meta := plugin.VectorSearchMetadata{ QuestionID: uid.DeShortID(q.ID), } @@ -83,11 +97,15 @@ func (p *PluginSyncer) buildQuestionContents(ctx context.Context, questions []*e // Get answers for this question answers := make([]*entity.Answer, 0) - err := p.data.DB.Context(ctx).Where("question_id = ?", q.ID).Find(&answers) + err := p.data.DB.Context(ctx).Where("question_id = ? AND status = ?", q.ID, + entity.AnswerStatusAvailable).Find(&answers) if err != nil { log.Warnf("get answers for question %s failed: %v", q.ID, err) } else { for _, a := range answers { + if a.Status != entity.AnswerStatusAvailable { + continue + } parts = append(parts, fmt.Sprintf("Answer: %s", a.OriginalText)) answerMeta := plugin.VectorSearchMetadataAnswer{ AnswerID: uid.DeShortID(a.ID), @@ -95,12 +113,16 @@ func (p *PluginSyncer) buildQuestionContents(ctx context.Context, questions []*e // Get comments on this answer answerComments := make([]*entity.Comment, 0) - err := p.data.DB.Context(ctx).Where("object_id = ?", a.ID). + err := p.data.DB.Context(ctx).Where("object_id = ? AND status = ?", a.ID, + entity.CommentStatusAvailable). OrderBy("created_at ASC").Limit(50).Find(&answerComments) if err != nil { log.Warnf("get comments for answer %s failed: %v", a.ID, err) } else { for _, c := range answerComments { + if c.Status != entity.CommentStatusAvailable { + continue + } parts = append(parts, fmt.Sprintf("Comment on answer: %s", c.OriginalText)) answerMeta.Comments = append(answerMeta.Comments, plugin.VectorSearchMetadataComment{ CommentID: uid.DeShortID(c.ID), @@ -113,12 +135,16 @@ func (p *PluginSyncer) buildQuestionContents(ctx context.Context, questions []*e // Get comments on the question questionComments := make([]*entity.Comment, 0) - err = p.data.DB.Context(ctx).Where("object_id = ?", q.ID). + err = p.data.DB.Context(ctx).Where("object_id = ? AND status = ?", q.ID, + entity.CommentStatusAvailable). OrderBy("created_at ASC").Limit(50).Find(&questionComments) if err != nil { log.Warnf("get comments for question %s failed: %v", q.ID, err) } else { for _, c := range questionComments { + if c.Status != entity.CommentStatusAvailable { + continue + } parts = append(parts, fmt.Sprintf("Comment: %s", c.OriginalText)) meta.Comments = append(meta.Comments, plugin.VectorSearchMetadataComment{ CommentID: uid.DeShortID(c.ID), @@ -143,6 +169,9 @@ func (p *PluginSyncer) buildAnswerContents(ctx context.Context, answers []*entit []*plugin.VectorSearchContent, error) { result := make([]*plugin.VectorSearchContent, 0, len(answers)) for _, a := range answers { + if a.Status != entity.AnswerStatusAvailable { + continue + } // Get parent question for title question := &entity.Question{} exist, err := p.data.DB.Context(ctx).Where("id = ?", a.QuestionID).Get(question) @@ -153,6 +182,9 @@ func (p *PluginSyncer) buildAnswerContents(ctx context.Context, answers []*entit if !exist { continue } + if !isPublicQuestion(question) { + continue + } meta := plugin.VectorSearchMetadata{ QuestionID: uid.DeShortID(a.QuestionID), @@ -169,12 +201,16 @@ func (p *PluginSyncer) buildAnswerContents(ctx context.Context, answers []*entit // Get comments on this answer answerComments := make([]*entity.Comment, 0) - err = p.data.DB.Context(ctx).Where("object_id = ?", a.ID). + err = p.data.DB.Context(ctx).Where("object_id = ? AND status = ?", a.ID, + entity.CommentStatusAvailable). OrderBy("created_at ASC").Limit(50).Find(&answerComments) if err != nil { log.Warnf("get comments for answer %s failed: %v", a.ID, err) } else { for _, c := range answerComments { + if c.Status != entity.CommentStatusAvailable { + continue + } parts = append(parts, fmt.Sprintf("Comment: %s", c.OriginalText)) answerMeta.Comments = append(answerMeta.Comments, plugin.VectorSearchMetadataComment{ CommentID: uid.DeShortID(c.ID), @@ -195,6 +231,11 @@ func (p *PluginSyncer) buildAnswerContents(ctx context.Context, answers []*entit return result, nil } +func isPublicQuestion(question *entity.Question) bool { + return question != nil && question.Show == entity.QuestionShow && + (question.Status == entity.QuestionStatusAvailable || question.Status == entity.QuestionStatusClosed) +} + // BuildQuestionContentByID builds vector content for one question using the same // aggregation semantics as bulk vector sync. func BuildQuestionContentByID(ctx context.Context, data *data.Data, questionID string) (*plugin.VectorSearchContent, error) { diff --git a/internal/repo/vector_search_sync/syncer_test.go b/internal/repo/vector_search_sync/syncer_test.go new file mode 100644 index 00000000..bfaaeef9 --- /dev/null +++ b/internal/repo/vector_search_sync/syncer_test.go @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package vector_search_sync + +import ( + "context" + "strings" + "testing" + + "github.com/apache/answer/internal/base/data" + "github.com/apache/answer/internal/entity" +) + +func TestPluginSyncerIndexesOnlyPublicContent(t *testing.T) { + ctx := context.Background() + db, err := data.NewDB(false, &data.Database{Driver: "sqlite", Connection: ":memory:"}) + if err != nil { + t.Fatalf("create SQLite database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + if err := db.Sync2(new(entity.Question), new(entity.Answer), new(entity.Comment)); err != nil { + t.Fatalf("create tables: %v", err) + } + + questions := []*entity.Question{ + {ID: "10010000000000001", Title: "public", OriginalText: "PUBLIC-QUESTION", Status: entity.QuestionStatusAvailable, Show: entity.QuestionShow}, + {ID: "10010000000000002", Title: "hidden", OriginalText: "HIDDEN-QUESTION", Status: entity.QuestionStatusAvailable, Show: entity.QuestionHide}, + {ID: "10010000000000003", Title: "pending", OriginalText: "PENDING-QUESTION", Status: entity.QuestionStatusPending, Show: entity.QuestionShow}, + } + for _, question := range questions { + if _, err := db.Context(ctx).Insert(question); err != nil { + t.Fatalf("insert question %s: %v", question.ID, err) + } + } + answers := []*entity.Answer{ + {ID: "10020000000000001", QuestionID: questions[0].ID, OriginalText: "PUBLIC-ANSWER", Status: entity.AnswerStatusAvailable}, + {ID: "10020000000000002", QuestionID: questions[0].ID, OriginalText: "DELETED-ANSWER", Status: entity.AnswerStatusDeleted}, + {ID: "10020000000000003", QuestionID: questions[1].ID, OriginalText: "HIDDEN-ANSWER", Status: entity.AnswerStatusAvailable}, + } + for _, answer := range answers { + if _, err := db.Context(ctx).Insert(answer); err != nil { + t.Fatalf("insert answer %s: %v", answer.ID, err) + } + } + comments := []*entity.Comment{ + {ID: "10030000000000001", ObjectID: questions[0].ID, QuestionID: questions[0].ID, OriginalText: "PUBLIC-COMMENT", Status: entity.CommentStatusAvailable}, + {ID: "10030000000000002", ObjectID: questions[0].ID, QuestionID: questions[0].ID, OriginalText: "DELETED-COMMENT", Status: entity.CommentStatusDeleted}, + } + for _, comment := range comments { + if _, err := db.Context(ctx).Insert(comment); err != nil { + t.Fatalf("insert comment %s: %v", comment.ID, err) + } + } + + syncer := NewPluginSyncer(&data.Data{DB: db}) + questionContents, err := syncer.GetQuestionsPage(ctx, 1, 10) + if err != nil { + t.Fatalf("get question page: %v", err) + } + if len(questionContents) != 1 { + t.Fatalf("indexed question count = %d, want 1", len(questionContents)) + } + content := questionContents[0].Content + for _, restricted := range []string{"HIDDEN-QUESTION", "PENDING-QUESTION", "DELETED-ANSWER", "DELETED-COMMENT"} { + if strings.Contains(content, restricted) { + t.Fatalf("question vector content contains restricted marker %q: %s", restricted, content) + } + } + + answerContents, err := syncer.GetAnswersPage(ctx, 1, 10) + if err != nil { + t.Fatalf("get answer page: %v", err) + } + if len(answerContents) != 1 || !strings.Contains(answerContents[0].Content, "PUBLIC-ANSWER") { + t.Fatalf("answer vectors = %#v, want only the public answer", answerContents) + } +} + +func TestBuildQuestionContentByIDDeletesRestrictedQuestion(t *testing.T) { + ctx := context.Background() + db, err := data.NewDB(false, &data.Database{Driver: "sqlite", Connection: ":memory:"}) + if err != nil { + t.Fatalf("create SQLite database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + if err := db.Sync2(new(entity.Question)); err != nil { + t.Fatalf("create question table: %v", err) + } + question := &entity.Question{ID: "10010000000000010", Title: "hidden", OriginalText: "secret", Status: entity.QuestionStatusAvailable, Show: entity.QuestionHide} + if _, err := db.Context(ctx).Insert(question); err != nil { + t.Fatalf("insert hidden question: %v", err) + } + + content, err := BuildQuestionContentByID(ctx, &data.Data{DB: db}, question.ID) + if err != nil { + t.Fatalf("build content: %v", err) + } + if content != nil { + t.Fatalf("restricted question produced vector content: %#v", content) + } +} diff --git a/internal/service/content/question_service.go b/internal/service/content/question_service.go index 3350ef9b..d20ee1ea 100644 --- a/internal/service/content/question_service.go +++ b/internal/service/content/question_service.go @@ -533,6 +533,19 @@ func (qs *QuestionService) OperationQuestion(ctx context.Context, req *schema.Op if err != nil { return err } + // Rebuild on visibility changes. The builder deletes the vector document + // when the question is no longer public, and restores it when it is shown. + if req.Operation == schema.QuestionOperationHide || req.Operation == schema.QuestionOperationShow { + action := vector_sync.ActionUpsert + if req.Operation == schema.QuestionOperationShow { + action = vector_sync.ActionRestoreQuestion + } + qs.vectorSyncService.Send(ctx, &vector_sync.Task{ + Action: action, + ObjectType: vector_sync.ObjectTypeQuestion, + ObjectID: questionInfo.ID, + }) + } actMap := make(map[string]constant.ActivityTypeKey) actMap[schema.QuestionOperationPin] = constant.ActQuestionPin @@ -791,7 +804,7 @@ func (qs *QuestionService) RecoverQuestion(ctx context.Context, req *schema.Ques OriginalObjectID: questionInfo.ID, ActivityTypeKey: constant.ActQuestionUndeleted, }) - qs.vectorSyncService.Send(ctx, &vector_sync.Task{Action: vector_sync.ActionUpsert, ObjectType: vector_sync.ObjectTypeQuestion, ObjectID: questionInfo.ID}) + qs.vectorSyncService.Send(ctx, &vector_sync.Task{Action: vector_sync.ActionRestoreQuestion, ObjectType: vector_sync.ObjectTypeQuestion, ObjectID: questionInfo.ID}) return nil } @@ -1691,7 +1704,11 @@ func (qs *QuestionService) AdminSetQuestionStatus(ctx context.Context, req *sche case entity.QuestionStatusDeleted: qs.vectorSyncService.Send(ctx, &vector_sync.Task{Action: vector_sync.ActionDelete, ObjectType: vector_sync.ObjectTypeQuestion, ObjectID: questionInfo.ID}) case entity.QuestionStatusAvailable: - qs.vectorSyncService.Send(ctx, &vector_sync.Task{Action: vector_sync.ActionUpsert, ObjectType: vector_sync.ObjectTypeQuestion, ObjectID: questionInfo.ID}) + action := vector_sync.ActionUpsert + if questionInfo.Status == entity.QuestionStatusDeleted { + action = vector_sync.ActionRestoreQuestion + } + qs.vectorSyncService.Send(ctx, &vector_sync.Task{Action: action, ObjectType: vector_sync.ObjectTypeQuestion, ObjectID: questionInfo.ID}) } return nil } diff --git a/internal/service/plugin_common/plugin_common_service.go b/internal/service/plugin_common/plugin_common_service.go index 2c1a72fd..88cc0214 100644 --- a/internal/service/plugin_common/plugin_common_service.go +++ b/internal/service/plugin_common/plugin_common_service.go @@ -26,6 +26,7 @@ import ( "github.com/apache/answer/internal/base/data" "github.com/apache/answer/internal/repo/search_sync" "github.com/apache/answer/internal/repo/vector_search_sync" + "github.com/apache/answer/internal/service/vector_sync" "github.com/segmentfault/pacman/errors" "github.com/segmentfault/pacman/log" @@ -106,7 +107,7 @@ func (ps *PluginCommonService) UpdatePluginConfig(ctx context.Context, req *sche }) _ = plugin.CallVectorSearch(func(vs plugin.VectorSearch) error { if vs.Info().SlugName == req.PluginSlugName { - vs.RegisterSyncer(ctx, vector_search_sync.NewPluginSyncer(ps.data)) + ps.registerVectorSyncer(vs) } return nil }) @@ -185,7 +186,7 @@ func (ps *PluginCommonService) initPluginData() { // register syncer for vector search plugins on startup _ = plugin.CallVectorSearch(func(vs plugin.VectorSearch) error { - vs.RegisterSyncer(context.Background(), vector_search_sync.NewPluginSyncer(ps.data)) + ps.registerVectorSyncer(vs) return nil }) @@ -229,3 +230,16 @@ func (ps *PluginCommonService) initPluginData() { } }() } + +// registerVectorSyncer removes historical restricted documents before asking a +// plugin to rebuild its index. This can involve remote vector-store calls, so it +// must not block startup or a plugin configuration request. +func (ps *PluginCommonService) registerVectorSyncer(vs plugin.VectorSearch) { + go func() { + ctx := context.Background() + if err := vector_sync.RemoveRestrictedContents(ctx, ps.data, vs); err != nil { + log.Errorf("remove restricted vector content for %s failed: %v", vs.Info().SlugName, err) + } + vs.RegisterSyncer(ctx, vector_search_sync.NewPluginSyncer(ps.data)) + }() +} diff --git a/internal/service/vector_sync/vector_sync.go b/internal/service/vector_sync/vector_sync.go index c7281ad4..f6c529f9 100644 --- a/internal/service/vector_sync/vector_sync.go +++ b/internal/service/vector_sync/vector_sync.go @@ -21,9 +21,11 @@ package vector_sync import ( "context" + "fmt" "github.com/apache/answer/internal/base/data" "github.com/apache/answer/internal/base/queue" + "github.com/apache/answer/internal/entity" "github.com/apache/answer/internal/repo/vector_search_sync" "github.com/apache/answer/pkg/uid" "github.com/apache/answer/plugin" @@ -33,6 +35,9 @@ import ( const ( ActionUpsert = "upsert" ActionDelete = "delete" + // ActionRestoreQuestion rebuilds a public question and its available + // answers after the question itself was restored or shown again. + ActionRestoreQuestion = "restore_question" ObjectTypeQuestion = "question" ObjectTypeAnswer = "answer" @@ -40,6 +45,8 @@ const ( const maxRetry = 3 +const cleanupPageSize = 500 + type Task struct { Action string ObjectType string @@ -87,9 +94,15 @@ func handle(ctx context.Context, data *data.Data, msg *Task) error { func handleOnce(ctx context.Context, data *data.Data, vectorSearch plugin.VectorSearch, action, objectType, objectID string) error { if action == ActionDelete { + if objectType == ObjectTypeQuestion { + return deleteQuestionAndAnswerContents(ctx, data, vectorSearch, objectID) + } return vectorSearch.DeleteContent(ctx, objectID) } if action != ActionUpsert { + if action == ActionRestoreQuestion && objectType == ObjectTypeQuestion { + return restoreQuestionAndAnswerContents(ctx, data, vectorSearch, objectID) + } return nil } @@ -109,7 +122,163 @@ func handleOnce(ctx context.Context, data *data.Data, vectorSearch plugin.Vector return err } if content == nil { + if objectType == ObjectTypeQuestion { + return deleteQuestionAndAnswerContents(ctx, data, vectorSearch, objectID) + } return vectorSearch.DeleteContent(ctx, objectID) } return vectorSearch.UpdateContent(ctx, content) } + +func restoreQuestionAndAnswerContents(ctx context.Context, data *data.Data, vectorSearch plugin.VectorSearch, questionID string) error { + questionID = uid.DeShortID(questionID) + content, err := vector_search_sync.BuildQuestionContentByID(ctx, data, questionID) + if err != nil { + return err + } + if content == nil { + return deleteQuestionAndAnswerContents(ctx, data, vectorSearch, questionID) + } + if err := vectorSearch.UpdateContent(ctx, content); err != nil { + return err + } + answers := make([]*entity.Answer, 0) + if err := data.DB.Context(ctx).Where("question_id = ? AND status = ?", questionID, + entity.AnswerStatusAvailable).Find(&answers); err != nil { + return fmt.Errorf("list answer vectors for restored question %s: %w", questionID, err) + } + for _, answer := range answers { + answerContent, err := vector_search_sync.BuildAnswerContentByID(ctx, data, answer.ID) + if err != nil { + return err + } + if answerContent == nil { + continue + } + if err := vectorSearch.UpdateContent(ctx, answerContent); err != nil { + return fmt.Errorf("restore answer vector %s: %w", answer.ID, err) + } + } + return nil +} + +func deleteQuestionAndAnswerContents(ctx context.Context, data *data.Data, vectorSearch plugin.VectorSearch, questionID string) error { + questionID = uid.DeShortID(questionID) + if err := vectorSearch.DeleteContent(ctx, questionID); err != nil { + return err + } + answers := make([]*entity.Answer, 0) + if err := data.DB.Context(ctx).Where("question_id = ?", questionID).Find(&answers); err != nil { + return fmt.Errorf("list answer vectors for question %s: %w", questionID, err) + } + for _, answer := range answers { + if err := vectorSearch.DeleteContent(ctx, uid.DeShortID(answer.ID)); err != nil { + return fmt.Errorf("delete answer vector %s: %w", answer.ID, err) + } + } + return nil +} + +// RemoveRestrictedContents removes documents that were indexed before visibility +// filtering was introduced. It deliberately uses the existing VectorSearch +// interface, so installed plugins remain binary-compatible. +func RemoveRestrictedContents(ctx context.Context, data *data.Data, vectorSearch plugin.VectorSearch) error { + if data == nil || data.DB == nil || vectorSearch == nil { + return nil + } + + questions := make([]*entity.Question, 0, cleanupPageSize) + for offset := 0; ; offset += cleanupPageSize { + questions = questions[:0] + if err := data.DB.Context(ctx).OrderBy("id ASC").Limit(cleanupPageSize, offset).Find(&questions); err != nil { + return fmt.Errorf("list questions for vector cleanup: %w", err) + } + for _, question := range questions { + if question.Show == entity.QuestionShow && + (question.Status == entity.QuestionStatusAvailable || question.Status == entity.QuestionStatusClosed) { + hasRestricted, err := hasRestrictedQuestionAggregate(ctx, data, question.ID) + if err != nil { + return err + } + if hasRestricted { + // The following plugin bulk sync rebuilds this document from the + // new filtered source. Delete the old aggregate first so deleted + // answers/comments cannot remain in the vector store. + if err := vectorSearch.DeleteContent(ctx, uid.DeShortID(question.ID)); err != nil { + return fmt.Errorf("delete stale question vector %s: %w", question.ID, err) + } + } + continue + } + if err := deleteQuestionAndAnswerContents(ctx, data, vectorSearch, question.ID); err != nil { + return fmt.Errorf("delete vectors for restricted question %s: %w", question.ID, err) + } + } + if len(questions) < cleanupPageSize { + break + } + } + + answers := make([]*entity.Answer, 0, cleanupPageSize) + for offset := 0; ; offset += cleanupPageSize { + answers = answers[:0] + if err := data.DB.Context(ctx).Where("status <> ?", entity.AnswerStatusAvailable). + OrderBy("id ASC").Limit(cleanupPageSize, offset).Find(&answers); err != nil { + return fmt.Errorf("list restricted answers for vector cleanup: %w", err) + } + for _, answer := range answers { + if err := vectorSearch.DeleteContent(ctx, uid.DeShortID(answer.ID)); err != nil { + return fmt.Errorf("delete restricted answer vector %s: %w", answer.ID, err) + } + } + if len(answers) < cleanupPageSize { + break + } + } + + for offset := 0; ; offset += cleanupPageSize { + answers = answers[:0] + if err := data.DB.Context(ctx).Where("status = ?", entity.AnswerStatusAvailable). + OrderBy("id ASC").Limit(cleanupPageSize, offset).Find(&answers); err != nil { + return fmt.Errorf("list available answers for vector cleanup: %w", err) + } + for _, answer := range answers { + hasRestricted, err := hasRestrictedAnswerAggregate(ctx, data, answer.ID) + if err != nil { + return err + } + if hasRestricted { + if err := vectorSearch.DeleteContent(ctx, uid.DeShortID(answer.ID)); err != nil { + return fmt.Errorf("delete stale answer vector %s: %w", answer.ID, err) + } + } + } + if len(answers) < cleanupPageSize { + break + } + } + return nil +} + +func hasRestrictedQuestionAggregate(ctx context.Context, data *data.Data, questionID string) (bool, error) { + answerCount, err := data.DB.Context(ctx).Where("question_id = ? AND status <> ?", questionID, + entity.AnswerStatusAvailable).Count(new(entity.Answer)) + if err != nil { + return false, fmt.Errorf("count restricted answers for question %s: %w", questionID, err) + } + commentCount, err := data.DB.Context(ctx).Where("question_id = ? AND status <> ?", questionID, + entity.CommentStatusAvailable).Count(new(entity.Comment)) + if err != nil { + return false, fmt.Errorf("count restricted comments for question %s: %w", questionID, err) + } + return answerCount > 0 || commentCount > 0, nil +} + +func hasRestrictedAnswerAggregate(ctx context.Context, data *data.Data, answerID string) (bool, error) { + count, err := data.DB.Context(ctx).Where("object_id = ? AND status <> ?", answerID, + entity.CommentStatusAvailable).Count(new(entity.Comment)) + if err != nil { + return false, fmt.Errorf("count restricted comments for answer %s: %w", answerID, err) + } + return count > 0, nil +} diff --git a/internal/service/vector_sync/vector_sync_test.go b/internal/service/vector_sync/vector_sync_test.go new file mode 100644 index 00000000..9649ee51 --- /dev/null +++ b/internal/service/vector_sync/vector_sync_test.go @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package vector_sync + +import ( + "context" + "testing" + + "github.com/apache/answer/internal/base/data" + "github.com/apache/answer/internal/entity" + "github.com/apache/answer/plugin" +) + +type cleanupTestVectorSearch struct { + deleted map[string]bool + updated map[string]bool +} + +func (*cleanupTestVectorSearch) Info() plugin.Info { return plugin.Info{} } +func (*cleanupTestVectorSearch) Description() plugin.VectorSearchDesc { + return plugin.VectorSearchDesc{} +} +func (*cleanupTestVectorSearch) RegisterSyncer(context.Context, plugin.VectorSearchSyncer) {} +func (*cleanupTestVectorSearch) SearchSimilar(context.Context, string, int) ([]plugin.VectorSearchResult, error) { + return nil, nil +} +func (s *cleanupTestVectorSearch) UpdateContent(_ context.Context, content *plugin.VectorSearchContent) error { + if s.updated != nil { + s.updated[content.ObjectID] = true + } + return nil +} +func (s *cleanupTestVectorSearch) DeleteContent(_ context.Context, objectID string) error { + s.deleted[objectID] = true + return nil +} + +func TestRemoveRestrictedContentsRemovesHistoricalVectors(t *testing.T) { + ctx := context.Background() + db, err := data.NewDB(false, &data.Database{Driver: "sqlite", Connection: ":memory:"}) + if err != nil { + t.Fatalf("create SQLite database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + if err := db.Sync2(new(entity.Question), new(entity.Answer), new(entity.Comment)); err != nil { + t.Fatalf("create tables: %v", err) + } + + hiddenQuestion := &entity.Question{ID: "10010000000000001", Status: entity.QuestionStatusAvailable, Show: entity.QuestionHide} + publicQuestion := &entity.Question{ID: "10010000000000002", Status: entity.QuestionStatusAvailable, Show: entity.QuestionShow} + for _, question := range []*entity.Question{hiddenQuestion, publicQuestion} { + if _, err := db.Context(ctx).Insert(question); err != nil { + t.Fatalf("insert question: %v", err) + } + } + hiddenAnswer := &entity.Answer{ID: "10020000000000001", QuestionID: hiddenQuestion.ID, Status: entity.AnswerStatusAvailable} + deletedAnswer := &entity.Answer{ID: "10020000000000002", QuestionID: publicQuestion.ID, Status: entity.AnswerStatusDeleted} + publicAnswer := &entity.Answer{ID: "10020000000000003", QuestionID: publicQuestion.ID, Status: entity.AnswerStatusAvailable} + for _, answer := range []*entity.Answer{hiddenAnswer, deletedAnswer, publicAnswer} { + if _, err := db.Context(ctx).Insert(answer); err != nil { + t.Fatalf("insert answer: %v", err) + } + } + deletedComment := &entity.Comment{ID: "10030000000000001", ObjectID: publicAnswer.ID, QuestionID: publicQuestion.ID, Status: entity.CommentStatusDeleted} + if _, err := db.Context(ctx).Insert(deletedComment); err != nil { + t.Fatalf("insert deleted comment: %v", err) + } + + vector := &cleanupTestVectorSearch{deleted: map[string]bool{}} + if err := RemoveRestrictedContents(ctx, &data.Data{DB: db}, vector); err != nil { + t.Fatalf("remove restricted contents: %v", err) + } + for _, objectID := range []string{hiddenQuestion.ID, hiddenAnswer.ID, deletedAnswer.ID, publicQuestion.ID, publicAnswer.ID} { + if !vector.deleted[objectID] { + t.Fatalf("restricted vector %s was not deleted: %v", objectID, vector.deleted) + } + } + if vector.deleted["not-a-vector"] { + t.Fatalf("unexpected vector deletion: %v", vector.deleted) + } +} + +func TestHandleOnceRemovesAnswerVectorsWhenQuestionBecomesHidden(t *testing.T) { + ctx := context.Background() + db, err := data.NewDB(false, &data.Database{Driver: "sqlite", Connection: ":memory:"}) + if err != nil { + t.Fatalf("create SQLite database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + if err := db.Sync2(new(entity.Question), new(entity.Answer), new(entity.Comment)); err != nil { + t.Fatalf("create tables: %v", err) + } + question := &entity.Question{ID: "10010000000000010", Status: entity.QuestionStatusAvailable, Show: entity.QuestionHide} + answer := &entity.Answer{ID: "10020000000000010", QuestionID: question.ID, Status: entity.AnswerStatusAvailable} + if _, err := db.Context(ctx).Insert(question); err != nil { + t.Fatalf("insert hidden question: %v", err) + } + if _, err := db.Context(ctx).Insert(answer); err != nil { + t.Fatalf("insert answer: %v", err) + } + + vector := &cleanupTestVectorSearch{deleted: map[string]bool{}} + if err := handleOnce(ctx, &data.Data{DB: db}, vector, ActionUpsert, ObjectTypeQuestion, question.ID); err != nil { + t.Fatalf("handle hidden question upsert: %v", err) + } + if !vector.deleted[question.ID] || !vector.deleted[answer.ID] { + t.Fatalf("hidden question must remove question and answer vectors: %v", vector.deleted) + } +} + +func TestHandleOnceRestoresAnswerVectorsWhenQuestionIsShown(t *testing.T) { + ctx := context.Background() + db, err := data.NewDB(false, &data.Database{Driver: "sqlite", Connection: ":memory:"}) + if err != nil { + t.Fatalf("create SQLite database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + if err := db.Sync2(new(entity.Question), new(entity.Answer), new(entity.Comment)); err != nil { + t.Fatalf("create tables: %v", err) + } + question := &entity.Question{ID: "10010000000000011", OriginalText: "question", Status: entity.QuestionStatusAvailable, Show: entity.QuestionShow} + answer := &entity.Answer{ID: "10020000000000011", QuestionID: question.ID, OriginalText: "answer", Status: entity.AnswerStatusAvailable} + if _, err := db.Context(ctx).Insert(question); err != nil { + t.Fatalf("insert shown question: %v", err) + } + if _, err := db.Context(ctx).Insert(answer); err != nil { + t.Fatalf("insert answer: %v", err) + } + + vector := &cleanupTestVectorSearch{deleted: map[string]bool{}, updated: map[string]bool{}} + if err := handleOnce(ctx, &data.Data{DB: db}, vector, ActionRestoreQuestion, ObjectTypeQuestion, question.ID); err != nil { + t.Fatalf("restore shown question: %v", err) + } + if !vector.updated[question.ID] || !vector.updated[answer.ID] { + t.Fatalf("shown question must restore question and answer vectors: %v", vector.updated) + } +}
