This is an automated email from the ASF dual-hosted git repository.

LinkinStars pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/answer.git


The following commit(s) were added to refs/heads/dev by this push:
     new e97d1957 fix: avoid skipping file records in orphan upload cleanup
e97d1957 is described below

commit e97d1957bd99056b6a3be47f97b6987667a7e64f
Author: lhk <[email protected]>
AuthorDate: Thu Sep 24 19:19:11 2026 +0800

    fix: avoid skipping file records in orphan upload cleanup
    
    CleanOrphanUploadFiles scans available file records with OFFSET/LIMIT
    pagination while marking the processed orphans as Deleted, which removes
    them from the available result set. The records that follow shift forward,
    so the offset of the next page skips records that were never checked in
    the same cleanup run.
    
    Scan the available records by id cursor instead: query the records with
    status = Available and id greater than the last scanned id, ordered by id
    ascending, and advance the cursor per record. Records marked as deleted
    during the scan no longer affect the position of the remaining records,
    so every eligible record is checked in a single run.
    
    Close #1631
---
 internal/repo/file_record/file_record_repo.go      |  21 ++-
 .../service/file_record/file_record_service.go     |  19 +-
 .../file_record/file_record_service_test.go        | 205 +++++++++++++++++++++
 3 files changed, 228 insertions(+), 17 deletions(-)

diff --git a/internal/repo/file_record/file_record_repo.go 
b/internal/repo/file_record/file_record_repo.go
index ce486c7a..b305effb 100644
--- a/internal/repo/file_record/file_record_repo.go
+++ b/internal/repo/file_record/file_record_repo.go
@@ -22,12 +22,10 @@ package file_record
 import (
        "context"
 
-       "github.com/apache/answer/internal/base/pager"
-       "github.com/apache/answer/internal/service/file_record"
-
        "github.com/apache/answer/internal/base/data"
        "github.com/apache/answer/internal/base/reason"
        "github.com/apache/answer/internal/entity"
+       "github.com/apache/answer/internal/service/file_record"
        "github.com/segmentfault/pacman/errors"
 )
 
@@ -52,13 +50,20 @@ func (fr *fileRecordRepo) AddFileRecord(ctx 
context.Context, fileRecord *entity.
        return
 }
 
-// GetFileRecordPage get fileRecord page
-func (fr *fileRecordRepo) GetFileRecordPage(ctx context.Context, page, 
pageSize int, cond *entity.FileRecord) (
-       fileRecordList []*entity.FileRecord, total int64, err error) {
+// GetFileRecordListAfterID get available file record list whose id is greater 
than lastID,
+// the result is ordered by id ascending. It is used to scan all available 
file records by
+// id cursor, because the scanned records may be marked as deleted during the 
scan and
+// offset pagination would skip records in that case.
+func (fr *fileRecordRepo) GetFileRecordListAfterID(ctx context.Context, 
lastID, limit int) (
+       fileRecordList []*entity.FileRecord, err error) {
        fileRecordList = make([]*entity.FileRecord, 0)
 
-       session := fr.data.DB.Context(ctx)
-       total, err = pager.Help(page, pageSize, &fileRecordList, cond, session)
+       err = fr.data.DB.Context(ctx).
+               Where("status = ?", entity.FileRecordStatusAvailable).
+               And("id > ?", lastID).
+               OrderBy("id ASC").
+               Limit(limit).
+               Find(&fileRecordList)
        if err != nil {
                err = 
errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
        }
diff --git a/internal/service/file_record/file_record_service.go 
b/internal/service/file_record/file_record_service.go
index aa526f01..f52eba14 100644
--- a/internal/service/file_record/file_record_service.go
+++ b/internal/service/file_record/file_record_service.go
@@ -43,8 +43,7 @@ import (
 type FileRecordRepo interface {
        AddFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err 
error)
        UpdateFileRecord(ctx context.Context, fileRecord *entity.FileRecord) 
(err error)
-       GetFileRecordPage(ctx context.Context, page, pageSize int, cond 
*entity.FileRecord) (
-               fileRecordList []*entity.FileRecord, total int64, err error)
+       GetFileRecordListAfterID(ctx context.Context, lastID, limit int) 
(fileRecordList []*entity.FileRecord, err error)
        DeleteFileRecord(ctx context.Context, id int) (err error)
        GetFileRecordByURL(ctx context.Context, fileURL string) (record 
*entity.FileRecord, err error)
 }
@@ -92,20 +91,23 @@ func (fs *FileRecordService) AddFileRecord(ctx 
context.Context, userID, filePath
 
 // CleanOrphanUploadFiles clean orphan upload files
 func (fs *FileRecordService) CleanOrphanUploadFiles(ctx context.Context) {
-       page, pageSize := 1, 1000
+       pageSize := 1000
+       // Scan available file records by id cursor. Records are marked as 
deleted during the
+       // scan, so they leave the available result set: offset pagination 
would shift the
+       // remaining records forward and skip some of them.
+       lastID := 0
 
        for {
-               fileRecordList, total, err := 
fs.fileRecordRepo.GetFileRecordPage(ctx, page, pageSize, &entity.FileRecord{
-                       Status: entity.FileRecordStatusAvailable,
-               })
+               fileRecordList, err := 
fs.fileRecordRepo.GetFileRecordListAfterID(ctx, lastID, pageSize)
                if err != nil {
-                       log.Errorf("get file record page error: %v", err)
+                       log.Errorf("get file record list error: %v", err)
                        return
                }
-               if len(fileRecordList) == 0 || total == 0 {
+               if len(fileRecordList) == 0 {
                        break
                }
                for _, fileRecord := range fileRecordList {
+                       lastID = fileRecord.ID
                        // If this file record created in 48 hours, no need to 
check
                        if fileRecord.CreatedAt.AddDate(0, 0, 
2).After(time.Now()) {
                                continue
@@ -154,7 +156,6 @@ func (fs *FileRecordService) CleanOrphanUploadFiles(ctx 
context.Context) {
                                log.Error(err)
                        }
                }
-               page++
        }
 }
 
diff --git a/internal/service/file_record/file_record_service_test.go 
b/internal/service/file_record/file_record_service_test.go
new file mode 100644
index 00000000..85bb5f00
--- /dev/null
+++ b/internal/service/file_record/file_record_service_test.go
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package file_record
+
+import (
+       "context"
+       "fmt"
+       "os"
+       "path/filepath"
+       "testing"
+       "time"
+
+       "github.com/apache/answer/internal/base/constant"
+       "github.com/apache/answer/internal/entity"
+       "github.com/apache/answer/internal/service/service_config"
+       "github.com/stretchr/testify/assert"
+       "github.com/stretchr/testify/require"
+       "xorm.io/xorm"
+)
+
+// fakeFileRecordRepo is an in-memory file record repo. It keeps the same 
semantics as the
+// database one: GetFileRecordListAfterID only returns available records 
ordered by id, and
+// DeleteFileRecord marks a record as deleted so that it leaves the available 
result set.
+type fakeFileRecordRepo struct {
+       records   []*entity.FileRecord
+       listCalls int
+}
+
+func (f *fakeFileRecordRepo) AddFileRecord(_ context.Context, fileRecord 
*entity.FileRecord) error {
+       f.records = append(f.records, fileRecord)
+       return nil
+}
+
+func (f *fakeFileRecordRepo) UpdateFileRecord(_ context.Context, fileRecord 
*entity.FileRecord) error {
+       return nil
+}
+
+func (f *fakeFileRecordRepo) GetFileRecordListAfterID(_ context.Context, 
lastID, limit int) (
+       []*entity.FileRecord, error) {
+       f.listCalls++
+       list := make([]*entity.FileRecord, 0)
+       for _, record := range f.records {
+               if record.Status != entity.FileRecordStatusAvailable || 
record.ID <= lastID {
+                       continue
+               }
+               list = append(list, record)
+               if len(list) == limit {
+                       break
+               }
+       }
+       return list, nil
+}
+
+func (f *fakeFileRecordRepo) DeleteFileRecord(_ context.Context, id int) error 
{
+       for _, record := range f.records {
+               if record.ID == id {
+                       record.Status = entity.FileRecordStatusDeleted
+                       return nil
+               }
+       }
+       return fmt.Errorf("file record %d not found", id)
+}
+
+func (f *fakeFileRecordRepo) GetFileRecordByURL(_ context.Context, fileURL 
string) (
+       *entity.FileRecord, error) {
+       for _, record := range f.records {
+               if record.FileURL == fileURL && record.Status == 
entity.FileRecordStatusAvailable {
+                       return record, nil
+               }
+       }
+       return nil, nil
+}
+
+// fakeRevisionRepo is a revision repo stub which reports the given file urls 
as referenced.
+type fakeRevisionRepo struct {
+       usedFileURLs map[string]*entity.Revision
+}
+
+func (f *fakeRevisionRepo) AddRevision(_ context.Context, _ *entity.Revision, 
_ bool) error {
+       return nil
+}
+
+func (f *fakeRevisionRepo) GetRevisionByID(_ context.Context, _ string) 
(*entity.Revision, bool, error) {
+       return nil, false, nil
+}
+
+func (f *fakeRevisionRepo) GetLastRevisionByObjectID(_ context.Context, _ 
string) (*entity.Revision, bool, error) {
+       return nil, false, nil
+}
+
+func (f *fakeRevisionRepo) GetLastRevisionByFileURL(_ context.Context, fileURL 
string) (*entity.Revision, bool, error) {
+       if revision, ok := f.usedFileURLs[fileURL]; ok {
+               return revision, true, nil
+       }
+       return nil, false, nil
+}
+
+func (f *fakeRevisionRepo) GetRevisionList(_ context.Context, _ 
*entity.Revision) ([]entity.Revision, error) {
+       return nil, nil
+}
+
+func (f *fakeRevisionRepo) UpdateObjectRevisionId(_ context.Context, _ 
*entity.Revision, _ *xorm.Session) error {
+       return nil
+}
+
+func (f *fakeRevisionRepo) ExistUnreviewedByObjectID(_ context.Context, _ 
string) (*entity.Revision, bool, error) {
+       return nil, false, nil
+}
+
+func (f *fakeRevisionRepo) GetUnreviewedRevisionPage(_ context.Context, _, _ 
int, _ []int) (
+       []*entity.Revision, int64, error) {
+       return nil, 0, nil
+}
+
+func (f *fakeRevisionRepo) CountUnreviewedRevision(_ context.Context, _ []int) 
(int64, error) {
+       return 0, nil
+}
+
+func (f *fakeRevisionRepo) UpdateStatus(_ context.Context, _ string, _ int, _ 
string) error {
+       return nil
+}
+
+// TestFileRecordService_CleanOrphanUploadFiles checks that all orphan files 
are cleaned even
+// though the scan marks records as deleted while iterating, which used to 
make offset
+// pagination skip records between pages.
+func TestFileRecordService_CleanOrphanUploadFiles(t *testing.T) {
+       const (
+               recordCount        = 2500
+               referencedRecordID = 1234
+               recentRecordID     = 2500
+       )
+
+       uploadPath := t.TempDir()
+       require.NoError(t, os.MkdirAll(filepath.Join(uploadPath, 
constant.DeletedSubPath), 0o755))
+       require.NoError(t, os.MkdirAll(filepath.Join(uploadPath, "files", 
"post"), 0o755))
+
+       repo := &fakeFileRecordRepo{}
+       revisionRepo := &fakeRevisionRepo{usedFileURLs: 
map[string]*entity.Revision{}}
+       oldCreatedAt := time.Now().Add(-time.Hour * 72)
+       for id := 1; id <= recordCount; id++ {
+               filePath := fmt.Sprintf("files/post/%d.txt", id)
+               fileURL := fmt.Sprintf("uploads/%s", filePath)
+               require.NoError(t, os.WriteFile(filepath.Join(uploadPath, 
filePath), []byte("x"), 0o644))
+
+               createdAt := oldCreatedAt
+               if id == recentRecordID {
+                       createdAt = time.Now()
+               }
+               repo.records = append(repo.records, &entity.FileRecord{
+                       ID:        id,
+                       CreatedAt: createdAt,
+                       FilePath:  filePath,
+                       FileURL:   fileURL,
+                       ObjectID:  "0",
+                       Status:    entity.FileRecordStatusAvailable,
+               })
+               if id == referencedRecordID {
+                       revisionRepo.usedFileURLs[fileURL] = 
&entity.Revision{ObjectID: "100"}
+               }
+       }
+
+       serviceConfig := &service_config.ServiceConfig{UploadPath: uploadPath}
+       service := NewFileRecordService(repo, revisionRepo, serviceConfig, nil, 
nil)
+
+       service.CleanOrphanUploadFiles(context.Background())
+
+       assert.GreaterOrEqual(t, repo.listCalls, 3, "all records should be 
scanned page by page")
+
+       deletedCount := 0
+       for _, record := range repo.records {
+               filePath := filepath.Join(uploadPath, record.FilePath)
+               deletedFilePath := filepath.Join(uploadPath, 
constant.DeletedSubPath, filepath.Base(record.FilePath))
+               switch record.ID {
+               case referencedRecordID, recentRecordID:
+                       assert.Equal(t, entity.FileRecordStatusAvailable, 
record.Status,
+                               "record %d should not be deleted", record.ID)
+                       assert.FileExists(t, filePath, "file of record %d 
should stay in place", record.ID)
+                       assert.NoFileExists(t, deletedFilePath, "file of record 
%d should not be moved", record.ID)
+               default:
+                       deletedCount++
+                       assert.Equal(t, entity.FileRecordStatusDeleted, 
record.Status,
+                               "record %d should be deleted", record.ID)
+                       assert.NoFileExists(t, filePath, "file of record %d 
should be moved away", record.ID)
+                       assert.FileExists(t, deletedFilePath, "file of record 
%d should be moved to deleted dir", record.ID)
+               }
+       }
+       assert.Equal(t, recordCount-2, deletedCount)
+}

Reply via email to