This is an automated email from the ASF dual-hosted git repository.

linkinstar pushed a commit to branch feat/1.4.5/file
in repository https://gitbox.apache.org/repos/asf/answer.git

commit 5cffab864939e08bdf351feb9c8fc9b64934cdfa
Author: LinkinStars <[email protected]>
AuthorDate: Fri Jan 10 10:43:35 2025 +0800

    feat(service): implement file cleanup and deletion functionality
---
 cmd/wire_gen.go                                    |  12 +-
 configs/config.yaml                                |   3 +
 go.mod                                             |   2 +-
 internal/base/constant/upload.go                   |   1 +
 internal/base/cron/cron.go                         |  43 +++++-
 internal/controller/upload_controller.go           |   9 +-
 internal/entity/file_record_entity.go              |  45 ++++++
 internal/migrations/migrations.go                  |   1 +
 .../service_config.go => migrations/v25.go}        |  13 +-
 internal/repo/answer/answer_repo.go                |  19 ++-
 internal/repo/file_record/file_record_repo.go      |  84 ++++++++++
 internal/repo/provider.go                          |   2 +
 internal/repo/question/question_repo.go            |  17 ++
 internal/repo/revision/revision_repo.go            |  12 +-
 internal/router/static_router.go                   |  12 +-
 internal/service/content/question_service.go       |   2 +
 .../service/file_record/file_record_service.go     | 172 +++++++++++++++++++++
 internal/service/object_info/object_info.go        |   3 +-
 internal/service/provider.go                       |   2 +
 internal/service/revision/revision.go              |   1 +
 internal/service/service_config/service_config.go  |   5 +-
 internal/service/tag_common/tag_common.go          |   9 +-
 internal/service/uploader/upload.go                |  52 +++++--
 pkg/writer/writer.go                               |   5 +
 24 files changed, 483 insertions(+), 43 deletions(-)

diff --git a/cmd/wire_gen.go b/cmd/wire_gen.go
index b44fadfb..702f60df 100644
--- a/cmd/wire_gen.go
+++ b/cmd/wire_gen.go
@@ -48,6 +48,7 @@ import (
        "github.com/apache/answer/internal/repo/comment"
        "github.com/apache/answer/internal/repo/config"
        "github.com/apache/answer/internal/repo/export"
+       "github.com/apache/answer/internal/repo/file_record"
        "github.com/apache/answer/internal/repo/limit"
        "github.com/apache/answer/internal/repo/meta"
        notification2 "github.com/apache/answer/internal/repo/notification"
@@ -84,6 +85,7 @@ import (
        "github.com/apache/answer/internal/service/dashboard"
        "github.com/apache/answer/internal/service/event_queue"
        export2 "github.com/apache/answer/internal/service/export"
+       file_record2 "github.com/apache/answer/internal/service/file_record"
        "github.com/apache/answer/internal/service/follow"
        "github.com/apache/answer/internal/service/importer"
        meta2 "github.com/apache/answer/internal/service/meta"
@@ -227,10 +229,10 @@ func initApplication(debug bool, serverConf *conf.Server, 
dbConf *data.Database,
        contentRevisionService := content.NewRevisionService(revisionRepo, 
userCommon, questionCommon, answerService, objService, questionRepo, 
answerRepo, tagRepo, tagCommonService, notificationQueueService, 
activityQueueService, reportRepo, reviewService, reviewActivityRepo)
        revisionController := 
controller.NewRevisionController(contentRevisionService, rankService)
        rankController := controller.NewRankController(rankService)
-       badgeAwardRepo := badge_award.NewBadgeAwardRepo(dataData, uniqueIDRepo)
+       userAdminRepo := user.NewUserAdminRepo(dataData, authRepo)
        notificationRepo := notification2.NewNotificationRepo(dataData)
        pluginUserConfigRepo := plugin_config.NewPluginUserConfigRepo(dataData)
-       userAdminRepo := user.NewUserAdminRepo(dataData, authRepo)
+       badgeAwardRepo := badge_award.NewBadgeAwardRepo(dataData, uniqueIDRepo)
        userAdminService := user_admin.NewUserAdminService(userAdminRepo, 
userRoleRelService, authService, userCommon, userActiveActivityRepo, 
siteInfoCommonService, emailService, questionRepo, answerRepo, 
commentCommonRepo, userExternalLoginRepo, notificationRepo, 
pluginUserConfigRepo, badgeAwardRepo)
        userAdminController := 
controller_admin.NewUserAdminController(userAdminService)
        reasonRepo := reason.NewReasonRepo(configService)
@@ -246,7 +248,9 @@ func initApplication(debug bool, serverConf *conf.Server, 
dbConf *data.Database,
        notificationController := 
controller.NewNotificationController(notificationService, rankService)
        dashboardService := dashboard.NewDashboardService(questionRepo, 
answerRepo, commentCommonRepo, voteRepo, userRepo, reportRepo, configService, 
siteInfoCommonService, serviceConf, reviewService, revisionRepo, dataData)
        dashboardController := 
controller.NewDashboardController(dashboardService)
-       uploaderService := uploader.NewUploaderService(serviceConf, 
siteInfoCommonService)
+       fileRecordRepo := file_record.NewFileRecordRepo(dataData)
+       fileRecordService := file_record2.NewFileRecordService(fileRecordRepo, 
revisionRepo, serviceConf, siteInfoCommonService)
+       uploaderService := uploader.NewUploaderService(serviceConf, 
siteInfoCommonService, fileRecordService)
        uploadController := controller.NewUploadController(uploaderService)
        activityActivityRepo := activity.NewActivityRepo(dataData, 
configService)
        activityCommon := activity_common2.NewActivityCommon(activityRepo, 
activityQueueService)
@@ -287,7 +291,7 @@ func initApplication(debug bool, serverConf *conf.Server, 
dbConf *data.Database,
        renderController := controller.NewRenderController()
        pluginAPIRouter := router.NewPluginAPIRouter(connectorController, 
userCenterController, captchaController, embedController, renderController)
        ginEngine := server.NewHTTPServer(debug, staticRouter, answerAPIRouter, 
swaggerRouter, uiRouter, authUserMiddleware, avatarMiddleware, 
shortIDMiddleware, templateRouter, pluginAPIRouter, uiConf)
-       scheduledTaskManager := 
cron.NewScheduledTaskManager(siteInfoCommonService, questionService)
+       scheduledTaskManager := 
cron.NewScheduledTaskManager(siteInfoCommonService, questionService, 
fileRecordService, serviceConf)
        application := newApplication(serverConf, ginEngine, 
scheduledTaskManager)
        return application, func() {
                cleanup2()
diff --git a/configs/config.yaml b/configs/config.yaml
index 52671010..69c4e303 100644
--- a/configs/config.yaml
+++ b/configs/config.yaml
@@ -33,6 +33,9 @@ swaggerui:
   address: ':80'
 service_config:
   upload_path: "/data/uploads"
+  clean_up_uploads: true
+  clean_orphan_uploads_period_hours: 48
+  purge_deleted_files_period_days: 30
 ui:
   public_url: '/'
   api_url: '/'
diff --git a/go.mod b/go.mod
index 7e207e6b..89c6876b 100644
--- a/go.mod
+++ b/go.mod
@@ -60,6 +60,7 @@ require (
        golang.org/x/crypto v0.27.0
        golang.org/x/image v0.20.0
        golang.org/x/net v0.29.0
+       golang.org/x/text v0.18.0
        gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
        gopkg.in/yaml.v3 v3.0.1
        modernc.org/sqlite v1.33.0
@@ -160,7 +161,6 @@ require (
        golang.org/x/arch v0.10.0 // indirect
        golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
        golang.org/x/sys v0.25.0 // indirect
-       golang.org/x/text v0.18.0 // indirect
        golang.org/x/tools v0.25.0 // indirect
        google.golang.org/protobuf v1.34.2 // indirect
        gopkg.in/alexcesaro/quotedprintable.v3 
v3.0.0-20150716171945-2caba252f4dc // indirect
diff --git a/internal/base/constant/upload.go b/internal/base/constant/upload.go
index c01c9279..d9001fde 100644
--- a/internal/base/constant/upload.go
+++ b/internal/base/constant/upload.go
@@ -25,4 +25,5 @@ const (
        PostSubPath        = "post"
        BrandingSubPath    = "branding"
        FilesPostSubPath   = "files/post"
+       DeletedSubPath     = "deleted"
 )
diff --git a/internal/base/cron/cron.go b/internal/base/cron/cron.go
index 2e7a92ab..98e59dec 100644
--- a/internal/base/cron/cron.go
+++ b/internal/base/cron/cron.go
@@ -24,6 +24,8 @@ import (
        "fmt"
 
        "github.com/apache/answer/internal/service/content"
+       "github.com/apache/answer/internal/service/file_record"
+       "github.com/apache/answer/internal/service/service_config"
        "github.com/apache/answer/internal/service/siteinfo_common"
        "github.com/robfig/cron/v3"
        "github.com/segmentfault/pacman/log"
@@ -31,29 +33,36 @@ import (
 
 // ScheduledTaskManager scheduled task manager
 type ScheduledTaskManager struct {
-       siteInfoService siteinfo_common.SiteInfoCommonService
-       questionService *content.QuestionService
+       siteInfoService   siteinfo_common.SiteInfoCommonService
+       questionService   *content.QuestionService
+       fileRecordService *file_record.FileRecordService
+       serviceConfig     *service_config.ServiceConfig
 }
 
 // NewScheduledTaskManager new scheduled task manager
 func NewScheduledTaskManager(
        siteInfoService siteinfo_common.SiteInfoCommonService,
        questionService *content.QuestionService,
+       fileRecordService *file_record.FileRecordService,
+       serviceConfig *service_config.ServiceConfig,
 ) *ScheduledTaskManager {
        manager := &ScheduledTaskManager{
-               siteInfoService: siteInfoService,
-               questionService: questionService,
+               siteInfoService:   siteInfoService,
+               questionService:   questionService,
+               fileRecordService: fileRecordService,
+               serviceConfig:     serviceConfig,
        }
        return manager
 }
 
 func (s *ScheduledTaskManager) Run() {
-       fmt.Println("start cron")
+       log.Infof("cron job manager start")
+
        s.questionService.SitemapCron(context.Background())
        c := cron.New()
        _, err := c.AddFunc("0 */1 * * *", func() {
                ctx := context.Background()
-               fmt.Println("sitemap cron execution")
+               log.Infof("sitemap cron execution")
                s.questionService.SitemapCron(ctx)
        })
        if err != nil {
@@ -62,12 +71,32 @@ func (s *ScheduledTaskManager) Run() {
 
        _, err = c.AddFunc("0 */1 * * *", func() {
                ctx := context.Background()
-               fmt.Println("refresh hottest cron execution")
+               log.Infof("refresh hottest cron execution")
                s.questionService.RefreshHottestCron(ctx)
        })
        if err != nil {
                log.Error(err)
        }
 
+       if s.serviceConfig.CleanUpUploads {
+               log.Infof("clean up uploads cron enabled")
+
+               conf := s.serviceConfig
+               _, err = c.AddFunc(fmt.Sprintf("0 */%d * * *", 
conf.CleanOrphanUploadsPeriodHours), func() {
+                       log.Infof("clean orphan upload files cron execution")
+                       
s.fileRecordService.CleanOrphanUploadFiles(context.Background())
+               })
+               if err != nil {
+                       log.Error(err)
+               }
+
+               _, err = c.AddFunc(fmt.Sprintf("0 0 */%d * *", 
conf.PurgeDeletedFilesPeriodDays), func() {
+                       log.Infof("purge deleted files cron execution")
+                       
s.fileRecordService.PurgeDeletedFiles(context.Background())
+               })
+               if err != nil {
+                       log.Error(err)
+               }
+       }
        c.Start()
 }
diff --git a/internal/controller/upload_controller.go 
b/internal/controller/upload_controller.go
index 0bd74bf3..337cf83d 100644
--- a/internal/controller/upload_controller.go
+++ b/internal/controller/upload_controller.go
@@ -70,19 +70,20 @@ func (uc *UploadController) UploadFile(ctx *gin.Context) {
        )
 
        source := ctx.PostForm("source")
+       userID := middleware.GetLoginUserIDFromContext(ctx)
        switch source {
        case fileFromAvatar:
-               url, err = uc.uploaderService.UploadAvatarFile(ctx)
+               url, err = uc.uploaderService.UploadAvatarFile(ctx, userID)
        case fileFromPost:
-               url, err = uc.uploaderService.UploadPostFile(ctx)
+               url, err = uc.uploaderService.UploadPostFile(ctx, userID)
        case fileFromBranding:
                if !middleware.GetIsAdminFromContext(ctx) {
                        handler.HandleResponse(ctx, 
errors.Forbidden(reason.ForbiddenError), nil)
                        return
                }
-               url, err = uc.uploaderService.UploadBrandingFile(ctx)
+               url, err = uc.uploaderService.UploadBrandingFile(ctx, userID)
        case fileFromPostAttachment:
-               url, err = uc.uploaderService.UploadPostAttachment(ctx)
+               url, err = uc.uploaderService.UploadPostAttachment(ctx, userID)
        default:
                handler.HandleResponse(ctx, 
errors.BadRequest(reason.UploadFileSourceUnsupported), nil)
                return
diff --git a/internal/entity/file_record_entity.go 
b/internal/entity/file_record_entity.go
new file mode 100644
index 00000000..83a917aa
--- /dev/null
+++ b/internal/entity/file_record_entity.go
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package entity
+
+import "time"
+
+const (
+       FileRecordStatusAvailable = 1
+       FileRecordStatusDeleted   = 10
+)
+
+// FileRecord file record
+type FileRecord struct {
+       ID        int       `xorm:"not null pk autoincr INT(10) id"`
+       CreatedAt time.Time `xorm:"not null default CURRENT_TIMESTAMP created 
TIMESTAMP created_at"`
+       UpdatedAt time.Time `xorm:"not null default CURRENT_TIMESTAMP updated 
TIMESTAMP updated_at"`
+       UserID    string    `xorm:"not null default 0 BIGINT(20) user_id"`
+       FilePath  string    `xorm:"not null VARCHAR(256) file_path"`
+       FileURL   string    `xorm:"not null VARCHAR(1024) file_url"`
+       ObjectID  string    `xorm:"not null default 0 INDEX BIGINT(20) 
object_id"`
+       Source    string    `xorm:"not null VARCHAR(128) source"`
+       Status    int       `xorm:"not null default 0 TINYINT(4) status"`
+}
+
+// TableName file record table name
+func (FileRecord) TableName() string {
+       return "file_record"
+}
diff --git a/internal/migrations/migrations.go 
b/internal/migrations/migrations.go
index bc2ecda7..57f4778e 100644
--- a/internal/migrations/migrations.go
+++ b/internal/migrations/migrations.go
@@ -100,6 +100,7 @@ var migrations = []Migration{
        NewMigration("v1.4.0", "add badge/badge_group/badge_award table", 
addBadges, true),
        NewMigration("v1.4.1", "add question link", addQuestionLink, true),
        NewMigration("v1.4.2", "add the number of question links", 
addQuestionLinkedCount, true),
+       NewMigration("v1.4.5", "add file record", addFileRecord, true),
 }
 
 func GetMigrations() []Migration {
diff --git a/internal/service/service_config/service_config.go 
b/internal/migrations/v25.go
similarity index 78%
copy from internal/service/service_config/service_config.go
copy to internal/migrations/v25.go
index c5f27669..a0e1f1c2 100644
--- a/internal/service/service_config/service_config.go
+++ b/internal/migrations/v25.go
@@ -17,8 +17,15 @@
  * under the License.
  */
 
-package service_config
+package migrations
 
-type ServiceConfig struct {
-       UploadPath string `json:"upload_path" mapstructure:"upload_path" 
yaml:"upload_path"`
+import (
+       "context"
+
+       "github.com/apache/answer/internal/entity"
+       "xorm.io/xorm"
+)
+
+func addFileRecord(ctx context.Context, x *xorm.Engine) error {
+       return x.Context(ctx).Sync(new(entity.FileRecord))
 }
diff --git a/internal/repo/answer/answer_repo.go 
b/internal/repo/answer/answer_repo.go
index 2c2787f6..cdfc5802 100644
--- a/internal/repo/answer/answer_repo.go
+++ b/internal/repo/answer/answer_repo.go
@@ -529,7 +529,24 @@ func (ar *answerRepo) updateSearch(ctx context.Context, 
answerID string) (err er
 }
 
 func (ar *answerRepo) DeletePermanentlyAnswers(ctx context.Context) error {
-       _, err := ar.data.DB.Context(ctx).Where("status = ?", 
entity.AnswerStatusDeleted).Delete(&entity.Answer{})
+       // get all deleted answers ids
+       ids := make([]string, 0)
+       err := 
ar.data.DB.Context(ctx).Select("id").Table(new(entity.Answer).TableName()).
+               Where("status = ?", entity.AnswerStatusDeleted).Find(&ids)
+       if err != nil {
+               return 
errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+       }
+       if len(ids) == 0 {
+               return nil
+       }
+
+       // delete all revisions permanently
+       _, err = ar.data.DB.Context(ctx).In("object_id", 
ids).Delete(&entity.Revision{})
+       if err != nil {
+               return 
errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+       }
+
+       _, err = ar.data.DB.Context(ctx).Where("status = ?", 
entity.AnswerStatusDeleted).Delete(&entity.Answer{})
        if err != nil {
                return 
errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
        }
diff --git a/internal/repo/file_record/file_record_repo.go 
b/internal/repo/file_record/file_record_repo.go
new file mode 100644
index 00000000..ed081be4
--- /dev/null
+++ b/internal/repo/file_record/file_record_repo.go
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package file_record
+
+import (
+       "context"
+
+       "github.com/apache/answer/internal/base/pager"
+       "github.com/apache/answer/internal/service/file_record"
+
+       "github.com/apache/answer/internal/base/data"
+       "github.com/apache/answer/internal/base/reason"
+       "github.com/apache/answer/internal/entity"
+       "github.com/segmentfault/pacman/errors"
+)
+
+// fileRecordRepo fileRecord repository
+type fileRecordRepo struct {
+       data *data.Data
+}
+
+// NewFileRecordRepo new repository
+func NewFileRecordRepo(data *data.Data) file_record.FileRecordRepo {
+       return &fileRecordRepo{
+               data: data,
+       }
+}
+
+// AddFileRecord add file record
+func (fr *fileRecordRepo) AddFileRecord(ctx context.Context, fileRecord 
*entity.FileRecord) (err error) {
+       _, err = fr.data.DB.Context(ctx).Insert(fileRecord)
+       if err != nil {
+               err = 
errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+       }
+       return
+}
+
+// GetFileRecordPage get fileRecord page
+func (fr *fileRecordRepo) GetFileRecordPage(ctx context.Context, page, 
pageSize int, cond *entity.FileRecord) (
+       fileRecordList []*entity.FileRecord, total int64, err error) {
+       fileRecordList = make([]*entity.FileRecord, 0)
+
+       session := fr.data.DB.Context(ctx)
+       total, err = pager.Help(page, pageSize, &fileRecordList, cond, session)
+       if err != nil {
+               err = 
errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+       }
+       return
+}
+
+// DeleteFileRecord delete file record
+func (fr *fileRecordRepo) DeleteFileRecord(ctx context.Context, id int) (err 
error) {
+       _, err = 
fr.data.DB.Context(ctx).ID(id).Cols("status").Update(&entity.FileRecord{Status: 
entity.FileRecordStatusDeleted})
+       if err != nil {
+               err = 
errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+       }
+       return
+}
+
+// UpdateFileRecord update file record
+func (fr *fileRecordRepo) UpdateFileRecord(ctx context.Context, fileRecord 
*entity.FileRecord) (err error) {
+       _, err = fr.data.DB.Context(ctx).ID(fileRecord.ID).Update(fileRecord)
+       if err != nil {
+               err = 
errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+       }
+       return
+}
diff --git a/internal/repo/provider.go b/internal/repo/provider.go
index 1b29cc5e..02f27f62 100644
--- a/internal/repo/provider.go
+++ b/internal/repo/provider.go
@@ -33,6 +33,7 @@ import (
        "github.com/apache/answer/internal/repo/comment"
        "github.com/apache/answer/internal/repo/config"
        "github.com/apache/answer/internal/repo/export"
+       "github.com/apache/answer/internal/repo/file_record"
        "github.com/apache/answer/internal/repo/limit"
        "github.com/apache/answer/internal/repo/meta"
        "github.com/apache/answer/internal/repo/notification"
@@ -107,4 +108,5 @@ var ProviderSetRepo = wire.NewSet(
        badge.NewEventRuleRepo,
        badge_group.NewBadgeGroupRepo,
        badge_award.NewBadgeAwardRepo,
+       file_record.NewFileRecordRepo,
 )
diff --git a/internal/repo/question/question_repo.go 
b/internal/repo/question/question_repo.go
index 117ee38f..978c9ce3 100644
--- a/internal/repo/question/question_repo.go
+++ b/internal/repo/question/question_repo.go
@@ -168,6 +168,23 @@ func (qr *questionRepo) 
UpdateQuestionStatusWithOutUpdateTime(ctx context.Contex
 }
 
 func (qr *questionRepo) DeletePermanentlyQuestions(ctx context.Context) (err 
error) {
+       // get all deleted question ids
+       ids := make([]string, 0)
+       err = 
qr.data.DB.Context(ctx).Select("id").Table(new(entity.Question).TableName()).
+               Where("status = ?", entity.QuestionStatusDeleted).Find(&ids)
+       if err != nil {
+               return 
errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+       }
+       if len(ids) == 0 {
+               return nil
+       }
+
+       // delete all revisions permanently
+       _, err = qr.data.DB.Context(ctx).In("object_id", 
ids).Delete(&entity.Revision{})
+       if err != nil {
+               return 
errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+       }
+
        _, err = qr.data.DB.Context(ctx).Where("status = ?", 
entity.QuestionStatusDeleted).Delete(&entity.Question{})
        if err != nil {
                return 
errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
diff --git a/internal/repo/revision/revision_repo.go 
b/internal/repo/revision/revision_repo.go
index ab00adf9..09ba1aac 100644
--- a/internal/repo/revision/revision_repo.go
+++ b/internal/repo/revision/revision_repo.go
@@ -155,7 +155,17 @@ func (rr *revisionRepo) GetLastRevisionByObjectID(ctx 
context.Context, objectID
        revision *entity.Revision, exist bool, err error,
 ) {
        revision = &entity.Revision{}
-       exist, err = rr.data.DB.Context(ctx).Where("object_id = ?", 
objectID).OrderBy("created_at DESC").Get(revision)
+       exist, err = rr.data.DB.Context(ctx).Where("object_id = ?", 
objectID).Desc("created_at").Get(revision)
+       if err != nil {
+               return nil, false, 
errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+       }
+       return
+}
+
+// GetLastRevisionByFileURL get object's last revision by file url
+func (rr *revisionRepo) GetLastRevisionByFileURL(ctx context.Context, fileURL 
string) (revision *entity.Revision, exist bool, err error) {
+       revision = &entity.Revision{}
+       exist, err = rr.data.DB.Context(ctx).Where("content LIKE ?", 
"%"+fileURL+"%").Desc("created_at").Get(revision)
        if err != nil {
                return nil, false, 
errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
        }
diff --git a/internal/router/static_router.go b/internal/router/static_router.go
index 807f4c73..a6c80fc0 100644
--- a/internal/router/static_router.go
+++ b/internal/router/static_router.go
@@ -20,11 +20,14 @@
 package router
 
 import (
+       "net/http"
+       "path/filepath"
+       "strings"
+
        "github.com/apache/answer/internal/base/constant"
        "github.com/apache/answer/internal/service/service_config"
+       "github.com/apache/answer/pkg/dir"
        "github.com/gin-gonic/gin"
-       "path/filepath"
-       "strings"
 )
 
 // StaticRouter static api router
@@ -54,6 +57,11 @@ func (a *StaticRouter) RegisterStaticRouter(r 
*gin.RouterGroup) {
                realFilename := strings.TrimSuffix(filePath, 
"/"+originalFilename) + filepath.Ext(originalFilename)
                // The file local path is /uploads/files/post/hash.pdf
                fileLocalPath := filepath.Join(a.serviceConfig.UploadPath, 
constant.FilesPostSubPath, realFilename)
+               // If the file is not exist, return 404
+               if !dir.CheckFileExist(fileLocalPath) {
+                       c.Redirect(http.StatusFound, "/404")
+                       return
+               }
                c.FileAttachment(fileLocalPath, originalFilename)
        })
 }
diff --git a/internal/service/content/question_service.go 
b/internal/service/content/question_service.go
index 109b6b10..f3641dcc 100644
--- a/internal/service/content/question_service.go
+++ b/internal/service/content/question_service.go
@@ -1398,6 +1398,8 @@ func (qs *QuestionService) GetQuestionPage(ctx 
context.Context, req *schema.Ques
                                return nil, 0, err
                        }
                        tagIDs = append(synTagIds, tagInfo.ID)
+               } else {
+                       return questions, 0, nil
                }
        }
 
diff --git a/internal/service/file_record/file_record_service.go 
b/internal/service/file_record/file_record_service.go
new file mode 100644
index 00000000..abb98376
--- /dev/null
+++ b/internal/service/file_record/file_record_service.go
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package file_record
+
+import (
+       "context"
+       "fmt"
+       "os"
+       "path/filepath"
+       "time"
+
+       "github.com/apache/answer/internal/base/constant"
+       "github.com/apache/answer/internal/entity"
+       "github.com/apache/answer/internal/service/revision"
+       "github.com/apache/answer/internal/service/service_config"
+       "github.com/apache/answer/internal/service/siteinfo_common"
+       "github.com/apache/answer/pkg/checker"
+       "github.com/apache/answer/pkg/dir"
+       "github.com/apache/answer/pkg/writer"
+       "github.com/segmentfault/pacman/log"
+)
+
+// FileRecordRepo file record repository
+type FileRecordRepo interface {
+       AddFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err 
error)
+       UpdateFileRecord(ctx context.Context, fileRecord *entity.FileRecord) 
(err error)
+       GetFileRecordPage(ctx context.Context, page, pageSize int, cond 
*entity.FileRecord) (
+               fileRecordList []*entity.FileRecord, total int64, err error)
+       DeleteFileRecord(ctx context.Context, id int) (err error)
+}
+
+// FileRecordService file record service
+type FileRecordService struct {
+       fileRecordRepo  FileRecordRepo
+       revisionRepo    revision.RevisionRepo
+       serviceConfig   *service_config.ServiceConfig
+       siteInfoService siteinfo_common.SiteInfoCommonService
+}
+
+// NewFileRecordService new file record service
+func NewFileRecordService(
+       fileRecordRepo FileRecordRepo,
+       revisionRepo revision.RevisionRepo,
+       serviceConfig *service_config.ServiceConfig,
+       siteInfoService siteinfo_common.SiteInfoCommonService,
+) *FileRecordService {
+       return &FileRecordService{
+               fileRecordRepo:  fileRecordRepo,
+               revisionRepo:    revisionRepo,
+               serviceConfig:   serviceConfig,
+               siteInfoService: siteInfoService,
+       }
+}
+
+// AddFileRecord add file record
+func (fs *FileRecordService) AddFileRecord(ctx context.Context, userID, 
filePath, fileURL, source string) {
+       record := &entity.FileRecord{
+               UserID:   userID,
+               FilePath: filePath,
+               FileURL:  fileURL,
+               Source:   source,
+               Status:   entity.FileRecordStatusAvailable,
+               ObjectID: "0",
+       }
+       if err := fs.fileRecordRepo.AddFileRecord(ctx, record); err != nil {
+               log.Errorf("add file record error: %v", err)
+       }
+}
+
+// CleanOrphanUploadFiles clean orphan upload files
+func (fs *FileRecordService) CleanOrphanUploadFiles(ctx context.Context) {
+       page, pageSize := 1, 1000
+
+       for {
+               fileRecordList, total, err := 
fs.fileRecordRepo.GetFileRecordPage(ctx, page, pageSize, &entity.FileRecord{
+                       Status: entity.FileRecordStatusAvailable,
+               })
+               if err != nil {
+                       log.Errorf("get file record page error: %v", err)
+                       return
+               }
+               if len(fileRecordList) == 0 || total == 0 {
+                       break
+               }
+               for _, fileRecord := range fileRecordList {
+                       // If this file record created in 48 hours, no need to 
check
+                       if fileRecord.CreatedAt.AddDate(0, 0, 
2).After(time.Now()) {
+                               continue
+                       }
+                       if checker.IsNotZeroString(fileRecord.ObjectID) {
+                               _, exist, err := 
fs.revisionRepo.GetLastRevisionByObjectID(ctx, fileRecord.ObjectID)
+                               if err != nil {
+                                       log.Errorf("get last revision by object 
id error: %v", err)
+                                       continue
+                               }
+                               if exist {
+                                       continue
+                               }
+                       } else {
+                               lastRevision, exist, err := 
fs.revisionRepo.GetLastRevisionByFileURL(ctx, fileRecord.FileURL)
+                               if err != nil {
+                                       log.Errorf("get last revision by file 
url error: %v", err)
+                                       continue
+                               }
+                               if exist {
+                                       // update the file record object id
+                                       fileRecord.ObjectID = 
lastRevision.ObjectID
+                                       if err := 
fs.fileRecordRepo.UpdateFileRecord(ctx, fileRecord); err != nil {
+                                               log.Errorf("update file record 
object id error: %v", err)
+                                       }
+                                       continue
+                               }
+                       }
+                       // Delete and move the file record
+                       if err := fs.deleteAndMoveFileRecord(ctx, fileRecord); 
err != nil {
+                               log.Error(err)
+                       }
+               }
+               page++
+       }
+}
+
+func (fs *FileRecordService) PurgeDeletedFiles(ctx context.Context) {
+       deletedPath := filepath.Join(fs.serviceConfig.UploadPath, 
constant.DeletedSubPath)
+       log.Infof("purge deleted files: %s", deletedPath)
+       err := os.RemoveAll(deletedPath)
+       if err != nil {
+               log.Errorf("purge deleted files error: %v", err)
+               return
+       }
+       err = dir.CreateDirIfNotExist(deletedPath)
+       if err != nil {
+               log.Errorf("create deleted directory error: %v", err)
+       }
+       return
+}
+
+func (fs *FileRecordService) deleteAndMoveFileRecord(ctx context.Context, 
fileRecord *entity.FileRecord) error {
+       // Delete the file record
+       if err := fs.fileRecordRepo.DeleteFileRecord(ctx, fileRecord.ID); err 
!= nil {
+               return fmt.Errorf("delete file record error: %v", err)
+       }
+
+       // Move the file to the deleted directory
+       oldFilename := filepath.Base(fileRecord.FilePath)
+       oldFilePath := filepath.Join(fs.serviceConfig.UploadPath, 
fileRecord.FilePath)
+       deletedPath := filepath.Join(fs.serviceConfig.UploadPath, 
constant.DeletedSubPath, oldFilename)
+
+       if err := writer.MoveFile(oldFilePath, deletedPath); err != nil {
+               return fmt.Errorf("move file error: %v", err)
+       }
+
+       log.Debugf("delete and move file: %s", fileRecord.FileURL)
+       return nil
+}
diff --git a/internal/service/object_info/object_info.go 
b/internal/service/object_info/object_info.go
index 76b1d86d..5ef438ff 100644
--- a/internal/service/object_info/object_info.go
+++ b/internal/service/object_info/object_info.go
@@ -21,6 +21,7 @@ package object_info
 
 import (
        "context"
+
        "github.com/apache/answer/internal/base/constant"
        "github.com/apache/answer/internal/base/reason"
        "github.com/apache/answer/internal/schema"
@@ -279,7 +280,7 @@ func (os *ObjService) GetInfo(ctx context.Context, objectID 
string) (objInfo *sc
                        ObjectID:   tagInfo.ID,
                        TagID:      tagInfo.ID,
                        ObjectType: objectType,
-                       Title:      tagInfo.ParsedText,
+                       Title:      tagInfo.SlugName,
                        Content:    tagInfo.ParsedText, // todo trim
                }
        }
diff --git a/internal/service/provider.go b/internal/service/provider.go
index 84073235..4b1b6427 100644
--- a/internal/service/provider.go
+++ b/internal/service/provider.go
@@ -36,6 +36,7 @@ import (
        "github.com/apache/answer/internal/service/dashboard"
        "github.com/apache/answer/internal/service/event_queue"
        "github.com/apache/answer/internal/service/export"
+       "github.com/apache/answer/internal/service/file_record"
        "github.com/apache/answer/internal/service/follow"
        "github.com/apache/answer/internal/service/importer"
        "github.com/apache/answer/internal/service/meta"
@@ -126,4 +127,5 @@ var ProviderSetService = wire.NewSet(
        badge.NewBadgeAwardService,
        badge.NewBadgeGroupService,
        importer.NewImporterService,
+       file_record.NewFileRecordService,
 )
diff --git a/internal/service/revision/revision.go 
b/internal/service/revision/revision.go
index 58427220..33ce285e 100644
--- a/internal/service/revision/revision.go
+++ b/internal/service/revision/revision.go
@@ -31,6 +31,7 @@ type RevisionRepo interface {
        AddRevision(ctx context.Context, revision *entity.Revision, 
autoUpdateRevisionID bool) (err error)
        GetRevisionByID(ctx context.Context, revisionID string) (revision 
*entity.Revision, exist bool, err error)
        GetLastRevisionByObjectID(ctx context.Context, objectID string) 
(revision *entity.Revision, exist bool, err error)
+       GetLastRevisionByFileURL(ctx context.Context, fileURL string) (revision 
*entity.Revision, exist bool, err error)
        GetRevisionList(ctx context.Context, revision *entity.Revision) 
(revisionList []entity.Revision, err error)
        UpdateObjectRevisionId(ctx context.Context, revision *entity.Revision, 
session *xorm.Session) (err error)
        ExistUnreviewedByObjectID(ctx context.Context, objectID string) 
(revision *entity.Revision, exist bool, err error)
diff --git a/internal/service/service_config/service_config.go 
b/internal/service/service_config/service_config.go
index c5f27669..90e399e1 100644
--- a/internal/service/service_config/service_config.go
+++ b/internal/service/service_config/service_config.go
@@ -20,5 +20,8 @@
 package service_config
 
 type ServiceConfig struct {
-       UploadPath string `json:"upload_path" mapstructure:"upload_path" 
yaml:"upload_path"`
+       UploadPath                    string `json:"upload_path" 
mapstructure:"upload_path" yaml:"upload_path"`
+       CleanUpUploads                bool   `json:"clean_up_uploads" 
mapstructure:"clean_up_uploads" yaml:"clean_up_uploads"`
+       CleanOrphanUploadsPeriodHours int    
`json:"clean_orphan_uploads_period_hours" 
mapstructure:"clean_orphan_uploads_period_hours" 
yaml:"clean_orphan_uploads_period_hours"`
+       PurgeDeletedFilesPeriodDays   int    
`json:"purge_deleted_files_period_days" 
mapstructure:"purge_deleted_files_period_days" 
yaml:"purge_deleted_files_period_days"`
 }
diff --git a/internal/service/tag_common/tag_common.go 
b/internal/service/tag_common/tag_common.go
index e5c136c3..4ce89975 100644
--- a/internal/service/tag_common/tag_common.go
+++ b/internal/service/tag_common/tag_common.go
@@ -349,10 +349,17 @@ func (ts *TagCommonService) AddTag(ctx context.Context, 
req *schema.AddTagReq) (
        }
        tagInfoJson, _ := json.Marshal(tagInfo)
        revisionDTO.Content = string(tagInfoJson)
-       _, err = ts.revisionService.AddRevision(ctx, revisionDTO, true)
+       revisionID, err := ts.revisionService.AddRevision(ctx, revisionDTO, 
true)
        if err != nil {
                return nil, err
        }
+       ts.activityQueueService.Send(ctx, &schema.ActivityMsg{
+               UserID:           req.UserID,
+               ObjectID:         tagInfo.ID,
+               OriginalObjectID: tagInfo.ID,
+               ActivityTypeKey:  constant.ActTagCreated,
+               RevisionID:       revisionID,
+       })
        return &schema.AddTagResp{SlugName: tagInfo.SlugName}, nil
 }
 
diff --git a/internal/service/uploader/upload.go 
b/internal/service/uploader/upload.go
index 4b018cb4..d5cc2dfb 100644
--- a/internal/service/uploader/upload.go
+++ b/internal/service/uploader/upload.go
@@ -31,6 +31,8 @@ import (
        "path/filepath"
        "strings"
 
+       "github.com/apache/answer/internal/service/file_record"
+
        "github.com/apache/answer/internal/base/constant"
        "github.com/apache/answer/internal/base/reason"
        "github.com/apache/answer/internal/service/service_config"
@@ -53,6 +55,7 @@ var (
                constant.PostSubPath,
                constant.BrandingSubPath,
                constant.FilesPostSubPath,
+               constant.DeletedSubPath,
        }
        supportedThumbFileExtMapping = map[string]imaging.Format{
                ".jpg":  imaging.JPEG,
@@ -63,22 +66,26 @@ var (
 )
 
 type UploaderService interface {
-       UploadAvatarFile(ctx *gin.Context) (url string, err error)
-       UploadPostFile(ctx *gin.Context) (url string, err error)
-       UploadPostAttachment(ctx *gin.Context) (url string, err error)
-       UploadBrandingFile(ctx *gin.Context) (url string, err error)
+       UploadAvatarFile(ctx *gin.Context, userID string) (url string, err 
error)
+       UploadPostFile(ctx *gin.Context, userID string) (url string, err error)
+       UploadPostAttachment(ctx *gin.Context, userID string) (url string, err 
error)
+       UploadBrandingFile(ctx *gin.Context, userID string) (url string, err 
error)
        AvatarThumbFile(ctx *gin.Context, fileName string, size int) (url 
string, err error)
 }
 
 // uploaderService uploader service
 type uploaderService struct {
-       serviceConfig   *service_config.ServiceConfig
-       siteInfoService siteinfo_common.SiteInfoCommonService
+       serviceConfig     *service_config.ServiceConfig
+       siteInfoService   siteinfo_common.SiteInfoCommonService
+       fileRecordService *file_record.FileRecordService
 }
 
 // NewUploaderService new upload service
-func NewUploaderService(serviceConfig *service_config.ServiceConfig,
-       siteInfoService siteinfo_common.SiteInfoCommonService) UploaderService {
+func NewUploaderService(
+       serviceConfig *service_config.ServiceConfig,
+       siteInfoService siteinfo_common.SiteInfoCommonService,
+       fileRecordService *file_record.FileRecordService,
+) UploaderService {
        for _, subPath := range subPathList {
                err := 
dir.CreateDirIfNotExist(filepath.Join(serviceConfig.UploadPath, subPath))
                if err != nil {
@@ -86,13 +93,14 @@ func NewUploaderService(serviceConfig 
*service_config.ServiceConfig,
                }
        }
        return &uploaderService{
-               serviceConfig:   serviceConfig,
-               siteInfoService: siteInfoService,
+               serviceConfig:     serviceConfig,
+               siteInfoService:   siteInfoService,
+               fileRecordService: fileRecordService,
        }
 }
 
 // UploadAvatarFile upload avatar file
-func (us *uploaderService) UploadAvatarFile(ctx *gin.Context) (url string, err 
error) {
+func (us *uploaderService) UploadAvatarFile(ctx *gin.Context, userID string) 
(url string, err error) {
        url, err = us.tryToUploadByPlugin(ctx, plugin.UserAvatar)
        if err != nil {
                return "", err
@@ -174,7 +182,7 @@ func (us *uploaderService) AvatarThumbFile(ctx 
*gin.Context, fileName string, si
        return saveFilePath, nil
 }
 
-func (us *uploaderService) UploadPostFile(ctx *gin.Context) (
+func (us *uploaderService) UploadPostFile(ctx *gin.Context, userID string) (
        url string, err error) {
        url, err = us.tryToUploadByPlugin(ctx, plugin.UserPost)
        if err != nil {
@@ -202,10 +210,15 @@ func (us *uploaderService) UploadPostFile(ctx 
*gin.Context) (
        fileExt := strings.ToLower(path.Ext(fileHeader.Filename))
        newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt)
        avatarFilePath := path.Join(constant.PostSubPath, newFilename)
-       return us.uploadImageFile(ctx, fileHeader, avatarFilePath)
+       url, err = us.uploadImageFile(ctx, fileHeader, avatarFilePath)
+       if err != nil {
+               return "", err
+       }
+       us.fileRecordService.AddFileRecord(ctx, userID, avatarFilePath, url, 
string(plugin.UserPost))
+       return url, nil
 }
 
-func (us *uploaderService) UploadPostAttachment(ctx *gin.Context) (
+func (us *uploaderService) UploadPostAttachment(ctx *gin.Context, userID 
string) (
        url string, err error) {
        url, err = us.tryToUploadByPlugin(ctx, plugin.UserPostAttachment)
        if err != nil {
@@ -232,11 +245,16 @@ func (us *uploaderService) UploadPostAttachment(ctx 
*gin.Context) (
 
        fileExt := strings.ToLower(path.Ext(fileHeader.Filename))
        newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt)
-       avatarFilePath := path.Join(constant.FilesPostSubPath, newFilename)
-       return us.uploadAttachmentFile(ctx, fileHeader, fileHeader.Filename, 
avatarFilePath)
+       attachmentFilePath := path.Join(constant.FilesPostSubPath, newFilename)
+       url, err = us.uploadAttachmentFile(ctx, fileHeader, 
fileHeader.Filename, attachmentFilePath)
+       if err != nil {
+               return "", err
+       }
+       us.fileRecordService.AddFileRecord(ctx, userID, attachmentFilePath, 
url, string(plugin.UserPostAttachment))
+       return url, nil
 }
 
-func (us *uploaderService) UploadBrandingFile(ctx *gin.Context) (
+func (us *uploaderService) UploadBrandingFile(ctx *gin.Context, userID string) 
(
        url string, err error) {
        url, err = us.tryToUploadByPlugin(ctx, plugin.AdminBranding)
        if err != nil {
diff --git a/pkg/writer/writer.go b/pkg/writer/writer.go
index 49ded3df..ff08d259 100644
--- a/pkg/writer/writer.go
+++ b/pkg/writer/writer.go
@@ -48,3 +48,8 @@ func WriteFile(filePath, content string) error {
        }
        return nil
 }
+
+// MoveFile move file to new path
+func MoveFile(oldPath, newPath string) error {
+       return os.Rename(oldPath, newPath)
+}


Reply via email to