This is an automated email from the ASF dual-hosted git repository. linkinstar pushed a commit to branch feat/1.4.2/file in repository https://gitbox.apache.org/repos/asf/incubator-answer.git
commit 932a33666db437767459a8da934126d659e1de1f Author: LinkinStars <[email protected]> AuthorDate: Thu Nov 21 15:53:58 2024 +0800 feat(upload): add support for file attachments and enhance image upload --- internal/base/constant/site_info.go | 6 +++ internal/controller/upload_controller.go | 6 ++- internal/migrations/init.go | 6 ++- internal/migrations/v24.go | 38 +++++++++++++++ internal/schema/siteinfo_schema.go | 36 +++++++++++++-- internal/service/uploader/upload.go | 79 ++++++++++++++++++++++++++++---- pkg/checker/file_type.go | 53 +++++++++++---------- plugin/storage.go | 7 +-- 8 files changed, 186 insertions(+), 45 deletions(-) diff --git a/internal/base/constant/site_info.go b/internal/base/constant/site_info.go index 0509c359..2d666834 100644 --- a/internal/base/constant/site_info.go +++ b/internal/base/constant/site_info.go @@ -48,3 +48,9 @@ const ( const ( EmailConfigKey = "email.config" ) + +const ( + DefaultMaxImageMegapixel = 40 * 1000 * 1000 + DefaultMaxImageSize = 4 * 1024 * 1024 + DefaultMaxAttachmentSize = 8 * 1024 * 1024 +) diff --git a/internal/controller/upload_controller.go b/internal/controller/upload_controller.go index 56eba804..a43ccf64 100644 --- a/internal/controller/upload_controller.go +++ b/internal/controller/upload_controller.go @@ -32,6 +32,8 @@ import ( const ( // file is uploaded by markdown(or something else) editor fileFromPost = "post" + // file is used to upload the post attachment + fileFromPostAttachment = "post_attachment" // file is used to change the user's avatar fileFromAvatar = "avatar" // file is logo/icon images @@ -56,7 +58,7 @@ func NewUploadController(uploaderService uploader.UploaderService) *UploadContro // @Tags Upload // @Accept multipart/form-data // @Security ApiKeyAuth -// @Param source formData string true "identify the source of the file upload" Enums(post, avatar, branding) +// @Param source formData string true "identify the source of the file upload" Enums(post, post_attachment, avatar, branding) // @Param file formData file true "file" // @Success 200 {object} handler.RespBody{data=string} // @Router /answer/api/v1/file [post] @@ -74,6 +76,8 @@ func (uc *UploadController) UploadFile(ctx *gin.Context) { url, err = uc.uploaderService.UploadPostFile(ctx) case fileFromBranding: url, err = uc.uploaderService.UploadBrandingFile(ctx) + case fileFromPostAttachment: + url, err = uc.uploaderService.UploadPostAttachment(ctx) default: handler.HandleResponse(ctx, errors.BadRequest(reason.UploadFileSourceUnsupported), nil) return diff --git a/internal/migrations/init.go b/internal/migrations/init.go index 4558ef60..6b1055a0 100644 --- a/internal/migrations/init.go +++ b/internal/migrations/init.go @@ -253,7 +253,11 @@ func (m *Mentor) initSiteInfoPrivilegeRank() { func (m *Mentor) initSiteInfoWrite() { writeData := map[string]interface{}{ - "restrict_answer": true, + "restrict_answer": true, + "max_image_size": 4, + "max_attachment_size": 8, + "max_image_megapixel": 40, + "authorized_extensions": []string{"jpg", "jpeg", "png", "gif", "webp"}, } writeDataBytes, _ := json.Marshal(writeData) _, m.err = m.engine.Context(m.ctx).Insert(&entity.SiteInfo{ diff --git a/internal/migrations/v24.go b/internal/migrations/v24.go index 015352ae..907fd1ef 100644 --- a/internal/migrations/v24.go +++ b/internal/migrations/v24.go @@ -21,12 +21,50 @@ package migrations import ( "context" + "encoding/json" + "fmt" + "github.com/apache/incubator-answer/internal/base/constant" "github.com/apache/incubator-answer/internal/entity" + "github.com/apache/incubator-answer/internal/schema" "xorm.io/xorm" ) func addQuestionLinkedCount(ctx context.Context, x *xorm.Engine) error { + writeSiteInfo := &entity.SiteInfo{ + Type: constant.SiteTypeWrite, + } + exist, err := x.Context(ctx).Get(writeSiteInfo) + if err != nil { + return fmt.Errorf("get config failed: %w", err) + } + if exist { + type OldSiteWriteReq struct { + RestrictAnswer bool `json:"restrict_answer"` + RequiredTag bool `json:"required_tag"` + RecommendTags []*schema.SiteWriteTag `json:"recommend_tags"` + ReservedTags []*schema.SiteWriteTag `json:"reserved_tags"` + MaxImageSize int `json:"max_image_size"` + MaxAttachmentSize int `json:"max_attachment_size"` + MaxImageMegapixel int `json:"max_image_megapixel"` + AuthorizedImageExtensions []string `json:"authorized_image_extensions"` + AuthorizedAttachmentExtensions []string `json:"authorized_attachment_extensions"` + } + content := &OldSiteWriteReq{} + _ = json.Unmarshal([]byte(writeSiteInfo.Content), content) + content.MaxImageSize = 4 + content.MaxAttachmentSize = 8 + content.MaxImageMegapixel = 40 + content.AuthorizedImageExtensions = []string{"jpg", "jpeg", "png", "gif", "webp"} + content.AuthorizedAttachmentExtensions = []string{} + data, _ := json.Marshal(content) + writeSiteInfo.Content = string(data) + _, err = x.Context(ctx).ID(writeSiteInfo.ID).Cols("content").Update(writeSiteInfo) + if err != nil { + return fmt.Errorf("update site info failed: %w", err) + } + } + type Question struct { LinkedCount int `xorm:"not null default 0 INT(11) linked_count"` } diff --git a/internal/schema/siteinfo_schema.go b/internal/schema/siteinfo_schema.go index 99266308..814465d9 100644 --- a/internal/schema/siteinfo_schema.go +++ b/internal/schema/siteinfo_schema.go @@ -72,11 +72,37 @@ type SiteBrandingReq struct { // SiteWriteReq site write request type SiteWriteReq struct { - RestrictAnswer bool `validate:"omitempty" json:"restrict_answer"` - RequiredTag bool `validate:"omitempty" json:"required_tag"` - RecommendTags []*SiteWriteTag `validate:"omitempty,dive" json:"recommend_tags"` - ReservedTags []*SiteWriteTag `validate:"omitempty,dive" json:"reserved_tags"` - UserID string `json:"-"` + RestrictAnswer bool `validate:"omitempty" json:"restrict_answer"` + RequiredTag bool `validate:"omitempty" json:"required_tag"` + RecommendTags []*SiteWriteTag `validate:"omitempty,dive" json:"recommend_tags"` + ReservedTags []*SiteWriteTag `validate:"omitempty,dive" json:"reserved_tags"` + MaxImageSize int `validate:"omitempty,gt=0" json:"max_image_size"` + MaxAttachmentSize int `validate:"omitempty,gt=0" json:"max_attachment_size"` + MaxImageMegapixel int `validate:"omitempty,gt=0" json:"max_image_megapixel"` + AuthorizedImageExtensions []string `validate:"omitempty,dive,gt=0,lte=128" json:"authorized_image_extensions"` + AuthorizedAttachmentExtensions []string `validate:"omitempty,dive,gt=0,lte=128" json:"authorized_attachment_extensions"` + UserID string `json:"-"` +} + +func (s *SiteWriteResp) GetMaxImageSize() int64 { + if s.MaxImageSize <= 0 { + return constant.DefaultMaxImageSize + } + return int64(s.MaxImageSize) * 1024 * 1024 +} + +func (s *SiteWriteResp) GetMaxAttachmentSize() int64 { + if s.MaxAttachmentSize <= 0 { + return constant.DefaultMaxAttachmentSize + } + return int64(s.MaxAttachmentSize) * 1024 * 1024 +} + +func (s *SiteWriteResp) GetMaxImageMegapixel() int { + if s.MaxImageMegapixel <= 0 { + return constant.DefaultMaxImageMegapixel + } + return s.MaxImageMegapixel * 1000 * 1000 } // SiteWriteTag site write response tag diff --git a/internal/service/uploader/upload.go b/internal/service/uploader/upload.go index 3619a6e1..9b810e53 100644 --- a/internal/service/uploader/upload.go +++ b/internal/service/uploader/upload.go @@ -69,6 +69,7 @@ var ( type UploaderService interface { UploadAvatarFile(ctx *gin.Context) (url string, err error) UploadPostFile(ctx *gin.Context) (url string, err error) + UploadPostAttachment(ctx *gin.Context) (url string, err error) UploadBrandingFile(ctx *gin.Context) (url string, err error) AvatarThumbFile(ctx *gin.Context, fileName string, size int) (url string, err error) } @@ -118,7 +119,7 @@ func (us *uploaderService) UploadAvatarFile(ctx *gin.Context) (url string, err e newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt) avatarFilePath := path.Join(avatarSubPath, newFilename) - return us.uploadFile(ctx, fileHeader, avatarFilePath) + return us.uploadImageFile(ctx, fileHeader, avatarFilePath) } func (us *uploaderService) AvatarThumbFile(ctx *gin.Context, fileName string, size int) (url string, err error) { @@ -183,21 +184,56 @@ func (us *uploaderService) UploadPostFile(ctx *gin.Context) ( return url, nil } - // max size - ctx.Request.Body = http.MaxBytesReader(ctx.Writer, ctx.Request.Body, 10*1024*1024) + siteWrite, err := us.siteInfoService.GetSiteWrite(ctx) + if err != nil { + return "", err + } + + ctx.Request.Body = http.MaxBytesReader(ctx.Writer, ctx.Request.Body, siteWrite.GetMaxImageSize()) file, fileHeader, err := ctx.Request.FormFile("file") if err != nil { return "", errors.BadRequest(reason.RequestFormatError).WithError(err) } defer file.Close() + if checker.IsUnAuthorizedExtension(fileHeader.Filename, siteWrite.AuthorizedImageExtensions) { + return "", errors.BadRequest(reason.RequestFormatError).WithError(err) + } + fileExt := strings.ToLower(path.Ext(fileHeader.Filename)) - if _, ok := plugin.DefaultFileTypeCheckMapping[plugin.UserPost][fileExt]; !ok { + newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt) + avatarFilePath := path.Join(postSubPath, newFilename) + return us.uploadImageFile(ctx, fileHeader, avatarFilePath) +} + +func (us *uploaderService) UploadPostAttachment(ctx *gin.Context) ( + url string, err error) { + url, err = us.tryToUploadByPlugin(ctx, plugin.UserPostAttachment) + if err != nil { + return "", err + } + if len(url) > 0 { + return url, nil + } + + resp, err := us.siteInfoService.GetSiteWrite(ctx) + if err != nil { + return "", err + } + + ctx.Request.Body = http.MaxBytesReader(ctx.Writer, ctx.Request.Body, resp.GetMaxAttachmentSize()) + file, fileHeader, err := ctx.Request.FormFile("file") + if err != nil { + return "", errors.BadRequest(reason.RequestFormatError).WithError(err) + } + defer file.Close() + if checker.IsUnAuthorizedExtension(fileHeader.Filename, resp.AuthorizedAttachmentExtensions) { return "", errors.BadRequest(reason.RequestFormatError).WithError(err) } + fileExt := strings.ToLower(path.Ext(fileHeader.Filename)) newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt) avatarFilePath := path.Join(postSubPath, newFilename) - return us.uploadFile(ctx, fileHeader, avatarFilePath) + return us.uploadAttachmentFile(ctx, fileHeader, avatarFilePath) } func (us *uploaderService) UploadBrandingFile(ctx *gin.Context) ( @@ -210,8 +246,12 @@ func (us *uploaderService) UploadBrandingFile(ctx *gin.Context) ( return url, nil } - // max size - ctx.Request.Body = http.MaxBytesReader(ctx.Writer, ctx.Request.Body, 10*1024*1024) + siteWrite, err := us.siteInfoService.GetSiteWrite(ctx) + if err != nil { + return "", err + } + + ctx.Request.Body = http.MaxBytesReader(ctx.Writer, ctx.Request.Body, siteWrite.GetMaxImageSize()) file, fileHeader, err := ctx.Request.FormFile("file") if err != nil { return "", errors.BadRequest(reason.RequestFormatError).WithError(err) @@ -224,15 +264,19 @@ func (us *uploaderService) UploadBrandingFile(ctx *gin.Context) ( newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt) avatarFilePath := path.Join(brandingSubPath, newFilename) - return us.uploadFile(ctx, fileHeader, avatarFilePath) + return us.uploadImageFile(ctx, fileHeader, avatarFilePath) } -func (us *uploaderService) uploadFile(ctx *gin.Context, file *multipart.FileHeader, fileSubPath string) ( +func (us *uploaderService) uploadImageFile(ctx *gin.Context, file *multipart.FileHeader, fileSubPath string) ( url string, err error) { siteGeneral, err := us.siteInfoService.GetSiteGeneral(ctx) if err != nil { return "", err } + siteWrite, err := us.siteInfoService.GetSiteWrite(ctx) + if err != nil { + return "", err + } filePath := path.Join(us.serviceConfig.UploadPath, fileSubPath) if err := ctx.SaveUploadedFile(file, filePath); err != nil { return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack() @@ -244,7 +288,7 @@ func (us *uploaderService) uploadFile(ctx *gin.Context, file *multipart.FileHead } defer src.Close() - if !checker.IsSupportedImageFile(filePath) { + if !checker.DecodeAndCheckImageFile(filePath, siteWrite.GetMaxImageMegapixel()) { return "", errors.BadRequest(reason.UploadFileUnsupportedFileFormat) } @@ -256,6 +300,21 @@ func (us *uploaderService) uploadFile(ctx *gin.Context, file *multipart.FileHead return url, nil } +func (us *uploaderService) uploadAttachmentFile(ctx *gin.Context, file *multipart.FileHeader, fileSubPath string) ( + url string, err error) { + siteGeneral, err := us.siteInfoService.GetSiteGeneral(ctx) + if err != nil { + return "", err + } + filePath := path.Join(us.serviceConfig.UploadPath, fileSubPath) + if err := ctx.SaveUploadedFile(file, filePath); err != nil { + return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack() + } + + url = fmt.Sprintf("%s/uploads/%s", siteGeneral.SiteUrl, fileSubPath) + return url, nil +} + func (us *uploaderService) tryToUploadByPlugin(ctx *gin.Context, source plugin.UploadSource) ( url string, err error) { _ = plugin.CallStorage(func(fn plugin.Storage) error { diff --git a/pkg/checker/file_type.go b/pkg/checker/file_type.go index fffcd960..51f687d6 100644 --- a/pkg/checker/file_type.go +++ b/pkg/checker/file_type.go @@ -34,39 +34,42 @@ import ( "golang.org/x/image/webp" ) -const ( - maxImageSize = 16384 * 16384 -) +// IsUnAuthorizedExtension check whether the file extension is not in the allowedExtensions +// WANING Only checks the file extension is not reliable, but `http.DetectContentType` and `mimetype` are not reliable for all file types. +func IsUnAuthorizedExtension(fileName string, allowedExtensions []string) bool { + ext := strings.ToLower(strings.Trim(filepath.Ext(fileName), ".")) + for _, extension := range allowedExtensions { + if extension == ext { + return false + } + } + return true +} -// IsSupportedImageFile currently answers support image type is +// DecodeAndCheckImageFile currently answers support image type is // `image/jpeg, image/jpg, image/png, image/gif, image/webp` -func IsSupportedImageFile(localFilePath string) bool { +func DecodeAndCheckImageFile(localFilePath string, maxImageMegapixel int) bool { ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(localFilePath), ".")) switch ext { - case "jpg", "jpeg", "png", "gif": // only allow for `image/jpeg,image/jpg,image/png, image/gif` - if !decodeAndCheckImageFile(localFilePath, standardImageConfigCheck) { + case "jpg", "jpeg", "png", "gif": // only allow for `image/jpeg, image/jpg, image/png, image/gif` + if !decodeAndCheckImageFile(localFilePath, maxImageMegapixel, standardImageConfigCheck) { return false } - if !decodeAndCheckImageFile(localFilePath, standardImageCheck) { + if !decodeAndCheckImageFile(localFilePath, maxImageMegapixel, standardImageCheck) { return false } - case "ico": - // TODO: There is currently no good Golang library to parse whether the image is in ico format. - return true case "webp": - if !decodeAndCheckImageFile(localFilePath, webpImageConfigCheck) { + if !decodeAndCheckImageFile(localFilePath, maxImageMegapixel, webpImageConfigCheck) { return false } - if !decodeAndCheckImageFile(localFilePath, webpImageCheck) { + if !decodeAndCheckImageFile(localFilePath, maxImageMegapixel, webpImageCheck) { return false } - default: - return false } return true } -func decodeAndCheckImageFile(localFilePath string, checker func(io.Reader) error) bool { +func decodeAndCheckImageFile(localFilePath string, maxImageMegapixel int, checker func(file io.Reader, maxImageMegapixel int) error) bool { file, err := os.Open(localFilePath) if err != nil { log.Errorf("open file error: %v", err) @@ -74,25 +77,25 @@ func decodeAndCheckImageFile(localFilePath string, checker func(io.Reader) error } defer file.Close() - if err = checker(file); err != nil { + if err = checker(file, maxImageMegapixel); err != nil { log.Errorf("check image format error: %v", err) return false } return true } -func standardImageConfigCheck(file io.Reader) error { +func standardImageConfigCheck(file io.Reader, maxImageMegapixel int) error { config, _, err := image.DecodeConfig(file) if err != nil { return fmt.Errorf("decode image config error: %v", err) } - if imageSizeTooLarge(config) { + if imageSizeTooLarge(config, maxImageMegapixel) { return fmt.Errorf("image size too large") } return nil } -func standardImageCheck(file io.Reader) error { +func standardImageCheck(file io.Reader, maxImageMegapixel int) error { _, _, err := image.Decode(file) if err != nil { return fmt.Errorf("decode image error: %v", err) @@ -100,18 +103,18 @@ func standardImageCheck(file io.Reader) error { return nil } -func webpImageConfigCheck(file io.Reader) error { +func webpImageConfigCheck(file io.Reader, maxImageMegapixel int) error { config, err := webp.DecodeConfig(file) if err != nil { return fmt.Errorf("decode webp image config error: %v", err) } - if imageSizeTooLarge(config) { + if imageSizeTooLarge(config, maxImageMegapixel) { return fmt.Errorf("image size too large") } return nil } -func webpImageCheck(file io.Reader) error { +func webpImageCheck(file io.Reader, maxImageMegapixel int) error { _, err := webp.Decode(file) if err != nil { return fmt.Errorf("decode webp image error: %v", err) @@ -119,6 +122,6 @@ func webpImageCheck(file io.Reader) error { return nil } -func imageSizeTooLarge(config image.Config) bool { - return config.Width*config.Height > maxImageSize +func imageSizeTooLarge(config image.Config, maxImageMegapixel int) bool { + return config.Width*config.Height > maxImageMegapixel } diff --git a/plugin/storage.go b/plugin/storage.go index c40a1710..39c29488 100644 --- a/plugin/storage.go +++ b/plugin/storage.go @@ -22,9 +22,10 @@ package plugin type UploadSource string const ( - UserAvatar UploadSource = "user_avatar" - UserPost UploadSource = "user_post" - AdminBranding UploadSource = "admin_branding" + UserAvatar UploadSource = "user_avatar" + UserPost UploadSource = "user_post" + UserPostAttachment UploadSource = "user_post_attachment" + AdminBranding UploadSource = "admin_branding" ) var (
