This is an automated email from the ASF dual-hosted git repository. linkinstar pushed a commit to branch feat/storage in repository https://gitbox.apache.org/repos/asf/incubator-answer-plugins.git
commit 51c4c94b429b36d4df4664887b05bfc642e87de6 Author: LinkinStars <[email protected]> AuthorDate: Tue Nov 26 15:28:08 2024 +0800 refactor(storage): Add support for attachment uploads --- storage-aliyunoss/README.md | 5 ++-- storage-aliyunoss/aliyunoss.go | 57 ++++++++++++++++---------------------- storage-s3/README.md | 5 ++-- storage-s3/s3.go | 62 ++++++++++++++++++++---------------------- 4 files changed, 57 insertions(+), 72 deletions(-) diff --git a/storage-aliyunoss/README.md b/storage-aliyunoss/README.md index e98a7d5..a8df85e 100644 --- a/storage-aliyunoss/README.md +++ b/storage-aliyunoss/README.md @@ -1,4 +1,4 @@ -# Aliyun OSS Storage (preview) +# Aliyun OSS Storage > This plugin can be used to store attachments and avatars to Aliyun OSS. ## How to use @@ -14,5 +14,4 @@ - `Object Key Prefix` - Prefix of the object key like 'answer/data/' that ending with '/' - `Access Key Id` - AccessKeyID of the AliCloud OSS storage - `Access Key Secret` - AccessKeySecret of the AliCloud OSS storage -- `Visit Url Prefix` - Prefix of access address for the uploaded file, ending with '/' such as https://example.com/xxx/ -- `Max File Size` - Max file size in MB, default is 10MB \ No newline at end of file +- `Visit Url Prefix` - Prefix of access address for the uploaded file, ending with '/' such as https://example.com/xxx/ \ No newline at end of file diff --git a/storage-aliyunoss/aliyunoss.go b/storage-aliyunoss/aliyunoss.go index af132ca..bb5dcad 100644 --- a/storage-aliyunoss/aliyunoss.go +++ b/storage-aliyunoss/aliyunoss.go @@ -26,8 +26,8 @@ import ( "encoding/json" "fmt" "github.com/apache/incubator-answer-plugins/util" + "github.com/apache/incubator-answer/pkg/checker" "path/filepath" - "strconv" "strings" "time" @@ -39,11 +39,6 @@ import ( //go:embed info.yaml var Info embed.FS -const ( - // 10MB - defaultMaxFileSize int64 = 10 * 1024 * 1024 -) - type Storage struct { Config *StorageConfig } @@ -55,7 +50,6 @@ type StorageConfig struct { AccessKeyID string `json:"access_key_id"` AccessKeySecret string `json:"access_key_secret"` VisitUrlPrefix string `json:"visit_url_prefix"` - MaxFileSize string `json:"max_file_size"` } func init() { @@ -78,7 +72,7 @@ func (s *Storage) Info() plugin.Info { } } -func (s *Storage) UploadFile(ctx *plugin.GinContext, source plugin.UploadSource) (resp plugin.UploadFileResponse) { +func (s *Storage) UploadFile(ctx *plugin.GinContext, condition plugin.UploadFileCondition) (resp plugin.UploadFileResponse) { resp = plugin.UploadFileResponse{} client, err := oss.New(s.Config.Endpoint, s.Config.AccessKeyID, s.Config.AccessKeySecret) if err != nil { @@ -101,13 +95,13 @@ func (s *Storage) UploadFile(ctx *plugin.GinContext, source plugin.UploadSource) return resp } - if !s.CheckFileType(file.Filename, source) { + if s.IsUnsupportedFileType(file.Filename, condition) { resp.OriginalError = fmt.Errorf("file type not allowed") resp.DisplayErrorMsg = plugin.MakeTranslator(i18n.ErrUnsupportedFileType) return resp } - if file.Size > s.maxFileSizeLimit() { + if s.ExceedFileSizeLimit(file.Size, condition) { resp.OriginalError = fmt.Errorf("file size too large") resp.DisplayErrorMsg = plugin.MakeTranslator(i18n.ErrOverFileSizeLimit) return resp @@ -121,7 +115,7 @@ func (s *Storage) UploadFile(ctx *plugin.GinContext, source plugin.UploadSource) } defer open.Close() - objectKey := s.createObjectKey(file.Filename, source) + objectKey := s.createObjectKey(file.Filename, condition.Source) request := &oss.PutObjectRequest{ ObjectKey: objectKey, Reader: open, @@ -145,6 +139,8 @@ func (s *Storage) createObjectKey(originalFilename string, source plugin.UploadS return s.Config.ObjectKeyPrefix + "avatar/" + randomString + ext case plugin.UserPost: return s.Config.ObjectKeyPrefix + "post/" + randomString + ext + case plugin.UserPostAttachment: + return s.Config.ObjectKeyPrefix + "attachment/" + randomString + ext case plugin.AdminBranding: return s.Config.ObjectKeyPrefix + "branding/" + randomString + ext default: @@ -158,23 +154,27 @@ func (s *Storage) randomObjectKey() string { return fmt.Sprintf("%d", time.Now().UnixNano()) + hex.EncodeToString(bytes) } -func (s *Storage) CheckFileType(originalFilename string, source plugin.UploadSource) bool { - ext := strings.ToLower(filepath.Ext(originalFilename)) - if _, ok := plugin.DefaultFileTypeCheckMapping[source][ext]; ok { +func (s *Storage) IsUnsupportedFileType(originalFilename string, condition plugin.UploadFileCondition) bool { + if condition.Source == plugin.AdminBranding || condition.Source == plugin.UserAvatar { + ext := strings.ToLower(filepath.Ext(originalFilename)) + if _, ok := plugin.DefaultFileTypeCheckMapping[condition.Source][ext]; ok { + return false + } return true } - return false -} -func (s *Storage) maxFileSizeLimit() int64 { - if len(s.Config.MaxFileSize) == 0 { - return defaultMaxFileSize + // check the post image and attachment file type check + if condition.Source == plugin.UserPost { + return checker.IsUnAuthorizedExtension(originalFilename, condition.AuthorizedImageExtensions) } - limit, _ := strconv.Atoi(s.Config.MaxFileSize) - if limit <= 0 { - return defaultMaxFileSize + return checker.IsUnAuthorizedExtension(originalFilename, condition.AuthorizedAttachmentExtensions) +} + +func (s *Storage) ExceedFileSizeLimit(fileSize int64, condition plugin.UploadFileCondition) bool { + if condition.Source == plugin.UserPostAttachment { + return fileSize > int64(condition.MaxAttachmentSize)*1024*1024 } - return int64(limit) * 1024 * 1024 + return fileSize > int64(condition.MaxImageSize)*1024*1024 } func (s *Storage) ConfigFields() []plugin.ConfigField { @@ -245,17 +245,6 @@ func (s *Storage) ConfigFields() []plugin.ConfigField { }, Value: s.Config.VisitUrlPrefix, }, - { - Name: "max_file_size", - Type: plugin.ConfigTypeInput, - Title: plugin.MakeTranslator(i18n.ConfigMaxFileSizeTitle), - Description: plugin.MakeTranslator(i18n.ConfigMaxFileSizeDescription), - Required: false, - UIOptions: plugin.ConfigFieldUIOptions{ - InputType: plugin.InputTypeNumber, - }, - Value: s.Config.MaxFileSize, - }, } } diff --git a/storage-s3/README.md b/storage-s3/README.md index a55d386..2923691 100644 --- a/storage-s3/README.md +++ b/storage-s3/README.md @@ -1,4 +1,4 @@ -# S3 Storage (preview) +# S3 Storage > This plugin can be used to store attachments and avatars to AWS S3. ## How to use @@ -15,5 +15,4 @@ - `Access Key Id` - AccessKeyId of the S3 - `Access Key Secret` - AccessKeySecret of the S3 - `Access Token` - AccessToken of the S3 -- `Visit Url Prefix` - Prefix of access address for the uploaded file, ending with '/' such as https://example.com/xxx/ -- `Max File Size` - Max file size in MB, default is 10MB \ No newline at end of file +- `Visit Url Prefix` - Prefix of access address for the uploaded file, ending with '/' such as https://example.com/xxx/ \ No newline at end of file diff --git a/storage-s3/s3.go b/storage-s3/s3.go index 93b0e42..c3935f6 100644 --- a/storage-s3/s3.go +++ b/storage-s3/s3.go @@ -26,8 +26,8 @@ import ( "encoding/json" "fmt" "github.com/apache/incubator-answer-plugins/util" + "github.com/apache/incubator-answer/pkg/checker" "path/filepath" - "strconv" "strings" "time" @@ -38,11 +38,6 @@ import ( //go:embed info.yaml var Info embed.FS -const ( - // 10MB - defaultMaxFileSize int64 = 10 * 1024 * 1024 -) - type Storage struct { Config *StorageConfig Client *Client @@ -81,7 +76,7 @@ func (s *Storage) Info() plugin.Info { } } -func (s *Storage) UploadFile(ctx *plugin.GinContext, source plugin.UploadSource) (resp plugin.UploadFileResponse) { +func (s *Storage) UploadFile(ctx *plugin.GinContext, condition plugin.UploadFileCondition) (resp plugin.UploadFileResponse) { resp = plugin.UploadFileResponse{} file, err := ctx.FormFile("file") @@ -91,13 +86,13 @@ func (s *Storage) UploadFile(ctx *plugin.GinContext, source plugin.UploadSource) return resp } - if !s.checkFileType(file.Filename, source) { + if s.IsUnsupportedFileType(file.Filename, condition) { resp.OriginalError = fmt.Errorf("file type not allowed") resp.DisplayErrorMsg = plugin.MakeTranslator(i18n.ErrUnsupportedFileType) return resp } - if file.Size > s.maxFileSizeLimit() { + if s.ExceedFileSizeLimit(file.Size, condition) { resp.OriginalError = fmt.Errorf("file size too large") resp.DisplayErrorMsg = plugin.MakeTranslator(i18n.ErrOverFileSizeLimit) return resp @@ -111,7 +106,7 @@ func (s *Storage) UploadFile(ctx *plugin.GinContext, source plugin.UploadSource) } defer openFile.Close() - objectKey := s.createObjectKey(file.Filename, source) + objectKey := s.createObjectKey(file.Filename, condition.Source) err = s.Client.PutObject(objectKey, strings.ToLower(filepath.Ext(file.Filename)), openFile) if err != nil { resp.OriginalError = fmt.Errorf("upload file failed: %v", err) @@ -122,6 +117,29 @@ func (s *Storage) UploadFile(ctx *plugin.GinContext, source plugin.UploadSource) return resp } +func (s *Storage) IsUnsupportedFileType(originalFilename string, condition plugin.UploadFileCondition) bool { + if condition.Source == plugin.AdminBranding || condition.Source == plugin.UserAvatar { + ext := strings.ToLower(filepath.Ext(originalFilename)) + if _, ok := plugin.DefaultFileTypeCheckMapping[condition.Source][ext]; ok { + return false + } + return true + } + + // check the post image and attachment file type check + if condition.Source == plugin.UserPost { + return checker.IsUnAuthorizedExtension(originalFilename, condition.AuthorizedImageExtensions) + } + return checker.IsUnAuthorizedExtension(originalFilename, condition.AuthorizedAttachmentExtensions) +} + +func (s *Storage) ExceedFileSizeLimit(fileSize int64, condition plugin.UploadFileCondition) bool { + if condition.Source == plugin.UserPostAttachment { + return fileSize > int64(condition.MaxAttachmentSize)*1024*1024 + } + return fileSize > int64(condition.MaxImageSize)*1024*1024 +} + func (s *Storage) createObjectKey(originalFilename string, source plugin.UploadSource) string { ext := strings.ToLower(filepath.Ext(originalFilename)) randomString := s.randomObjectKey() @@ -130,6 +148,8 @@ func (s *Storage) createObjectKey(originalFilename string, source plugin.UploadS return s.Config.ObjectKeyPrefix + "avatar/" + randomString + ext case plugin.UserPost: return s.Config.ObjectKeyPrefix + "post/" + randomString + ext + case plugin.UserPostAttachment: + return s.Config.ObjectKeyPrefix + "attachment/" + randomString + ext case plugin.AdminBranding: return s.Config.ObjectKeyPrefix + "branding/" + randomString + ext default: @@ -151,17 +171,6 @@ func (s *Storage) checkFileType(originalFilename string, source plugin.UploadSou return false } -func (s *Storage) maxFileSizeLimit() int64 { - if len(s.Config.MaxFileSize) == 0 { - return defaultMaxFileSize - } - limit, _ := strconv.Atoi(s.Config.MaxFileSize) - if limit <= 0 { - return defaultMaxFileSize - } - return int64(limit) * 1024 * 1024 -} - func (s *Storage) ConfigFields() []plugin.ConfigField { return []plugin.ConfigField{ { @@ -241,17 +250,6 @@ func (s *Storage) ConfigFields() []plugin.ConfigField { }, Value: s.Config.VisitUrlPrefix, }, - { - Name: "max_file_size", - Type: plugin.ConfigTypeInput, - Title: plugin.MakeTranslator(i18n.ConfigMaxFileSizeTitle), - Description: plugin.MakeTranslator(i18n.ConfigMaxFileSizeDescription), - Required: false, - UIOptions: plugin.ConfigFieldUIOptions{ - InputType: plugin.InputTypeNumber, - }, - Value: s.Config.MaxFileSize, - }, { Name: "region", Type: plugin.ConfigTypeInput,
