This is an automated email from the ASF dual-hosted git repository. kumfo pushed a commit to branch feat/cdn/oss in repository https://gitbox.apache.org/repos/asf/incubator-answer-plugins.git
commit fdbd186f2355995ce622b1eb14c973552aff3688 Author: kumfo <[email protected]> AuthorDate: Mon Jul 8 11:23:13 2024 +0800 feat(cdn):Feat #128, Closes #128. Support CDN with Aliyun OSS --- cdn-aliyunoss/README.md | 18 ++ cdn-aliyunoss/aliyunoss.go | 383 ++++++++++++++++++++++++++++++++++++++ cdn-aliyunoss/go.mod | 47 +++++ cdn-aliyunoss/i18n/en_US.yaml | 72 +++++++ cdn-aliyunoss/i18n/translation.go | 45 +++++ cdn-aliyunoss/i18n/zh_CN.yaml | 72 +++++++ 6 files changed, 637 insertions(+) diff --git a/cdn-aliyunoss/README.md b/cdn-aliyunoss/README.md new file mode 100644 index 0000000..ceabbfe --- /dev/null +++ b/cdn-aliyunoss/README.md @@ -0,0 +1,18 @@ +# CDN With Aliyun OSS Storage (preview) +> This plugin can be used to store static files to Aliyun OSS. + +## How to use + +### Build +```bash +./answer build --with github.com/apache/incubator-answer-plugins/cdn-aliyunoss +``` + +### Configuration +- `Endpoint` - Endpoint of AliCloud OSS storage, such as oss-cn-hangzhou.aliyuncs.com +- `Bucket Name` - Your bucket name +- `Object Key Prefix` - Prefix of the object key like 'static/' that ending with '/' +- `Access Key Id` - AccessKeyID of the AliCloud OSS storage +- `Access Key Secret` - AccessKeySecret of the AliCloud OSS storage +- `Visit Url Prefix` - Prefix of access address for the CDN file, ending with '/' such as https://static.example.com/xxx/ +- `Max File Size` - Max file size in MB, default is 10MB \ No newline at end of file diff --git a/cdn-aliyunoss/aliyunoss.go b/cdn-aliyunoss/aliyunoss.go new file mode 100644 index 0000000..9c741b8 --- /dev/null +++ b/cdn-aliyunoss/aliyunoss.go @@ -0,0 +1,383 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package aliyunoss + +import ( + "encoding/json" + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/apache/incubator-answer-plugins/cdn-aliyunoss/i18n" + "github.com/apache/incubator-answer/plugin" + "github.com/apache/incubator-answer/ui" + "github.com/segmentfault/pacman/log" + "io" + "os" + "path/filepath" + "strconv" + "strings" +) + +var staticPath = os.Getenv("ANSWER_STATIC_PATH") + +const ( + // 10MB + defaultMaxFileSize int64 = 10 * 1024 * 1024 +) + +type CDN struct { + Config *CDNConfig +} + +type CDNConfig struct { + Endpoint string `json:"endpoint"` + BucketName string `json:"bucket_name"` + ObjectKeyPrefix string `json:"object_key_prefix"` + AccessKeyID string `json:"access_key_id"` + AccessKeySecret string `json:"access_key_secret"` + VisitUrlPrefix string `json:"visit_url_prefix"` + MaxFileSize string `json:"max_file_size"` +} + +type CustomFile struct { + file io.Reader + cdnPrefix string + old string +} + +func (f *CustomFile) Read(p []byte) (n int, err error) { + c := make([]byte, len(p)) + n, err = f.file.Read(c) + if f.old != "" { + c = []byte(strings.ReplaceAll(string(c), f.old, "\""+f.cdnPrefix+"/static")) + } + + n = copy(p, c) + return +} + +func (f *CustomFile) Close() error { return nil } + +func init() { + plugin.Register(&CDN{ + Config: &CDNConfig{}, + }) +} + +func (c *CDN) Info() plugin.Info { + return plugin.Info{ + Name: plugin.MakeTranslator(i18n.InfoName), + SlugName: "aliyunoss_cdn", + Description: plugin.MakeTranslator(i18n.InfoDescription), + Author: "answerdev", + Version: "1.0.0", + Link: "https://github.com/apache/incubator-answer-plugins/tree/main/cdn-aliyunoss";, + } +} + +// GetStaticPrefix get static prefix +func (c *CDN) GetStaticPrefix() string { + return c.Config.VisitUrlPrefix + c.Config.ObjectKeyPrefix +} + +// scanFiles scan all the static files in the build directory +func (c *CDN) scanFiles() { + if staticPath == "" { + c.scanEmbedFiles("build") + log.Info("complete scan embed files") + return + } + c.scanStaticPathFiles(staticPath) + log.Info("complete scan static path files") +} + +// scanStaticPathFiles scan static path files +func (c *CDN) scanStaticPathFiles(fileName string) { + // scan static path files + entry, err := os.ReadDir(fileName) + if err != nil { + log.Error("read static dir failed: %v", err) + return + } + for _, info := range entry { + if info.IsDir() { + c.scanStaticPathFiles(filepath.Join(fileName, info.Name())) + continue + } + + filePath := filepath.Join(fileName, info.Name()) + fi, _ := info.Info() + size := fi.Size() + file, err := os.Open(filePath) + if err != nil { + log.Error("open file failed: %v", err) + continue + } + + suffix := staticPath[:1] + if suffix != "/" { + suffix = "" + } + filePath = strings.TrimPrefix(filePath, staticPath+suffix) + + // rebuild custom io.Reader + ns := strings.Split(info.Name(), ".") + if info.Name() == "asset-manifest.json" { + c.Upload(filePath, c.rebuildReader(file, map[string]string{ + "\"/static": "", + }), size) + continue + } + if ns[0] == "main" { + ext := strings.ToLower(filepath.Ext(filePath)) + if ext == ".js" || ext == ".map" { + c.Upload(filePath, c.rebuildReader(file, map[string]string{ + "\"static": "", + "=\"/\",": "=\"\",", + }), size) + } + continue + } + + c.Upload(filePath, file, size) + } +} + +func (c *CDN) scanEmbedFiles(fileName string) { + entry, err := ui.Build.ReadDir(fileName) + if err != nil { + log.Error("read static dir failed: %v", err) + return + } + for _, info := range entry { + if info.IsDir() { + c.scanEmbedFiles(filepath.Join(fileName, info.Name())) + continue + } + + filePath := filepath.Join(fileName, info.Name()) + fi, _ := info.Info() + size := fi.Size() + file, err := ui.Build.Open(filePath) + defer file.Close() + if err != nil { + log.Error("open file failed: %v", err) + continue + } + + filePath = strings.TrimPrefix(filePath, "build/") + + // rebuild custom io.Reader + ns := strings.Split(info.Name(), ".") + if info.Name() == "asset-manifest.json" { + c.Upload(filePath, c.rebuildReader(file, map[string]string{ + "\"/static": "", + }), size) + continue + } + if ns[0] == "main" { + ext := strings.ToLower(filepath.Ext(filePath)) + if ext == ".js" || ext == ".map" { + c.Upload(filePath, c.rebuildReader(file, map[string]string{ + "\"static": "", + "=\"/\",": "=\"\",", + }), size) + } + continue + } + + c.Upload(filePath, file, size) + } +} + +func (c *CDN) rebuildReader(file io.Reader, replaceMap map[string]string) io.Reader { + var ( + bufr = make([]byte, 0) + res string + ) + + for { + buf := make([]byte, 1024) + n, err := file.Read(buf) + if err != nil { + break + } + bufr = append(bufr, buf[:n]...) + } + + res = string(bufr) + + for oldStr, newStr := range replaceMap { + if oldStr != "" { + if newStr == "" { + newStr = "\"" + c.GetStaticPrefix() + "/static" + } + res = strings.ReplaceAll(res, oldStr, newStr) + } + } + return strings.NewReader(res) +} + +func (c *CDN) Upload(filePath string, file io.Reader, size int64) { + client, err := oss.New(c.Config.Endpoint, c.Config.AccessKeyID, c.Config.AccessKeySecret) + if err != nil { + log.Error(plugin.MakeTranslator(i18n.ErrMisStorageConfig), err) + return + } + bucket, err := client.Bucket(c.Config.BucketName) + if err != nil { + log.Error(plugin.MakeTranslator(i18n.ErrMisStorageConfig), err) + return + } + + if !c.CheckFileType(filePath) { + log.Error(plugin.MakeTranslator(i18n.ErrUnsupportedFileType), filePath) + return + } + + if size > c.maxFileSizeLimit() { + log.Error(plugin.MakeTranslator(i18n.ErrOverFileSizeLimit)) + return + } + + objectKey := c.createObjectKey(filePath) + request := &oss.PutObjectRequest{ + ObjectKey: objectKey, + Reader: file, + } + respBody, err := bucket.DoPutObject(request, nil) + if err != nil { + log.Error(plugin.MakeTranslator(i18n.ErrUploadFileFailed), err) + return + } + defer respBody.Close() +} + +func (c *CDN) createObjectKey(filePath string) string { + return c.Config.ObjectKeyPrefix + filePath +} + +func (c *CDN) CheckFileType(filePath string) bool { + ext := strings.ToLower(filepath.Ext(filePath)) + if _, ok := plugin.DefaultCDNFileType[ext]; ok { + return true + } + return false +} + +func (c *CDN) maxFileSizeLimit() int64 { + if len(c.Config.MaxFileSize) == 0 { + return defaultMaxFileSize + } + limit, _ := strconv.Atoi(c.Config.MaxFileSize) + if limit <= 0 { + return defaultMaxFileSize + } + return int64(limit) * 1024 * 1024 +} + +func (c *CDN) ConfigFields() []plugin.ConfigField { + return []plugin.ConfigField{ + { + Name: "endpoint", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigEndpointTitle), + Description: plugin.MakeTranslator(i18n.ConfigEndpointDescription), + Required: true, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeText, + }, + Value: c.Config.Endpoint, + }, + { + Name: "bucket_name", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigBucketNameTitle), + Description: plugin.MakeTranslator(i18n.ConfigBucketNameDescription), + Required: true, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeText, + }, + Value: c.Config.BucketName, + }, + { + Name: "object_key_prefix", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigObjectKeyPrefixTitle), + Description: plugin.MakeTranslator(i18n.ConfigObjectKeyPrefixDescription), + Required: false, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeText, + }, + Value: c.Config.ObjectKeyPrefix, + }, + { + Name: "access_key_id", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigAccessKeyIdTitle), + Description: plugin.MakeTranslator(i18n.ConfigAccessKeyIdDescription), + Required: true, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeText, + }, + Value: c.Config.AccessKeyID, + }, + { + Name: "access_key_secret", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigAccessKeySecretTitle), + Description: plugin.MakeTranslator(i18n.ConfigAccessKeySecretDescription), + Required: true, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeText, + }, + Value: c.Config.AccessKeySecret, + }, + { + Name: "visit_url_prefix", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigVisitUrlPrefixTitle), + Description: plugin.MakeTranslator(i18n.ConfigVisitUrlPrefixDescription), + Required: true, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeText, + }, + Value: c.Config.VisitUrlPrefix, + }, + { + Name: "max_file_size", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigMaxFileSizeTitle), + Description: plugin.MakeTranslator(i18n.ConfigMaxFileSizeDescription), + Required: false, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeNumber, + }, + Value: c.Config.MaxFileSize, + }, + } +} + +func (c *CDN) ConfigReceiver(config []byte) error { + cfg := &CDNConfig{} + _ = json.Unmarshal(config, cfg) + c.Config = cfg + + go c.scanFiles() + return nil +} diff --git a/cdn-aliyunoss/go.mod b/cdn-aliyunoss/go.mod new file mode 100644 index 0000000..140928c --- /dev/null +++ b/cdn-aliyunoss/go.mod @@ -0,0 +1,47 @@ +module github.com/apache/incubator-answer-plugins/cdn-aliyunoss + +go 1.19 + +require ( + github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible + github.com/apache/incubator-answer v1.3.6 +) + +require ( + github.com/LinkinStars/go-i18n/v2 v2.2.2 // indirect + github.com/aymerick/douceur v0.2.0 // indirect + github.com/bytedance/sonic v1.9.1 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gin-gonic/gin v1.9.1 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.14.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/google/wire v0.5.0 // indirect + github.com/gorilla/css v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/leodido/go-urn v1.2.4 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/microcosm-cc/bluemonday v1.0.21 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/segmentfault/pacman v1.0.5-0.20230822083413-c0075a2d401f // indirect + github.com/segmentfault/pacman/contrib/i18n v0.0.0-20230516093754-b76aef1c1150 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.11 // indirect + golang.org/x/arch v0.3.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/net v0.21.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/cdn-aliyunoss/i18n/en_US.yaml b/cdn-aliyunoss/i18n/en_US.yaml new file mode 100644 index 0000000..2aa7e27 --- /dev/null +++ b/cdn-aliyunoss/i18n/en_US.yaml @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +plugin: + aliyunoss_storage: + backend: + info: + name: + other: Aliyun CDN OSS storage + description: + other: Upload files to AliCloud CDN OSS storage + config: + endpoint: + title: + other: Endpoint + description: + other: Endpoint of AliCloud OSS storage + bucket_name: + title: + other: Bucket name + description: + other: Bucket name of AliCloud OSS storage + object_key_prefix: + title: + other: Object Key prefix + description: + other: prefix of the object key like 'answer/data/' that ending with '/' + access_key_id: + title: + other: AccessKeyID + description: + other: AccessKeyID of the AliCloud OSS storage + access_key_secret: + title: + other: AccessKeySecret + description: + other: AccessKeySecret of AliCloud OSS storage + visit_url_prefix: + title: + other: Access URL prefix + description: + other: prefix of the final access address of the CDN, ending with '/' https://example.com/xxx/ + max_file_size: + title: + other: Maximum file size(MB) + description: + other: Limit the maximum size of uploaded files, in MB, default is 10MB + err: + mis_storage_config: + other: Wrong storage configuration causes upload failure. + file_not_found: + other: File not found. + unsupported_file_type: + other: Unsupported file type. + over_file_size_limit: + other: File size limit exceeded. + upload_file_failed: + other: Failed to upload a file. \ No newline at end of file diff --git a/cdn-aliyunoss/i18n/translation.go b/cdn-aliyunoss/i18n/translation.go new file mode 100644 index 0000000..773fe17 --- /dev/null +++ b/cdn-aliyunoss/i18n/translation.go @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package i18n + +const ( + InfoName = "plugin.aliyunoss_cdn.backend.info.name" + InfoDescription = "plugin.aliyunoss_cdn.backend.info.description" + + ConfigEndpointTitle = "plugin.aliyunoss_cdn.backend.config.endpoint.title" + ConfigEndpointDescription = "plugin.aliyunoss_cdn.backend.config.endpoint.description" + ConfigBucketNameTitle = "plugin.aliyunoss_cdn.backend.config.bucket_name.title" + ConfigBucketNameDescription = "plugin.aliyunoss_cdn.backend.config.bucket_name.description" + ConfigObjectKeyPrefixTitle = "plugin.aliyunoss_cdn.backend.config.object_key_prefix.title" + ConfigObjectKeyPrefixDescription = "plugin.aliyunoss_cdn.backend.config.object_key_prefix.description" + ConfigAccessKeyIdTitle = "plugin.aliyunoss_cdn.backend.config.access_key_id.title" + ConfigAccessKeyIdDescription = "plugin.aliyunoss_cdn.backend.config.access_key_id.description" + ConfigAccessKeySecretTitle = "plugin.aliyunoss_cdn.backend.config.access_key_secret.title" + ConfigAccessKeySecretDescription = "plugin.aliyunoss_cdn.backend.config.access_key_secret.description" + ConfigVisitUrlPrefixTitle = "plugin.aliyunoss_cdn.backend.config.visit_url_prefix.title" + ConfigVisitUrlPrefixDescription = "plugin.aliyunoss_cdn.backend.config.visit_url_prefix.description" + ConfigMaxFileSizeTitle = "plugin.aliyunoss_cdn.backend.config.max_file_size.title" + ConfigMaxFileSizeDescription = "plugin.aliyunoss_cdn.backend.config.max_file_size.description" + + ErrMisStorageConfig = "plugin.aliyunoss_cdn.backend.err.mis_storage_config" + ErrUnsupportedFileType = "plugin.aliyunoss_cdn.backend.err.unsupported_file_type" + ErrOverFileSizeLimit = "plugin.aliyunoss_cdn.backend.err.over_file_size_limit" + ErrUploadFileFailed = "plugin.aliyunoss_cdn.backend.err.upload_file_failed" +) diff --git a/cdn-aliyunoss/i18n/zh_CN.yaml b/cdn-aliyunoss/i18n/zh_CN.yaml new file mode 100644 index 0000000..7044a64 --- /dev/null +++ b/cdn-aliyunoss/i18n/zh_CN.yaml @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +plugin: + aliyunoss_storage: + backend: + info: + name: + other: 阿里云CDN OSS + description: + other: 上传文件到阿里云CDN OSS + config: + endpoint: + title: + other: Endpoint + description: + other: 阿里云OSS存储的Endpoint + bucket_name: + title: + other: Bucket名称 + description: + other: 阿里云OSS存储的Bucket名称 + object_key_prefix: + title: + other: 对象Key前缀 + description: + other: 对象键的前缀，如'answer/data/'，以'/'结尾 + access_key_id: + title: + other: AccessKeyID + description: + other: 阿里云OSS存储的AccessKeyID + access_key_secret: + title: + other: AccessKeySecret + description: + other: 阿里云OSS存储的AccessKeySecret + visit_url_prefix: + title: + other: 访问URL前缀 + description: + other: CDN最终访问地址的前缀，以 '/' 结尾 https://example.com/xxx/ + max_file_size: + title: + other: 最大文件大小(MB) + description: + other: 限制上传文件的最大大小，单位为MB，默认为 10MB + err: + mis_storage_config: + other: 错误的存储配置导致上传失败 + file_not_found: + other: 文件未找到 + unsupported_file_type: + other: 不支持的文件类型 + over_file_size_limit: + other: 超过文件大小限制 + upload_file_failed: + other: 上传文件失败 \ No newline at end of file
