This is an automated email from the ASF dual-hosted git repository. kumfo pushed a commit to branch feat/cdn/oss in repository https://gitbox.apache.org/repos/asf/incubator-answer-plugins.git
commit ab64ede2550a85a22fc618a8ca54945fd332d238 Merge: 6734872 2b935cd Author: kumfo <[email protected]> AuthorDate: Wed Jul 17 10:00:21 2024 +0800 refactor(cdn-oss): Merge branch 'main' into feat/cdn/oss .github/workflows/sync-info.yml | 13 +- cache-redis/go.mod | 2 +- cache-redis/go.sum | 4 +- cache-redis/redis.go | 6 +- captcha-basic/basic.go | 6 +- captcha-basic/go.mod | 2 +- captcha-basic/go.sum | 4 +- captcha-google-v2/go.mod | 2 +- captcha-google-v2/go.sum | 4 +- captcha-google-v2/recaptcha.go | 6 +- cdn-aliyunoss/aliyunoss.go | 6 +- cdn-aliyunoss/go.mod | 4 +- cdn-aliyunoss/go.sum | 11 +- connector-apache/apache.go | 6 +- connector-apache/go.mod | 4 +- connector-apache/go.sum | 4 +- connector-apache/info.yaml | 2 +- connector-basic/basic.go | 5 +- connector-basic/go.mod | 2 +- connector-basic/go.sum | 4 +- connector-basic/info.yaml | 2 +- connector-dingtalk/dingtalk.go | 6 +- connector-dingtalk/go.mod | 2 +- connector-dingtalk/go.sum | 9 +- connector-dingtalk/info.yaml | 2 +- connector-github/github.go | 6 +- connector-github/go.mod | 2 +- connector-github/go.sum | 4 +- connector-github/info.yaml | 2 +- connector-google/go.mod | 2 +- connector-google/go.sum | 4 +- connector-google/google.go | 6 +- connector-google/info.yaml | 2 +- editor-chart/chart.go | 6 +- editor-chart/go.mod | 2 +- editor-chart/go.sum | 4 +- editor-chart/info.yaml | 2 +- editor-formula/formula.go | 6 +- editor-formula/go.mod | 2 +- editor-formula/go.sum | 4 +- editor-formula/info.yaml | 2 +- embed-basic/basic.go | 10 +- embed-basic/go.mod | 2 +- embed-basic/go.sum | 4 +- embed-basic/info.yaml | 2 +- notification-slack/go.mod | 2 +- notification-slack/go.sum | 4 +- notification-slack/info.yaml | 2 +- notification-slack/slack_notification.go | 6 +- plugins_desc.json | 212 ++++++++++++++++++++++++++++--- reviewer-akismet/basic.go | 6 +- reviewer-akismet/go.mod | 1 + reviewer-akismet/go.sum | 2 + reviewer-akismet/info.yaml | 2 +- reviewer-basic/basic.go | 6 +- reviewer-basic/go.mod | 2 +- reviewer-basic/go.sum | 4 +- reviewer-basic/info.yaml | 2 +- script/sync-info.sh | 46 ++++++- search-algolia/algolia.go | 6 +- search-algolia/go.mod | 2 +- search-algolia/go.sum | 4 +- search-algolia/info.yaml | 4 +- search-elasticsearch/es.go | 6 +- search-elasticsearch/go.mod | 2 +- search-elasticsearch/go.sum | 4 +- search-elasticsearch/info.yaml | 2 +- search-meilisearch/go.mod | 3 +- search-meilisearch/go.sum | 7 +- search-meilisearch/info.yaml | 2 +- search-meilisearch/meilisearch.go | 6 +- storage-aliyunoss/aliyunoss.go | 6 +- storage-aliyunoss/go.mod | 2 +- storage-aliyunoss/go.sum | 4 +- storage-aliyunoss/info.yaml | 2 +- storage-s3/go.mod | 2 +- storage-s3/go.sum | 4 +- storage-s3/info.yaml | 2 +- storage-s3/s3.go | 6 +- user-center-wecom/go.mod | 2 +- user-center-wecom/go.sum | 4 +- user-center-wecom/info.yaml | 2 +- user-center-wecom/wecom_user_center.go | 6 +- util/go.mod | 2 +- util/util.go | 12 +- 85 files changed, 442 insertions(+), 151 deletions(-) diff --cc cdn-aliyunoss/aliyunoss.go index 66eb194,0000000..a7ffcf2 mode 100644,000000..100644 --- a/cdn-aliyunoss/aliyunoss.go +++ b/cdn-aliyunoss/aliyunoss.go @@@ -1,400 -1,0 +1,404 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package aliyunoss + +import ( ++ "embed" + "encoding/json" + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/apache/incubator-answer-plugins/cdn-aliyunoss/i18n" + "github.com/apache/incubator-answer-plugins/util" + "github.com/apache/incubator-answer/plugin" + "github.com/apache/incubator-answer/ui" + "github.com/segmentfault/pacman/log" + "io" + "os" + "path/filepath" + "strconv" + "strings" +) + +var staticPath = os.Getenv("ANSWER_STATIC_PATH") + ++//go:embed info.yaml ++var Info embed.FS ++ +const ( + // 10MB + defaultMaxFileSize int64 = 10 * 1024 * 1024 +) + +type CDN struct { + Config *CDNConfig +} + +type CDNConfig struct { + Endpoint string `json:"endpoint"` + BucketName string `json:"bucket_name"` + ObjectKeyPrefix string `json:"object_key_prefix"` + AccessKeyID string `json:"access_key_id"` + AccessKeySecret string `json:"access_key_secret"` + VisitUrlPrefix string `json:"visit_url_prefix"` + MaxFileSize string `json:"max_file_size"` +} + +type CustomFile struct { + file io.Reader + cdnPrefix string + old string +} + +func (f *CustomFile) Read(p []byte) (n int, err error) { + c := make([]byte, len(p)) + n, err = f.file.Read(c) + if f.old != "" { + c = []byte(strings.ReplaceAll(string(c), f.old, "\""+f.cdnPrefix+"/static")) + } + + n = copy(p, c) + return +} + +func (f *CustomFile) Close() error { return nil } + +func init() { + plugin.Register(&CDN{ + Config: &CDNConfig{}, + }) +} + +func (c *CDN) Info() plugin.Info { + info := util.Info{} - info.GetInfo() ++ info.GetInfo(Info) + return plugin.Info{ + Name: plugin.MakeTranslator(i18n.InfoName), + SlugName: info.SlugName, + Description: plugin.MakeTranslator(i18n.InfoDescription), + Author: info.Author, + Version: info.Version, + Link: info.Link, + } +} + +// GetStaticPrefix get static prefix +func (c *CDN) GetStaticPrefix() string { + return c.Config.VisitUrlPrefix + c.Config.ObjectKeyPrefix +} + +// scanFiles scan all the static files in the build directory +func (c *CDN) scanFiles() { + if staticPath == "" { + c.scanEmbedFiles("build") + log.Info("complete scan embed files") + return + } + c.scanStaticPathFiles(staticPath) + log.Info("complete scan static path files") +} + +// scanStaticPathFiles scan static path files +func (c *CDN) scanStaticPathFiles(fileName string) { + // scan static path files + entry, err := os.ReadDir(fileName) + if err != nil { + log.Error("read static dir failed: %v", err) + return + } + for _, info := range entry { + if info.IsDir() { + c.scanStaticPathFiles(filepath.Join(fileName, info.Name())) + continue + } + + filePath := filepath.Join(fileName, info.Name()) + fi, _ := info.Info() + size := fi.Size() + file, err := os.Open(filePath) + if err != nil { + log.Error("open file failed: %v", err) + continue + } + + suffix := staticPath[:1] + if suffix != "/" { + suffix = "" + } + filePath = strings.TrimPrefix(filePath, staticPath+suffix) + + // rebuild custom io.Reader + ns := strings.Split(info.Name(), ".") + if info.Name() == "asset-manifest.json" { + c.Upload(filePath, c.rebuildReader(file, map[string]string{ + "\"/static": "", + }), size) + continue + } + if ns[0] == "main" { + ext := strings.ToLower(filepath.Ext(filePath)) + if ext == ".js" || ext == ".map" { + c.Upload(filePath, c.rebuildReader(file, map[string]string{ + "\"static": "", + "=\"/\",": "=\"\",", + }), size) + continue + } + + if ext == ".css" { + c.Upload(filePath, c.rebuildReader(file, map[string]string{ + "url(/static": "url(../../static", + }), size) + continue + } + } + + c.Upload(filePath, file, size) + } +} + +func (c *CDN) scanEmbedFiles(fileName string) { + entry, err := ui.Build.ReadDir(fileName) + if err != nil { + log.Error("read static dir failed: %v", err) + return + } + for _, info := range entry { + if info.IsDir() { + c.scanEmbedFiles(filepath.Join(fileName, info.Name())) + continue + } + + filePath := filepath.Join(fileName, info.Name()) + fi, _ := info.Info() + size := fi.Size() + file, err := ui.Build.Open(filePath) + defer file.Close() + if err != nil { + log.Error("open file failed: %v", err) + continue + } + + filePath = strings.TrimPrefix(filePath, "build/") + + // rebuild custom io.Reader + ns := strings.Split(info.Name(), ".") + if info.Name() == "asset-manifest.json" { + c.Upload(filePath, c.rebuildReader(file, map[string]string{ + "\"/static": "", + }), size) + continue + } + if ns[0] == "main" { + ext := strings.ToLower(filepath.Ext(filePath)) + if ext == ".js" || ext == ".map" { + c.Upload(filePath, c.rebuildReader(file, map[string]string{ + "\"static": "", + "=\"/\",": "=\"\",", + }), size) + continue + } + + if ext == ".css" { + c.Upload(filePath, c.rebuildReader(file, map[string]string{ + "url(/static": "url(../../static", + }), size) + continue + } + } + + c.Upload(filePath, file, size) + } +} + +func (c *CDN) rebuildReader(file io.Reader, replaceMap map[string]string) io.Reader { + var ( + bufr = make([]byte, 0) + res string + ) + + for { + buf := make([]byte, 1024) + n, err := file.Read(buf) + if err != nil { + break + } + bufr = append(bufr, buf[:n]...) + } + + res = string(bufr) + + for oldStr, newStr := range replaceMap { + if oldStr != "" { + if newStr == "" { + newStr = "\"" + c.GetStaticPrefix() + "/static" + } + res = strings.ReplaceAll(res, oldStr, newStr) + } + } + return strings.NewReader(res) +} + +func (c *CDN) Upload(filePath string, file io.Reader, size int64) { + client, err := oss.New(c.Config.Endpoint, c.Config.AccessKeyID, c.Config.AccessKeySecret) + if err != nil { + log.Error(plugin.MakeTranslator(i18n.ErrMisStorageConfig), err) + return + } + bucket, err := client.Bucket(c.Config.BucketName) + if err != nil { + log.Error(plugin.MakeTranslator(i18n.ErrMisStorageConfig), err) + return + } + + if !c.CheckFileType(filePath) { + log.Error(plugin.MakeTranslator(i18n.ErrUnsupportedFileType), filePath) + return + } + + if size > c.maxFileSizeLimit() { + log.Error(plugin.MakeTranslator(i18n.ErrOverFileSizeLimit)) + return + } + + objectKey := c.createObjectKey(filePath) + request := &oss.PutObjectRequest{ + ObjectKey: objectKey, + Reader: file, + } + respBody, err := bucket.DoPutObject(request, nil) + if err != nil { + log.Error(plugin.MakeTranslator(i18n.ErrUploadFileFailed), err) + return + } + defer respBody.Close() +} + +func (c *CDN) createObjectKey(filePath string) string { + return c.Config.ObjectKeyPrefix + filePath +} + +func (c *CDN) CheckFileType(filePath string) bool { + ext := strings.ToLower(filepath.Ext(filePath)) + if _, ok := plugin.DefaultCDNFileType[ext]; ok { + return true + } + return false +} + +func (c *CDN) maxFileSizeLimit() int64 { + if len(c.Config.MaxFileSize) == 0 { + return defaultMaxFileSize + } + limit, _ := strconv.Atoi(c.Config.MaxFileSize) + if limit <= 0 { + return defaultMaxFileSize + } + return int64(limit) * 1024 * 1024 +} + +func (c *CDN) ConfigFields() []plugin.ConfigField { + return []plugin.ConfigField{ + { + Name: "endpoint", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigEndpointTitle), + Description: plugin.MakeTranslator(i18n.ConfigEndpointDescription), + Required: true, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeText, + }, + Value: c.Config.Endpoint, + }, + { + Name: "bucket_name", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigBucketNameTitle), + Description: plugin.MakeTranslator(i18n.ConfigBucketNameDescription), + Required: true, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeText, + }, + Value: c.Config.BucketName, + }, + { + Name: "object_key_prefix", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigObjectKeyPrefixTitle), + Description: plugin.MakeTranslator(i18n.ConfigObjectKeyPrefixDescription), + Required: false, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeText, + }, + Value: c.Config.ObjectKeyPrefix, + }, + { + Name: "access_key_id", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigAccessKeyIdTitle), + Description: plugin.MakeTranslator(i18n.ConfigAccessKeyIdDescription), + Required: true, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeText, + }, + Value: c.Config.AccessKeyID, + }, + { + Name: "access_key_secret", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigAccessKeySecretTitle), + Description: plugin.MakeTranslator(i18n.ConfigAccessKeySecretDescription), + Required: true, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeText, + }, + Value: c.Config.AccessKeySecret, + }, + { + Name: "visit_url_prefix", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigVisitUrlPrefixTitle), + Description: plugin.MakeTranslator(i18n.ConfigVisitUrlPrefixDescription), + Required: true, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeText, + }, + Value: c.Config.VisitUrlPrefix, + }, + { + Name: "max_file_size", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigMaxFileSizeTitle), + Description: plugin.MakeTranslator(i18n.ConfigMaxFileSizeDescription), + Required: false, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeNumber, + }, + Value: c.Config.MaxFileSize, + }, + } +} + +func (c *CDN) ConfigReceiver(config []byte) error { + cfg := &CDNConfig{} + _ = json.Unmarshal(config, cfg) + c.Config = cfg + + go c.scanFiles() + return nil +} diff --cc cdn-aliyunoss/go.mod index 0d34d3e,8384a2b..9923962 --- a/cdn-aliyunoss/go.mod +++ b/cdn-aliyunoss/go.mod @@@ -3,9 -3,9 +3,9 @@@ module github.com/apache/incubator-answ go 1.19 require ( - github.com/apache/incubator-answer v1.3.1-0.20240506084933-9681c026adfe + github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible - github.com/apache/incubator-answer v1.3.6 - github.com/apache/incubator-answer-plugins/util v0.0.0-20240712061634-0eab52b019e6 ++ github.com/apache/incubator-answer v1.3.5 + github.com/apache/incubator-answer-plugins/util v1.0.2 - github.com/go-resty/resty/v2 v2.12.0 github.com/segmentfault/pacman v1.0.5-0.20230822083413-c0075a2d401f ) diff --cc cdn-aliyunoss/go.sum index 9c0cc90,6d5b052..5aee93b --- a/cdn-aliyunoss/go.sum +++ b/cdn-aliyunoss/go.sum @@@ -2,10 -2,10 +2,12 @@@ github.com/BurntSushi/toml v1.0.0 h1:dt github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/LinkinStars/go-i18n/v2 v2.2.2 h1:ZfjpzbW13dv6btv3RALKZkpN9A+7K1JA//2QcNeWaxU= github.com/LinkinStars/go-i18n/v2 v2.2.2/go.mod h1:hLglSJ4/3M0Y7ZVcoEJI+OwqkglHCA32DdjuJJR2LbM= -github.com/apache/incubator-answer v1.3.1-0.20240506084933-9681c026adfe h1:qjPRGlo6u24SQJDKqGpLNz2ju/BuTH4FO5Xy5RPXZC0= -github.com/apache/incubator-answer v1.3.1-0.20240506084933-9681c026adfe/go.mod h1:YKwpG0rwRC0kHcbILcIyIbPMwsWaZ8j5lHJ34DPIdMI= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= - github.com/apache/incubator-answer-plugins/util v0.0.0-20240712061634-0eab52b019e6 h1:6Sld5x3mxGV0sDE8IyuyqwRC8z6DlMBNDQQgjbY7n9M= - github.com/apache/incubator-answer-plugins/util v0.0.0-20240712061634-0eab52b019e6/go.mod h1:dq/SmyUtl98iV85NswmBGmi2jdc/1dOMDzWD2JGBA0o= ++github.com/apache/incubator-answer v1.3.5 h1:vTsm54JUUK2xqZPni0gkv6PYupc03UZSYdIVGO1IOrI= ++github.com/apache/incubator-answer v1.3.5/go.mod h1:YKwpG0rwRC0kHcbILcIyIbPMwsWaZ8j5lHJ34DPIdMI= + github.com/apache/incubator-answer-plugins/util v1.0.2 h1:PontocVaiEm+oTj+4aDonwWDZnxywUeHsaTwlQgclfA= + github.com/apache/incubator-answer-plugins/util v1.0.2/go.mod h1:KPMSiM4ec4uEl2njaGINYuSl6zVmHdvPB2nHUxVcQDo= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= @@@ -50,9 -51,9 +51,8 @@@ github.com/klauspost/cpuid/v2 v2.0.9/go github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= - github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
