asifdxtreme closed pull request #496: SCB-1032 Support compress the response URL: https://github.com/apache/servicecomb-service-center/pull/496
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/glide.yaml b/glide.yaml index 16133dc5..21b66d67 100644 --- a/glide.yaml +++ b/glide.yaml @@ -289,6 +289,9 @@ import: - package: github.com/natefinch/lumberjack version: a96e63847dc3c67d17befa69c303767e2f84e54f repo: https://github.com/natefinch/lumberjack +- package: github.com/NYTimes/gziphandler + version: 253f1acb9d9f896d86c313a3dc994c0b114f0e12 + repo: https://github.com/NYTimes/gziphandler # k8s 1.10.4 deps - package: k8s.io/client-go diff --git a/go.mod b/go.mod index bf3d4b3f..238ba934 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ replace ( ) require ( + github.com/NYTimes/gziphandler v1.0.2-0.20180820182813-253f1acb9d9f github.com/Shopify/sarama v1.18.0 // indirect github.com/apache/thrift v0.0.0-20180125231006-3d556248a8b9 // indirect github.com/astaxie/beego v1.8.0 diff --git a/pkg/client/sc/config.go b/pkg/client/sc/config.go index 128df5c9..1a8e4bb9 100644 --- a/pkg/client/sc/config.go +++ b/pkg/client/sc/config.go @@ -43,5 +43,6 @@ func (cfg *Config) Merge() rest.URLClientOption { if cfg.RequestTimeout == 0 { cfg.RequestTimeout = defaultRequestTimeout } + cfg.Compressed = true return cfg.URLClientOption } diff --git a/pkg/rest/client.go b/pkg/rest/client.go index 4a2d2f61..82506378 100644 --- a/pkg/rest/client.go +++ b/pkg/rest/client.go @@ -17,12 +17,15 @@ package rest import ( "bytes" + "compress/gzip" "crypto/tls" "errors" "fmt" "github.com/apache/servicecomb-service-center/pkg/tlsutil" "github.com/apache/servicecomb-service-center/pkg/util" "golang.org/x/net/context" + "io" + "io/ioutil" "net/http" "net/url" "os" @@ -57,6 +60,24 @@ type URLClientOption struct { ConnsPerHost int } +type gzipBodyReader struct { + *gzip.Reader + Body io.ReadCloser +} + +func (w *gzipBodyReader) Close() error { + w.Reader.Close() + return w.Body.Close() +} + +func NewGZipBodyReader(body io.ReadCloser) (io.ReadCloser, error) { + reader, err := gzip.NewReader(body) + if err != nil { + return nil, err + } + return &gzipBodyReader{reader, body}, nil +} + type URLClient struct { *http.Client @@ -76,18 +97,18 @@ func (client *URLClient) HttpDoWithContext(ctx context.Context, method string, r headers = make(http.Header) } - if _, ok := headers["Host"]; !ok { + if _, ok := headers[HEADER_HOST]; !ok { parsedURL, err := url.Parse(rawURL) if err != nil { return nil, err } - headers.Set("Host", parsedURL.Host) + headers.Set(HEADER_HOST, parsedURL.Host) } - if _, ok := headers["Accept"]; !ok { - headers.Set("Accept", "*/*") + if _, ok := headers[HEADER_ACCEPT]; !ok { + headers.Set(HEADER_ACCEPT, ACCEPT_ANY) } - if _, ok := headers["Accept-Encoding"]; !ok && client.Cfg.Compressed { - headers.Set("Accept-Encoding", "deflate, gzip") + if _, ok := headers[HEADER_ACCEPT_ENCODING]; !ok && client.Cfg.Compressed { + headers.Set(HEADER_ACCEPT_ENCODING, "deflate, gzip") } req, err := http.NewRequest(method, rawURL, bytes.NewBuffer(body)) @@ -101,6 +122,16 @@ func (client *URLClient) HttpDoWithContext(ctx context.Context, method string, r if err != nil { return nil, err } + switch resp.Header.Get(HEADER_CONTENT_ENCODING) { + case "gzip": + reader, err := NewGZipBodyReader(resp.Body) + if err != nil { + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + return nil, err + } + resp.Body = reader + } if os.Getenv("DEBUG_MODE") == "1" { fmt.Println("--- BEGIN ---") diff --git a/pkg/rest/common.go b/pkg/rest/common.go index 71858388..1d99ffc7 100644 --- a/pkg/rest/common.go +++ b/pkg/rest/common.go @@ -36,12 +36,14 @@ const ( HEADER_RESPONSE_STATUS = "X-Response-Status" HEADER_ALLOW = "Allow" + HEADER_HOST = "Host" HEADER_SERVER = "Server" HEADER_CONTENT_TYPE = "Content-Type" HEADER_CONTENT_ENCODING = "Content-Encoding" HEADER_ACCEPT = "Accept" HEADER_ACCEPT_ENCODING = "Accept-Encoding" + ACCEPT_ANY = "*/*" ACCEPT_JSON = "application/json" CONTENT_TYPE_JSON = "application/json; charset=UTF-8" diff --git a/pkg/rest/server.go b/pkg/rest/server.go index 8f33573e..37daacc8 100644 --- a/pkg/rest/server.go +++ b/pkg/rest/server.go @@ -17,7 +17,9 @@ package rest import ( + "compress/gzip" "crypto/tls" + "github.com/NYTimes/gziphandler" "github.com/apache/servicecomb-service-center/pkg/grace" "github.com/apache/servicecomb-service-center/pkg/log" "net" @@ -46,6 +48,8 @@ type ServerConfig struct { GraceTimeout time.Duration MaxHeaderBytes int TLSConfig *tls.Config + Compressed bool + CompressMinBytes int } func DefaultServerConfig() *ServerConfig { @@ -57,6 +61,8 @@ func DefaultServerConfig() *ServerConfig { KeepAliveTimeout: 1 * time.Minute, GraceTimeout: 3 * time.Second, MaxHeaderBytes: 16384, + Compressed: true, + CompressMinBytes: 1400, // 1.4KB } } @@ -64,7 +70,7 @@ func NewServer(srvCfg *ServerConfig) *Server { if srvCfg == nil { srvCfg = DefaultServerConfig() } - return &Server{ + s := &Server{ Server: &http.Server{ Addr: srvCfg.Addr, Handler: srvCfg.Handler, @@ -80,6 +86,11 @@ func NewServer(srvCfg *ServerConfig) *Server { state: serverStateInit, Network: "tcp", } + if srvCfg.Compressed && srvCfg.CompressMinBytes > 0 && srvCfg.Handler != nil { + wrapper, _ := gziphandler.NewGzipLevelAndMinSize(gzip.DefaultCompression, srvCfg.CompressMinBytes) + s.Handler = wrapper(srvCfg.Handler) + } + return s } type Server struct { diff --git a/scripts/release/LICENSE b/scripts/release/LICENSE index e723ef00..abdb96ac 100644 --- a/scripts/release/LICENSE +++ b/scripts/release/LICENSE @@ -247,6 +247,7 @@ github.com/google/gofuzz (24818f796faf91cd76ec7bddd72458fbced7a6c1) gopkg.in/inf.v0 (v0.9.1) github.com/modern-go/concurrent (1.0.3) github.com/modern-go/reflect2 (1.0.0) +github.com/NYTimes/gziphandler (253f1acb9d9f896d86c313a3dc994c0b114f0e12) ================================================================ For beorn7/perks (4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9) diff --git a/server/plugin/pkg/discovery/servicecenter/aggregate.go b/server/plugin/pkg/discovery/servicecenter/aggregate.go index bafa2963..b9fd9c80 100644 --- a/server/plugin/pkg/discovery/servicecenter/aggregate.go +++ b/server/plugin/pkg/discovery/servicecenter/aggregate.go @@ -76,7 +76,9 @@ func (c *SCClientAggregate) GetScCache(ctx context.Context) (*model.Cache, map[s func (c *SCClientAggregate) cacheAppend(name string, setter model.Setter, getter model.Getter) { getter.ForEach(func(_ int, v *model.KV) bool { - v.ClusterName = name + if len(v.ClusterName) == 0 || v.ClusterName == registry.DefaultClusterName { + v.ClusterName = name + } setter.SetValue(v) return true }) diff --git a/server/plugin/pkg/registry/common.go b/server/plugin/pkg/registry/common.go index 6509c715..0c41e358 100644 --- a/server/plugin/pkg/registry/common.go +++ b/server/plugin/pkg/registry/common.go @@ -59,7 +59,7 @@ const ( defaultDialTimeout = 10 * time.Second defaultRequestTimeout = 30 * time.Second - defaultClusterName = "default" + DefaultClusterName = "default" ) func WithTimeout(ctx context.Context) (context.Context, context.CancelFunc) { diff --git a/server/plugin/pkg/registry/config.go b/server/plugin/pkg/registry/config.go index 60f1701a..6113cce7 100644 --- a/server/plugin/pkg/registry/config.go +++ b/server/plugin/pkg/registry/config.go @@ -84,7 +84,7 @@ func (c *Config) RegistryAddresses() []string { func Configuration() *Config { configOnce.Do(func() { var err error - defaultRegistryConfig.ClusterName = beego.AppConfig.DefaultString("manager_name", defaultClusterName) + defaultRegistryConfig.ClusterName = beego.AppConfig.DefaultString("manager_name", DefaultClusterName) defaultRegistryConfig.ManagerAddress = beego.AppConfig.String("manager_addr") defaultRegistryConfig.ClusterAddresses = beego.AppConfig.DefaultString("manager_cluster", "http://127.0.0.1:2379";) defaultRegistryConfig.InitClusters() diff --git a/server/rest/server.go b/server/rest/server.go index 6a4d74a9..0bd55872 100644 --- a/server/rest/server.go +++ b/server/rest/server.go @@ -44,6 +44,7 @@ func LoadConfig() (srvCfg *rest.ServerConfig, err error) { srvCfg.WriteTimeout = writeTimeout srvCfg.MaxHeaderBytes = maxHeaderBytes srvCfg.TLSConfig = tlsConfig + srvCfg.Handler = DefaultServerMux return } @@ -54,7 +55,6 @@ func NewServer(ipAddr string) (srv *rest.Server, err error) { } srvCfg.Addr = ipAddr srv = rest.NewServer(srvCfg) - srv.Handler = DefaultServerMux if srvCfg.TLSConfig == nil { err = srv.Listen() ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
