This is an automated email from the ASF dual-hosted git repository. littlecui pushed a commit to branch page in repository https://gitbox.apache.org/repos/asf/servicecomb-service-center.git
commit 7b6f99db17becbf072d5be15835f00a05e7b0200 Author: little-cui <[email protected]> AuthorDate: Tue Apr 27 10:39:41 2021 +0800 SCB-2176 Fix: raise err 'etcdserver: request too large' when list 'dep-queue' --- server/core/backend/common.go | 2 +- server/plugin/discovery/common.go | 2 + server/plugin/discovery/config.go | 19 ++- server/plugin/discovery/config_test.go | 3 +- server/plugin/discovery/etcd/adaptor.go | 2 +- server/plugin/discovery/etcd/cacher_kv.go | 2 +- server/plugin/discovery/etcd/indexer_cache.go | 2 +- server/plugin/discovery/etcd/indexer_etcd.go | 21 +-- server/plugin/discovery/etcd/indexer_test.go | 16 ++- server/plugin/discovery/etcd/listwatch.go | 10 +- server/plugin/discovery/etcd/listwatch_inner.go | 10 +- server/plugin/discovery/etcd/listwatch_test.go | 19 +-- server/plugin/discovery/etcd/watcher_test.go | 11 +- server/plugin/registry/etcd/etcd.go | 2 +- server/plugin/registry/etcd/etcd_test.go | 12 +- server/plugin/registry/option.go | 8 +- server/service/auth/datasource.go | 35 ----- server/service/auth/etcd/etcd.go | 163 ------------------------ server/service/auth/etcd/etcd_test.go | 121 ------------------ server/service/auth/etcd/key_generator.go | 78 ------------ server/service/auth/etcd/key_generator_test.go | 37 ------ server/service/auth/manager.go | 58 --------- server/service/auth/manager_test.go | 48 ------- server/service/auth/options.go | 22 ---- server/service/auth/types.go | 18 --- 25 files changed, 87 insertions(+), 634 deletions(-) diff --git a/server/core/backend/common.go b/server/core/backend/common.go index 53fa826..60009b3 100644 --- a/server/core/backend/common.go +++ b/server/core/backend/common.go @@ -90,7 +90,7 @@ func registerInnerTypes() { WithInitSize(100).WithParser(proto.DependencyRuleParser))) DependencyQueue = Store().MustInstall(NewAddOn("DEPENDENCY_QUEUE", discovery.Configure().WithPrefix(core.GetServiceDependencyQueueRootKey("")). - WithInitSize(100).WithParser(proto.DependencyQueueParser))) + WithInitSize(100).WithPageSize(1500).WithParser(proto.DependencyQueueParser))) PROJECT = Store().MustInstall(NewAddOn("PROJECT", discovery.Configure().WithPrefix(core.GetProjectRootKey("")). WithInitSize(100).WithParser(proto.StringParser))) diff --git a/server/plugin/discovery/common.go b/server/plugin/discovery/common.go index 39232aa..85ed1e6 100644 --- a/server/plugin/discovery/common.go +++ b/server/plugin/discovery/common.go @@ -23,4 +23,6 @@ import ( const ( DefaultTimeout = 30 * time.Second DefaultCacheInitSize = 100 + // grpc does not allow to transport a large body more then 4MB in a request + DefaultPageSize = 4096 ) diff --git a/server/plugin/discovery/config.go b/server/plugin/discovery/config.go index d0e53f0..ba488e7 100644 --- a/server/plugin/discovery/config.go +++ b/server/plugin/discovery/config.go @@ -24,8 +24,11 @@ import ( type Config struct { // Key is the prefix to unique specify resource type - Key string - InitSize int + Key string + // InitSize the init cache size, disable cache when InitSize = 0 + InitSize int + // PageSize try to batch query backend to get all KVs + PageSize int64 Timeout time.Duration Period time.Duration DeferHandler DeferHandler @@ -35,8 +38,8 @@ type Config struct { } func (cfg *Config) String() string { - return fmt.Sprintf("{key: %s, timeout: %s, period: %s, lease: %v}", - cfg.Key, cfg.Timeout, cfg.Period, cfg.Lease) + return fmt.Sprintf("{key: %s, timeout: %s, period: %s, lease: %v, initSize: %v, pageSize: %v}", + cfg.Key, cfg.Timeout, cfg.Period, cfg.Lease, cfg.InitSize, cfg.PageSize) } func (cfg *Config) WithPrefix(key string) *Config { @@ -49,6 +52,11 @@ func (cfg *Config) WithInitSize(size int) *Config { return cfg } +func (cfg *Config) WithPageSize(size int64) *Config { + cfg.PageSize = size + return cfg +} + func (cfg *Config) WithTimeout(ot time.Duration) *Config { cfg.Timeout = ot return cfg @@ -94,9 +102,10 @@ func (cfg *Config) WithLease() *Config { func Configure() *Config { return &Config{ Key: "/", + InitSize: DefaultCacheInitSize, + PageSize: DefaultPageSize, Timeout: DefaultTimeout, Period: time.Second, - InitSize: DefaultCacheInitSize, Parser: proto.BytesParser, } } diff --git a/server/plugin/discovery/config_test.go b/server/plugin/discovery/config_test.go index 4a39dc3..83f2caa 100644 --- a/server/plugin/discovery/config_test.go +++ b/server/plugin/discovery/config_test.go @@ -17,6 +17,7 @@ package discovery import ( "github.com/apache/servicecomb-service-center/server/core/proto" + "strings" "testing" "time" ) @@ -74,7 +75,7 @@ func TestConfigure(t *testing.T) { if cfg.Parser != proto.MapParser { t.Fatalf("TestConfigure failed") } - if cfg.String() != "{key: /test, timeout: 2s, period: 3s, lease: false}" { + if !strings.HasPrefix(cfg.String(), "{key:") { t.Fatalf("TestConfigure failed") } } diff --git a/server/plugin/discovery/etcd/adaptor.go b/server/plugin/discovery/etcd/adaptor.go index fcc5302..6933c1f 100644 --- a/server/plugin/discovery/etcd/adaptor.go +++ b/server/plugin/discovery/etcd/adaptor.go @@ -60,7 +60,7 @@ func NewEtcdAdaptor(name string, cfg *discovery.Config) *Adaptor { "core will not cache '%s' and ignore all events of it, cache enabled: %v, init size: %d", name, core.ServerInfo.Config.EnableCache, cfg.InitSize) adaptor.Cacher = discovery.NullCacher - adaptor.Indexer = NewEtcdIndexer(cfg.Key, cfg.Parser) + adaptor.Indexer = NewEtcdIndexer(cfg) } return &adaptor } diff --git a/server/plugin/discovery/etcd/cacher_kv.go b/server/plugin/discovery/etcd/cacher_kv.go index a61d1e3..af1dda8 100644 --- a/server/plugin/discovery/etcd/cacher_kv.go +++ b/server/plugin/discovery/etcd/cacher_kv.go @@ -129,7 +129,7 @@ func (c *KvCacher) ListAndWatch(ctx context.Context) error { defer log.Recover() // ensure ListAndWatch never raise panic cfg := ListWatchConfig{ - Timeout: c.Cfg.Timeout, + Config: c.Cfg, Context: ctx, } diff --git a/server/plugin/discovery/etcd/indexer_cache.go b/server/plugin/discovery/etcd/indexer_cache.go index 4afa847..8f8242d 100644 --- a/server/plugin/discovery/etcd/indexer_cache.go +++ b/server/plugin/discovery/etcd/indexer_cache.go @@ -64,7 +64,7 @@ func (i *CacheIndexer) Creditable() bool { func NewCacheIndexer(cfg *discovery.Config, cache discovery.Cache) *CacheIndexer { return &CacheIndexer{ - Indexer: NewEtcdIndexer(cfg.Key, cfg.Parser), + Indexer: NewEtcdIndexer(cfg), CacheIndexer: discovery.NewCacheIndexer(cache), } } diff --git a/server/plugin/discovery/etcd/indexer_etcd.go b/server/plugin/discovery/etcd/indexer_etcd.go index 3f5c8ff..aaaa6b1 100644 --- a/server/plugin/discovery/etcd/indexer_etcd.go +++ b/server/plugin/discovery/etcd/indexer_etcd.go @@ -18,7 +18,6 @@ package etcd import ( "fmt" - "github.com/apache/servicecomb-service-center/server/core/proto" "strings" "github.com/apache/servicecomb-service-center/pkg/log" @@ -33,20 +32,19 @@ import ( // Indexer implements discovery.Indexer. // Indexer searches data from etcd server. type Indexer struct { + Cfg *discovery.Config Client registry.Registry - Parser proto.Parser - Root string } func (i *Indexer) CheckPrefix(key string) error { - if strings.Index(key, i.Root) != 0 { - return fmt.Errorf("search '%s' mismatch pattern %s", key, i.Root) + if strings.Index(key, i.Cfg.Key) != 0 { + return fmt.Errorf("search '%s' mismatch pattern %s", key, i.Cfg.Key) } return nil } func (i *Indexer) Search(ctx context.Context, opts ...registry.PluginOpOption) (r *discovery.Response, err error) { - op := registry.OpGet(opts...) + op := i.toGetOpts(opts) key := util.BytesToStringWithNoCopy(op.Key) log.Debugf("search '%s' match special options, request etcd server, opts: %s", key, op) @@ -66,7 +64,7 @@ func (i *Indexer) Search(ctx context.Context, opts ...registry.PluginOpOption) ( return } - p := i.Parser + p := i.Cfg.Parser if op.KeyOnly { p = nil } @@ -83,11 +81,16 @@ func (i *Indexer) Search(ctx context.Context, opts ...registry.PluginOpOption) ( return } +func (i *Indexer) toGetOpts(opts []registry.PluginOpOption) registry.PluginOp { + op := registry.OpGet(append([]registry.PluginOpOption{registry.WithLimit(i.Cfg.PageSize)}, opts...)...) + return op +} + // Creditable implements discovery.Indexer.Creditable. func (i *Indexer) Creditable() bool { return true } -func NewEtcdIndexer(root string, p proto.Parser) (indexer *Indexer) { - return &Indexer{Client: backend.Registry(), Parser: p, Root: root} +func NewEtcdIndexer(cfg *discovery.Config) (indexer *Indexer) { + return &Indexer{Cfg: cfg, Client: backend.Registry()} } diff --git a/server/plugin/discovery/etcd/indexer_test.go b/server/plugin/discovery/etcd/indexer_test.go index 57cd156..fc79ea3 100644 --- a/server/plugin/discovery/etcd/indexer_test.go +++ b/server/plugin/discovery/etcd/indexer_test.go @@ -28,7 +28,10 @@ import ( func TestEtcdIndexer_Search(t *testing.T) { data := ®istry.PluginResponse{Revision: 1} c := &mockRegistry{} - i := &Indexer{Client: c, Root: "/", Parser: proto.BytesParser} + cfg := &discovery.Config{ + Key: "/", Parser: proto.BytesParser, + } + i := &Indexer{Cfg: cfg, Client: c} // case: key does not contain prefix resp, err := i.Search(context.Background(), registry.WithStrKey("a")) @@ -52,14 +55,14 @@ func TestEtcdIndexer_Search(t *testing.T) { // case: parse error data.Count = 2 data.Kvs = []*mvccpb.KeyValue{{Key: []byte("/a/b"), Value: []byte("abc")}, {Key: []byte("/a/c"), Value: []byte("{}")}} - old := i.Parser - i.Parser = proto.MapParser + old := i.Cfg.Parser + i.Cfg.Parser = proto.MapParser c.Response = data resp, err = i.Search(context.Background(), registry.WithStrKey("/a")) if err != nil || resp == nil || resp.Count != 2 || len(resp.Kvs) != 1 { t.Fatalf("TestEtcdIndexer_Search failed, %v, %v", err, resp) } - i.Parser = old + i.Cfg.Parser = old // case: normal data.Count = 2 @@ -81,6 +84,9 @@ func TestEtcdIndexer_Search(t *testing.T) { func TestCacheIndexer_Search(t *testing.T) { c := &mockCache{} + cfg := &discovery.Config{ + Key: "/", + } cli := &mockRegistry{Response: ®istry.PluginResponse{ Revision: 1, Kvs: []*mvccpb.KeyValue{{Key: []byte("/a/b"), Value: []byte("abc")}}, @@ -88,7 +94,7 @@ func TestCacheIndexer_Search(t *testing.T) { }} i := &CacheIndexer{ Indexer: &Indexer{ - Root: "/", + Cfg: cfg, Client: cli, }, CacheIndexer: discovery.NewCacheIndexer(c), diff --git a/server/plugin/discovery/etcd/listwatch.go b/server/plugin/discovery/etcd/listwatch.go index b7f7470..818d040 100644 --- a/server/plugin/discovery/etcd/listwatch.go +++ b/server/plugin/discovery/etcd/listwatch.go @@ -19,23 +19,23 @@ package etcd import ( "context" "fmt" + "github.com/apache/servicecomb-service-center/server/plugin/discovery" "github.com/apache/servicecomb-service-center/server/plugin/registry" - "time" ) type ListWatchConfig struct { - Timeout time.Duration + *discovery.Config Context context.Context } func (lo *ListWatchConfig) String() string { - return fmt.Sprintf("{timeout: %s}", lo.Timeout) + return fmt.Sprintf("{Config: %v, timeout: %s}", lo.Config, lo.Timeout) } type ListWatch interface { - List(op ListWatchConfig) (*registry.PluginResponse, error) + List(cfg ListWatchConfig) (*registry.PluginResponse, error) // not support new multiple watchers - Watch(op ListWatchConfig) Watcher + Watch(cfg ListWatchConfig) Watcher // DoWatch(ctx context.Context, f func(*registry.PluginResponse)) error Revision() int64 diff --git a/server/plugin/discovery/etcd/listwatch_inner.go b/server/plugin/discovery/etcd/listwatch_inner.go index e7aa7f8..3853249 100644 --- a/server/plugin/discovery/etcd/listwatch_inner.go +++ b/server/plugin/discovery/etcd/listwatch_inner.go @@ -30,10 +30,10 @@ type innerListWatch struct { rev int64 } -func (lw *innerListWatch) List(op ListWatchConfig) (*registry.PluginResponse, error) { - otCtx, cancel := context.WithTimeout(op.Context, op.Timeout) +func (lw *innerListWatch) List(cfg ListWatchConfig) (*registry.PluginResponse, error) { + otCtx, cancel := context.WithTimeout(cfg.Context, cfg.Timeout) defer cancel() - resp, err := lw.Client.Do(otCtx, registry.WatchPrefixOpOptions(lw.Prefix)...) + resp, err := lw.Client.Do(otCtx, lw.toGetOpts(cfg)...) if err != nil { log.Errorf(err, "list prefix %s failed, current rev: %d", lw.Prefix, lw.Revision()) return nil, err @@ -42,6 +42,10 @@ func (lw *innerListWatch) List(op ListWatchConfig) (*registry.PluginResponse, er return resp, nil } +func (lw *innerListWatch) toGetOpts(cfg ListWatchConfig) []registry.PluginOpOption { + return append([]registry.PluginOpOption{registry.WithLimit(cfg.PageSize)}, registry.WatchPrefixOpOptions(lw.Prefix)...) +} + func (lw *innerListWatch) Revision() int64 { return lw.rev } diff --git a/server/plugin/discovery/etcd/listwatch_test.go b/server/plugin/discovery/etcd/listwatch_test.go index c1e1b83..099ff59 100644 --- a/server/plugin/discovery/etcd/listwatch_test.go +++ b/server/plugin/discovery/etcd/listwatch_test.go @@ -19,9 +19,11 @@ package etcd import ( "context" "fmt" + "github.com/apache/servicecomb-service-center/server/plugin/discovery" "github.com/apache/servicecomb-service-center/server/plugin/registry" "github.com/apache/servicecomb-service-center/server/plugin/registry/buildin" "github.com/coreos/etcd/mvcc/mvccpb" + "strings" "testing" "time" ) @@ -61,11 +63,12 @@ func TestPrefixListWatch(t *testing.T) { Prefix: "a", rev: 1, } - resp, err := lw.List(ListWatchConfig{Timeout: time.Second, Context: context.Background()}) + cfg := &discovery.Config{Timeout: time.Second} + resp, err := lw.List(ListWatchConfig{Config: cfg, Context: context.Background()}) if resp != nil || err == nil || lw.Revision() != 1 { t.Fatalf("TestPrefixListWatch failed") } - w := lw.Watch(ListWatchConfig{Timeout: time.Second, Context: context.Background()}) + w := lw.Watch(ListWatchConfig{Config: cfg, Context: context.Background()}) resp = <-w.EventBus() if resp != nil || lw.Revision() != 0 { t.Fatalf("TestPrefixListWatch failed") @@ -80,11 +83,11 @@ func TestPrefixListWatch(t *testing.T) { Prefix: "a", rev: 1, } - resp, err = lw.List(ListWatchConfig{Timeout: time.Second, Context: context.Background()}) + resp, err = lw.List(ListWatchConfig{Config: cfg, Context: context.Background()}) if resp == nil || err != nil || lw.Revision() != 2 { t.Fatalf("TestPrefixListWatch failed") } - w = lw.Watch(ListWatchConfig{Timeout: time.Second, Context: context.Background()}) + w = lw.Watch(ListWatchConfig{Config: cfg, Context: context.Background()}) resp = <-w.EventBus() if resp != nil || lw.Revision() != 0 { t.Fatalf("TestPrefixListWatch failed") @@ -100,11 +103,11 @@ func TestPrefixListWatch(t *testing.T) { Prefix: "a", rev: 1, } - resp, err = lw.List(ListWatchConfig{Timeout: time.Second, Context: context.Background()}) + resp, err = lw.List(ListWatchConfig{Config: cfg, Context: context.Background()}) if resp == nil || err != nil || lw.Revision() != 4 { t.Fatalf("TestPrefixListWatch failed") } - w = lw.Watch(ListWatchConfig{Timeout: time.Second, Context: context.Background()}) + w = lw.Watch(ListWatchConfig{Config: cfg, Context: context.Background()}) resp = <-w.EventBus() if resp == nil || lw.Revision() != 3 { t.Fatalf("TestPrefixListWatch failed") @@ -113,8 +116,8 @@ func TestPrefixListWatch(t *testing.T) { } func TestListWatchConfig_String(t *testing.T) { - lw := ListWatchConfig{Timeout: time.Second, Context: context.Background()} - if lw.String() != "{timeout: 1s}" { + lw := ListWatchConfig{Config: &discovery.Config{Timeout: time.Second}, Context: context.Background()} + if strings.Contains(lw.String(), "timeout: 1s") { t.Fatalf("TestListWatchConfig_String failed") } } diff --git a/server/plugin/discovery/etcd/watcher_test.go b/server/plugin/discovery/etcd/watcher_test.go index 2576193..9161738 100644 --- a/server/plugin/discovery/etcd/watcher_test.go +++ b/server/plugin/discovery/etcd/watcher_test.go @@ -19,6 +19,7 @@ package etcd import ( "context" "fmt" + "github.com/apache/servicecomb-service-center/server/plugin/discovery" "github.com/apache/servicecomb-service-center/server/plugin/registry" "testing" "time" @@ -41,7 +42,7 @@ type mockListWatch struct { Rev int64 } -func (lw *mockListWatch) List(op ListWatchConfig) (*registry.PluginResponse, error) { +func (lw *mockListWatch) List(_ ListWatchConfig) (*registry.PluginResponse, error) { if lw.ListResponse == nil { return nil, fmt.Errorf("error") } @@ -59,7 +60,7 @@ func (lw *mockListWatch) DoWatch(ctx context.Context, f func(*registry.PluginRes <-ctx.Done() return nil } -func (lw *mockListWatch) Watch(op ListWatchConfig) Watcher { +func (lw *mockListWatch) Watch(_ ListWatchConfig) Watcher { return lw.Watcher } func (lw *mockListWatch) Revision() int64 { @@ -67,7 +68,8 @@ func (lw *mockListWatch) Revision() int64 { } func TestInnerWatcher_EventBus(t *testing.T) { - w := newInnerWatcher(&mockListWatch{}, ListWatchConfig{Timeout: time.Second, Context: context.Background()}) + w := newInnerWatcher(&mockListWatch{}, ListWatchConfig{ + Config: &discovery.Config{Timeout: time.Second}, Context: context.Background()}) resp := <-w.EventBus() if resp != nil { t.Fatalf("TestInnerWatcher_EventBus failed") @@ -77,7 +79,8 @@ func TestInnerWatcher_EventBus(t *testing.T) { test := ®istry.PluginResponse{ Action: registry.Put, } - w = newInnerWatcher(&mockListWatch{ListResponse: test}, ListWatchConfig{Timeout: time.Second, Context: context.Background()}) + w = newInnerWatcher(&mockListWatch{ListResponse: test}, ListWatchConfig{ + Config: &discovery.Config{Timeout: time.Second}, Context: context.Background()}) resp = <-w.EventBus() if resp == nil || resp.Action != registry.Put { t.Fatalf("TestInnerWatcher_EventBus failed") diff --git a/server/plugin/registry/etcd/etcd.go b/server/plugin/registry/etcd/etcd.go index 159204f..6ba4305 100644 --- a/server/plugin/registry/etcd/etcd.go +++ b/server/plugin/registry/etcd/etcd.go @@ -420,7 +420,7 @@ func (c *Client) paging(ctx context.Context, op registry.PluginOp) (*clientv3.Ge limit++ start = 1 } - ops := append(baseOps, clientv3.WithLimit(int64(limit))) + ops := append(baseOps, clientv3.WithLimit(limit)) recordResp, err := c.Client.Get(ctx, nextKey, ops...) if err != nil { return nil, err diff --git a/server/plugin/registry/etcd/etcd_test.go b/server/plugin/registry/etcd/etcd_test.go index bfdee00..bdc4393 100644 --- a/server/plugin/registry/etcd/etcd_test.go +++ b/server/plugin/registry/etcd/etcd_test.go @@ -357,7 +357,7 @@ func TestEtcdClient(t *testing.T) { // large data var wg sync.WaitGroup - for i := 0; i < registry.DefaultPageCount+1; i++ { + for i := 0; i < registry.DefaultPageSize+1; i++ { wg.Add(1) v := strconv.Itoa(i) go func() { @@ -373,14 +373,14 @@ func TestEtcdClient(t *testing.T) { resp, err = etcdc.Do(context.Background(), registry.GET, registry.WithStrKey("/test_page/"), registry.WithStrEndKey("/test_page/9999")) - if err != nil || !resp.Succeeded || resp.Count != registry.DefaultPageCount+1 || - len(resp.Kvs) != registry.DefaultPageCount+1 { + if err != nil || !resp.Succeeded || resp.Count != registry.DefaultPageSize+1 || + len(resp.Kvs) != registry.DefaultPageSize+1 { t.Fatalf("TestEtcdClient_Do failed, %#v", err) } resp, err = etcdc.Do(context.Background(), registry.GET, registry.WithStrKey("/test_page/"), registry.WithPrefix(), registry.WithDescendOrder()) - if err != nil || !resp.Succeeded || resp.Count != registry.DefaultPageCount+1 || - len(resp.Kvs) != registry.DefaultPageCount+1 || + if err != nil || !resp.Succeeded || resp.Count != registry.DefaultPageSize+1 || + len(resp.Kvs) != registry.DefaultPageSize+1 || string(resp.Kvs[0].Key) != "/test_page/999" { t.Fatalf("TestEtcdClient_Do failed, %#v", err) } @@ -826,7 +826,7 @@ func TestEtcdClient_paging(t *testing.T) { op := registry.PluginOp{ Offset: -1, - Limit: registry.DefaultPageCount, + Limit: registry.DefaultPageSize, } r, err := c.paging(context2.Background(), op) if err != nil { diff --git a/server/plugin/registry/option.go b/server/plugin/registry/option.go index f137a1e..bbc6869 100644 --- a/server/plugin/registry/option.go +++ b/server/plugin/registry/option.go @@ -35,9 +35,11 @@ type PluginOp struct { IgnoreLease bool Mode CacheMode WatchCallback WatchCallback - Offset int64 - Limit int64 - Global bool + // Offset >= 0 will paging the results, -1 will return all results. + Offset int64 + // Limit is page size. not work if Offset = -1 + Limit int64 + Global bool } func (op PluginOp) String() string { diff --git a/server/service/auth/datasource.go b/server/service/auth/datasource.go deleted file mode 100644 index 3f081ed..0000000 --- a/server/service/auth/datasource.go +++ /dev/null @@ -1,35 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package auth - -import ( - "context" - "github.com/apache/servicecomb-service-center/pkg/rbacframe" -) - -type DataSource interface { - AccountExist(ctx context.Context, key string) (bool, error) - GetAccount(ctx context.Context, key string) (*rbacframe.Account, error) - ListAccount(ctx context.Context, key string) ([]*rbacframe.Account, int64, error) - DeleteAccount(ctx context.Context, key string) (bool, error) - UpdateAccount(ctx context.Context, key string, account *rbacframe.Account) error - - AddDomain(ctx context.Context, domain string) (bool, error) - DomainExist(ctx context.Context, domain string) (bool, error) - - AddProject(ctx context.Context, domain, project string) (bool, error) - ProjectExist(ctx context.Context, domain, project string) (bool, error) -} diff --git a/server/service/auth/etcd/etcd.go b/server/service/auth/etcd/etcd.go deleted file mode 100644 index 89d3420..0000000 --- a/server/service/auth/etcd/etcd.go +++ /dev/null @@ -1,163 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package etcd - -import ( - "context" - "encoding/json" - "errors" - "github.com/apache/servicecomb-service-center/pkg/log" - "github.com/apache/servicecomb-service-center/pkg/rbacframe" - "github.com/apache/servicecomb-service-center/server/core/backend" - "github.com/apache/servicecomb-service-center/server/plugin/registry" - utils "github.com/apache/servicecomb-service-center/server/service/util" -) - -// TODO: define error with names here - -var ErrNotUnique = errors.New("kv result is not unique") - -func init() { - // TODO: set logger - // TODO: register storage plugin to plugin manager -} - -type DataSource struct{} - -func NewDataSource() *DataSource { - // TODO: construct a reasonable DataSource instance - log.Warnf("auth data source enable etcd mode") - - inst := &DataSource{} - // TODO: deal with exception - if err := inst.initialize(); err != nil { - return inst - } - return inst -} - -func (ds *DataSource) initialize() error { - // TODO: init DataSource members - return nil -} - -func (ds *DataSource) AccountExist(ctx context.Context, key string) (bool, error) { - resp, err := backend.Registry().Do(ctx, registry.GET, - registry.WithStrKey(GenerateETCDAccountKey(key))) - if err != nil { - return false, err - } - if resp.Count == 0 { - return false, nil - } - return true, nil -} - -func (ds *DataSource) GetAccount(ctx context.Context, key string) (*rbacframe.Account, error) { - resp, err := backend.Registry().Do(ctx, registry.GET, - registry.WithStrKey(GenerateETCDAccountKey(key))) - if err != nil { - return nil, err - } - if resp.Count != 1 { - return nil, ErrNotUnique - } - account := &rbacframe.Account{} - err = json.Unmarshal(resp.Kvs[0].Value, account) - if err != nil { - log.Errorf(err, "account info format invalid") - return nil, err - } - return account, nil -} -func (ds *DataSource) ListAccount(ctx context.Context, key string) ([]*rbacframe.Account, int64, error) { - resp, err := backend.Registry().Do(ctx, registry.GET, - registry.WithStrKey(GenerateETCDAccountKey(key)), registry.WithPrefix()) - if err != nil { - return nil, 0, err - } - accounts := make([]*rbacframe.Account, 0, resp.Count) - for _, v := range resp.Kvs { - a := &rbacframe.Account{} - err = json.Unmarshal(v.Value, a) - if err != nil { - log.Error("account info format invalid:", err) - continue //do not fail if some account is invalid - } - a.Password = "" - accounts = append(accounts, a) - } - return accounts, resp.Count, nil -} -func (ds *DataSource) DeleteAccount(ctx context.Context, key string) (bool, error) { - resp, err := backend.Registry().Do(ctx, registry.DEL, - registry.WithStrKey(GenerateETCDAccountKey(key))) - if err != nil { - return false, err - } - return resp.Count != 0, nil -} -func (ds *DataSource) UpdateAccount(ctx context.Context, key string, account *rbacframe.Account) error { - value, err := json.Marshal(account) - if err != nil { - log.Errorf(err, "account info is invalid") - return err - } - _, err = backend.Registry().Do(ctx, registry.PUT, - registry.WithStrKey(GenerateETCDAccountKey(key)), - registry.WithValue(value)) - return err -} - -func (ds *DataSource) AddDomain(ctx context.Context, domain string) (bool, error) { - ok, err := backend.Registry().PutNoOverride(ctx, - registry.WithStrKey(GenerateETCDDomainKey(domain))) - if err != nil { - return false, err - } - return ok, nil -} - -func (ds *DataSource) DomainExist(ctx context.Context, domain string) (bool, error) { - opts := append(utils.FromContext(ctx), - registry.WithStrKey(GenerateETCDDomainKey(domain)), - registry.WithCountOnly()) - rsp, err := backend.Store().Domain().Search(ctx, opts...) - if err != nil { - return false, err - } - return rsp.Count > 0, nil -} - -func (ds *DataSource) AddProject(ctx context.Context, domain, project string) (bool, error) { - ok, err := backend.Registry().PutNoOverride(ctx, - registry.WithStrKey(GenerateETCDProjectKey(domain, project))) - if err != nil { - return ok, err - } - return ok, nil -} - -func (ds *DataSource) ProjectExist(ctx context.Context, domain, project string) (bool, error) { - opts := append(utils.FromContext(ctx), - registry.WithStrKey(GenerateETCDProjectKey(domain, project)), - registry.WithCountOnly()) - rsp, err := backend.Store().Project().Search(ctx, opts...) - if err != nil { - return false, err - } - return rsp.Count > 0, nil -} diff --git a/server/service/auth/etcd/etcd_test.go b/server/service/auth/etcd/etcd_test.go deleted file mode 100644 index 7e2b418..0000000 --- a/server/service/auth/etcd/etcd_test.go +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package etcd - -import ( - "context" - "github.com/apache/servicecomb-service-center/pkg/rbacframe" - mgr "github.com/apache/servicecomb-service-center/server/plugin" - "github.com/apache/servicecomb-service-center/server/plugin/discovery/etcd" - etcd2 "github.com/apache/servicecomb-service-center/server/plugin/registry/etcd" - "github.com/apache/servicecomb-service-center/server/plugin/tracing/pzipkin" - "github.com/apache/servicecomb-service-center/server/service/auth" - "github.com/astaxie/beego" - "github.com/stretchr/testify/assert" - "testing" -) - -var ( - a1 = rbacframe.Account{ - ID: "11111-22222-33333", - Name: "test-account1", - Password: "tnuocca-tset", - Role: "admin", - TokenExpirationTime: "2020-12-30", - CurrentPassword: "tnuocca-tset1", - } - a2 = rbacframe.Account{ - ID: "11111-22222-33333-44444", - Name: "test-account2", - Password: "tnuocca-tset", - Role: "admin", - TokenExpirationTime: "2020-12-30", - CurrentPassword: "tnuocca-tset1", - } -) - -func init() { - beego.AppConfig.Set("registry_plugin", "etcd") - mgr.RegisterPlugin(mgr.Plugin{PName: mgr.REGISTRY, Name: "etcd", New: etcd2.NewRegistry}) - mgr.RegisterPlugin(mgr.Plugin{PName: mgr.DISCOVERY, Name: "buildin", New: etcd.NewRepository}) - mgr.RegisterPlugin(mgr.Plugin{PName: mgr.DISCOVERY, Name: "etcd", New: etcd.NewRepository}) - mgr.RegisterPlugin(mgr.Plugin{PName: mgr.TRACING, Name: "buildin", New: pzipkin.New}) - auth.Install("etcd", func(opts auth.Options) (auth.DataSource, error) { - return NewDataSource(), nil - }) - err := auth.Init(auth.Options{ - Endpoint: "", - PluginImplName: "etcd", - }) - if err != nil { - panic("failed to register etcd auth plugin") - } -} - -func TestAccount(t *testing.T) { - t.Run("add and get account", func(t *testing.T) { - err := auth.Auth().UpdateAccount(context.Background(), "test-account-key", &a1) - assert.NoError(t, err) - r, err := auth.Auth().GetAccount(context.Background(), "test-account-key") - assert.NoError(t, err) - assert.Equal(t, a1, *r) - }) - t.Run("account should exist", func(t *testing.T) { - exist, err := auth.Auth().AccountExist(context.Background(), "test-account-key") - assert.NoError(t, err) - assert.True(t, exist) - }) - t.Run("delete account", func(t *testing.T) { - err := auth.Auth().UpdateAccount(context.Background(), "test-account-key222", &a1) - assert.NoError(t, err) - _, err = auth.Auth().DeleteAccount(context.Background(), "test-account-key222") - assert.NoError(t, err) - }) - t.Run("add two accounts and list", func(t *testing.T) { - err := auth.Auth().UpdateAccount(context.Background(), "key1", &a1) - assert.NoError(t, err) - err = auth.Auth().UpdateAccount(context.Background(), "key2", &a2) - assert.NoError(t, err) - accs, n, err := auth.Auth().ListAccount(context.Background(), "key") - assert.NoError(t, err) - assert.Equal(t, int64(2), n) - t.Log(accs) - }) -} - -func TestDomain(t *testing.T) { - t.Run("test domain", func(t *testing.T) { - _, err := auth.Auth().AddDomain(context.Background(), "test-domain") - assert.NoError(t, err) - r, err := auth.Auth().DomainExist(context.Background(), "test-domain") - assert.NoError(t, err) - assert.Equal(t, true, r) - }) -} - -func TestProject(t *testing.T) { - t.Run("test project", func(t *testing.T) { - _, err := auth.Auth().AddProject(context.Background(), "test-domain", "test-project") - assert.NoError(t, err) - r, err := auth.Auth().ProjectExist(context.Background(), "test-domain", "test-project") - assert.NoError(t, err) - assert.Equal(t, true, r) - }) -} diff --git a/server/service/auth/etcd/key_generator.go b/server/service/auth/etcd/key_generator.go deleted file mode 100644 index d2c83bb..0000000 --- a/server/service/auth/etcd/key_generator.go +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package etcd - -import "github.com/apache/servicecomb-service-center/pkg/util" - -const ( - RegistryRootKey = "cse-sr" - RegistryProjectKey = "projects" - SPLIT = "/" - RegistryDomainKey = "domains" -) - -func GetRootKey() string { - return SPLIT + RegistryRootKey -} - -func (ds *DataSource) GenerateAccountKey(name string) string { - return util.StringJoin([]string{ - GetRootKey(), - "accounts", - name, - }, SPLIT) -} - -func GenerateETCDAccountKey(name string) string { - return util.StringJoin([]string{ - GetRootKey(), - "accounts", - name, - }, SPLIT) -} - -func GenerateETCDProjectKey(domain, project string) string { - return util.StringJoin([]string{ - GetProjectRootKey(domain), - project, - }, SPLIT) -} - -func GetProjectRootKey(domain string) string { - return util.StringJoin([]string{ - GetRootKey(), - RegistryProjectKey, - domain, - }, SPLIT) -} - -func GenerateETCDDomainKey(domain string) string { - return util.StringJoin([]string{ - GetDomainRootKey(), - domain, - }, SPLIT) -} - -func GetDomainRootKey() string { - return util.StringJoin([]string{ - GetRootKey(), - RegistryDomainKey, - }, SPLIT) -} diff --git a/server/service/auth/etcd/key_generator_test.go b/server/service/auth/etcd/key_generator_test.go deleted file mode 100644 index 80d4a3a..0000000 --- a/server/service/auth/etcd/key_generator_test.go +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package etcd - -import ( - "github.com/stretchr/testify/assert" - "testing" -) - -func TestGenerateETCDAccountKey(t *testing.T) { - assert.Equal(t, "/cse-sr/accounts/admin", GenerateETCDAccountKey("admin")) -} - -func TestGenerateETCDProjectKey(t *testing.T) { - assert.Equal(t, "/cse-sr/projects/domain/project", GenerateETCDProjectKey("domain", "project")) -} - -func TestGenerateETCDDomainKey(t *testing.T) { - assert.Equal(t, "/cse-sr/domains/domain", GenerateETCDDomainKey("domain")) -} diff --git a/server/service/auth/manager.go b/server/service/auth/manager.go deleted file mode 100644 index c772e23..0000000 --- a/server/service/auth/manager.go +++ /dev/null @@ -1,58 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package auth - -import ( - "fmt" - "github.com/apache/servicecomb-service-center/pkg/log" -) - -type dataSourceEngine func(opts Options) (DataSource, error) - -var ( - plugins = make(map[ImplName]dataSourceEngine) - authDataSourceInst DataSource -) - -// load plugins configuration into plugins -func Install(pluginImplName string, engineFunc dataSourceEngine) { - plugins[ImplName(pluginImplName)] = engineFunc -} - -// construct storage plugin instance -// invoked by sc main process -func Init(opts Options) error { - if opts.PluginImplName == "" { - return nil - } - - authDataSourceEngine, ok := plugins[opts.PluginImplName] - if !ok { - return fmt.Errorf("plugin implement not supported [%s]", opts.PluginImplName) - } - var err error - authDataSourceInst, err = authDataSourceEngine(opts) - if err != nil { - return err - } - log.Info(fmt.Sprintf("storage shim plugin [%s@%s] enabled", opts.PluginImplName, opts.Endpoint)) - return nil -} - -// usage: auth.Auth().CreateAccount() -func Auth() DataSource { - return authDataSourceInst -} diff --git a/server/service/auth/manager_test.go b/server/service/auth/manager_test.go deleted file mode 100644 index 40c77a9..0000000 --- a/server/service/auth/manager_test.go +++ /dev/null @@ -1,48 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package auth - -import ( - "github.com/apache/servicecomb-service-center/server/service/auth/etcd" - "github.com/go-chassis/go-archaius" - "github.com/stretchr/testify/assert" - "testing" -) - -func TestInit(t *testing.T) { - _ = archaius.Init(archaius.WithMemorySource()) - _ = archaius.Set("servicecomb.auth.name", "etcd") - t.Run("circuit datasource auth", func(t *testing.T) { - err := Init(Options{ - Endpoint: "", - PluginImplName: "", - }) - assert.NoError(t, err) - }) - t.Run("install and init", func(t *testing.T) { - Install("etcd", - func(opts Options) (DataSource, error) { - return etcd.NewDataSource(), nil - }) - - // sc main function initialize step - err := Init(Options{ - Endpoint: "", - PluginImplName: ImplName(archaius.GetString("servicecomb.auth.name", "etcd")), - }) - assert.NoError(t, err) - }) -} diff --git a/server/service/auth/options.go b/server/service/auth/options.go deleted file mode 100644 index 3166411..0000000 --- a/server/service/auth/options.go +++ /dev/null @@ -1,22 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package auth - -//Options contains configuration for plugins -type Options struct { - Endpoint string - PluginImplName ImplName -} diff --git a/server/service/auth/types.go b/server/service/auth/types.go deleted file mode 100644 index 01355e3..0000000 --- a/server/service/auth/types.go +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package auth - -type ImplName string
