tianxiaoliang commented on a change in pull request #789: URL: https://github.com/apache/servicecomb-service-center/pull/789#discussion_r547747631
########## File path: datasource/mongo/sd/listwatch.go ########## @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sd + +import ( + "context" + "fmt" + "time" + + "go.mongodb.org/mongo-driver/bson" +) + +type ListWatchConfig struct { + Timeout time.Duration + Context context.Context +} + +func (lo *ListWatchConfig) String() string { + return fmt.Sprintf("{timeout: %s}", lo.Timeout) +} + +type ListWatch interface { + List(op ListWatchConfig) (*MongoListWatchResponse, error) + // not support new multiple watchers + Watch(op ListWatchConfig) Watcher + + DoWatch(ctx context.Context, f func(*MongoListWatchResponse)) error Review comment: 这些接口不能复用etcd么，不要绑定mongodb的概念，etcd和mongo应该有个共通的抽象 ########## File path: datasource/mongo/sd/listwatch_inner.go ########## @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sd + +import ( + "context" + "fmt" + + "github.com/apache/servicecomb-service-center/datasource/mongo/client" + "github.com/apache/servicecomb-service-center/pkg/log" + "go.mongodb.org/mongo-driver/bson" + md "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +type innerListWatch struct { + Key string + resumeToken bson.Raw +} + +func (lw *innerListWatch) List(op ListWatchConfig) (*MongoListWatchResponse, error) { + otCtx, cancel := context.WithTimeout(op.Context, op.Timeout) + defer cancel() + + resp, err := client.GetMongoClient().Find(otCtx, lw.Key, bson.M{}) + if err != nil { + log.Error(fmt.Sprintf("list key %s failed", lw.Key), err) + return nil, err + } + + lwRsp := &MongoListWatchResponse{} + lwRsp.Infos = make([]MongoInfo, 0) + for resp.Next(context.Background()) { + info := lw.doParseDocumentToMongoInfo(resp.Current) + lwRsp.Infos = append(lwRsp.Infos, info) + } + + return lwRsp, nil +} + +func (lw *innerListWatch) ResumeToken() bson.Raw { + return lw.resumeToken +} + +func (lw *innerListWatch) setResumeToken(resumeToken bson.Raw) { + lw.resumeToken = resumeToken +} + +func (lw *innerListWatch) Watch(op ListWatchConfig) Watcher { + return newInnerWatcher(lw, op) +} + +func (lw *innerListWatch) DoWatch(ctx context.Context, f func(*MongoListWatchResponse)) error { + csOptions := &options.ChangeStreamOptions{} + csOptions.SetFullDocument(options.UpdateLookup) + + resumeToken := lw.ResumeToken() + if resumeToken != nil { + csOptions.SetResumeAfter(resumeToken) + } + + resp, err := client.GetMongoClient().Watch(ctx, lw.Key, md.Pipeline{}, csOptions) + + if err != nil { + log.Error(fmt.Sprintf("watch table %s failed", lw.Key), err) + f(nil) + return err + } + + for resp.Next(ctx) { + lwRsp := &MongoListWatchResponse{} + + lw.setResumeToken(resp.ResumeToken()) + + wRsp := &MongoWatchResponse{} + err := bson.Unmarshal(resp.Current, &wRsp) + + if err != nil { + log.Error("error to parse bson raw to mongo watch response", err) + return err + } + + info := lw.doParseWatchRspToMongoInfo(wRsp) + + lwRsp.OperationType = wRsp.OperationType + lwRsp.Infos = append(lwRsp.Infos, info) + + f(lwRsp) + } + + return err +} + +func (lw *innerListWatch) doParseDocumentToMongoInfo(fullDocument bson.Raw) (info MongoInfo) { + var err error + + documentID := MongoDocument{} + err = bson.Unmarshal(fullDocument, &documentID) + if err != nil { + return + } + + info.DocumentID = documentID.ID.Hex() + + switch lw.Key { + case INSTANCE: + instance := Instance{} + err = bson.Unmarshal(fullDocument, &instance) + if err != nil { + log.Error("error to parse bson raw to documentInfo", err) + return + } + info.BusinessID = instance.InstanceInfo.InstanceId + info.Value = instance + case SERVICE: + service := Service{} + err := bson.Unmarshal(fullDocument, &service) + if err != nil { + log.Error("error to parse bson raw to documentInfo", err) + return + } + info.BusinessID = service.ServiceInfo.ServiceId + info.Value = service + default: + return + } + return +} + +func (lw *innerListWatch) doParseWatchRspToMongoInfo(wRsp *MongoWatchResponse) (info MongoInfo) { + switch wRsp.OperationType { + case DELETE: + //delete operation has no fullDocumentValue + info.DocumentID = wRsp.DocumentKey.ID.Hex() + return + case INSERT, UPDATE, REPLACE: + return lw.doParseDocumentToMongoInfo(wRsp.FullDocument) Review comment: 应有default分支做日志记录 ########## File path: datasource/mongo/sd/types.go ########## @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sd + +import ( + "time" + + "github.com/go-chassis/cari/discovery" + pb "github.com/go-chassis/cari/discovery" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +const ( + SERVICE = "service" Review comment: 常量名，另外etcd没定义么？ ########## File path: datasource/mongo/sd/mongo_cache.go ########## @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sd + +import ( + "sync" + + "github.com/apache/servicecomb-service-center/pkg/util" +) + +// MongoCache implements Cache. +// MongoCache is dedicated to stores service discovery data, +// e.g. service, instance, lease. +type MongoCache struct { + Cfg *Config + name string + store map[string]interface{} + documentStore map[string]string + rwMux sync.RWMutex + dirty bool +} + +func (c *MongoCache) Name() string { + return c.name +} + +func (c *MongoCache) Size() (l int) { + c.rwMux.RLock() + l = int(util.Sizeof(c.store)) + c.rwMux.RUnlock() + return +} + +func (c *MongoCache) Get(id string) (v interface{}) { + c.rwMux.RLock() + if p, ok := c.store[id]; ok { + v = p + } + c.rwMux.RUnlock() + return +} + +func (c *MongoCache) GetKeyByDocumentID(documentKey string) (id string) { + c.rwMux.RLock() + id = c.documentStore[documentKey] + c.rwMux.RUnlock() + return +} + +func (c *MongoCache) GetDocumentIDByID(id string) (documentID string) { + c.rwMux.RLock() + for k, v := range c.documentStore { + if v == id { + documentID = k + break + } + } + c.rwMux.RUnlock() + return +} + +func (c *MongoCache) Put(id string, v interface{}) { + c.rwMux.Lock() + c.store[id] = v + c.rwMux.Unlock() +} + +func (c *MongoCache) PutDocumentID(id string, documentID string) { + c.rwMux.Lock() + c.documentStore[documentID] = id + c.rwMux.Unlock() +} + +func (c *MongoCache) Remove(id string) { + c.rwMux.Lock() + delete(c.store, id) + c.rwMux.Unlock() +} + +func (c *MongoCache) RemoveDocumentID(documentID string) { + c.rwMux.Lock() + + delete(c.documentStore, documentID) + + c.rwMux.Unlock() +} + +func (c *MongoCache) MarkDirty() { + c.dirty = true +} + +func (c *MongoCache) Dirty() bool { return c.dirty } + +func (c *MongoCache) Clear() { + c.rwMux.Lock() + c.dirty = false + c.store = make(map[string]interface{}) + c.rwMux.Unlock() +} + +func (c *MongoCache) ForEach(iter func(k string, v interface{}) (next bool)) { + c.rwMux.RLock() +loopParent: + for k, v := range c.store { + if v == nil { + continue loopParent + } + if !iter(k, v) { + break loopParent + } + } + c.rwMux.RUnlock() +} + +func NewMongoCache(name string, cfg *Config) *MongoCache { + return &MongoCache{ + Cfg: cfg, + name: name, Review comment: 缓存没有过期时间？那么如何保证数据最终一致，通过list and watch？ ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
