This is an automated email from the ASF dual-hosted git repository. asifdxtreme pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/servicecomb-kie.git
commit 95972504161e9920fead916288bf817f22d9eada Author: tian <xiaoliang.t...@gmail.com> AuthorDate: Thu Jun 6 17:27:05 2019 +0800 add license, unify response kv struct --- pkg/common/common.go | 1 + pkg/model/kv.go | 6 ++--- pkg/model/mongodb_doc.go | 2 +- proxy.sh | 15 ++++++++++- scripts/start.sh | 18 ++++++++++++- server/dao/kie_api.go | 53 ++++++++++++++++++++++++++++++++++----- server/dao/kv.go | 2 +- server/dao/kv_test.go | 20 ++++++++++++--- server/dao/options.go | 8 ++++++ server/resource/v1/common.go | 16 +++++++++++- server/resource/v1/kv_resource.go | 26 +++++++++++++------ 11 files changed, 141 insertions(+), 26 deletions(-) diff --git a/pkg/common/common.go b/pkg/common/common.go index 47e224e..345086a 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -24,4 +24,5 @@ const ( const ( HeaderMatch = "X-Match" + HeaderDepth = "X-Depth" ) diff --git a/pkg/model/kv.go b/pkg/model/kv.go index be9830e..332c074 100644 --- a/pkg/model/kv.go +++ b/pkg/model/kv.go @@ -17,9 +17,7 @@ package model - - - - type KVResponse struct { + LabelDoc *LabelDoc `json:"labels"` + Data []*KVDoc `json:"data"` } diff --git a/pkg/model/mongodb_doc.go b/pkg/model/mongodb_doc.go index a298ca2..7645a14 100644 --- a/pkg/model/mongodb_doc.go +++ b/pkg/model/mongodb_doc.go @@ -22,7 +22,7 @@ import "go.mongodb.org/mongo-driver/bson/primitive" type LabelDoc struct { ID primitive.ObjectID `json:"_id,omitempty" bson:"_id,omitempty"` Labels map[string]string `json:"labels,omitempty"` - Revision int `json:"revision"` + Revision int `json:"revision,omitempty"` Domain string `json:"domain,omitempty"` //tenant info } type KVDoc struct { diff --git a/proxy.sh b/proxy.sh index da4430b..5fcc641 100755 --- a/proxy.sh +++ b/proxy.sh @@ -1,3 +1,16 @@ #!/usr/bin/env bash - +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. export GOPROXY=https://goproxy.io diff --git a/scripts/start.sh b/scripts/start.sh index 35e62ea..682634c 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -19,7 +19,9 @@ root_dir=/opt/servicecomb-kie net_name=$(ip -o -4 route show to default | awk '{print $5}') listen_addr=$(ifconfig ${net_name} | grep -E 'inet\W' | grep -o -E [0-9]+.[0-9]+.[0-9]+.[0-9]+ | head -n 1) - +if [ -z "${LOG_LEVEL}" ]; then + export LOG_LEVEL="DEBUG" +fi writeConfig(){ echo "write template config..." @@ -38,7 +40,21 @@ cse: Provider: default: auth-handler,ratelimiter-provider EOM +cat <<EOM > ${root_dir}/conf/lager.yaml +logger_level: ${LOG_LEVEL} + +logger_file: log/chassis.log + +log_format_text: true + +rollingPolicy: size +log_rotate_date: 1 + +log_rotate_size: 10 + +log_backup_count: 7 +EOM cat <<EOM > /etc/servicecomb-kie/kie-conf.yaml db: uri: mongodb://${MONGODB_USER}:${MONGODB_PWD}@${MONGODB_ADDR}/kie diff --git a/server/dao/kie_api.go b/server/dao/kie_api.go index c5ae9bd..aaf613f 100644 --- a/server/dao/kie_api.go +++ b/server/dao/kie_api.go @@ -26,6 +26,7 @@ import ( "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" + "reflect" "time" ) @@ -184,7 +185,7 @@ func (s *MongodbService) FindKVByLabelID(ctx context.Context, domain, labelID, k //FindKV get kvs by key, labels //because labels has a a lot of combination, //you can use WithExactLabels to return only one kv which's labels exactly match the criteria -func (s *MongodbService) FindKV(ctx context.Context, domain string, options ...FindOption) ([]*model.KVDoc, error) { +func (s *MongodbService) FindKV(ctx context.Context, domain string, options ...FindOption) ([]*model.KVResponse, error) { opts := FindOptions{} for _, o := range options { o(&opts) @@ -213,9 +214,11 @@ func (s *MongodbService) FindKV(ctx context.Context, domain string, options ...F if cur.Err() != nil { return nil, err } + kvResp := make([]*model.KVResponse, 0) if opts.ExactLabels { openlogging.Debug(fmt.Sprintf("find one [%s] with lables [%s] in [%s]", opts.Key, opts.Labels, domain)) curKV := &model.KVDoc{} //reuse this pointer to reduce GC, only clear label + //check label length to get the exact match for cur.Next(ctx) { //although complexity is O(n), but there won't be so much labels for one key curKV.Labels = nil @@ -226,27 +229,65 @@ func (s *MongodbService) FindKV(ctx context.Context, domain string, options ...F } if len(curKV.Labels) == len(opts.Labels) { openlogging.Debug("hit exact labels") + curKV.Domain = "" curKV.Labels = nil //exact match don't need to return labels - return []*model.KVDoc{curKV}, nil + labelGroup := &model.KVResponse{ + LabelDoc: &model.LabelDoc{ + Labels: opts.Labels, + ID: primitive.NilObjectID, + }, + Data: make([]*model.KVDoc, 0), + } + labelGroup.Data = append(labelGroup.Data, curKV) + kvResp = append(kvResp, labelGroup) + return kvResp, nil } } return nil, ErrKeyNotExists } else { - kvs := make([]*model.KVDoc, 0) + if opts.Depth == 0 { + opts.Depth = 1 + } for cur.Next(ctx) { curKV := &model.KVDoc{} + if err := cur.Decode(curKV); err != nil { openlogging.Error("decode to KVs error: " + err.Error()) return nil, err } - kvs = append(kvs, curKV) + if (len(curKV.Labels) - len(opts.Labels)) > opts.Depth { + //because it is query by labels, so result can not be minus + //so many labels,then continue + openlogging.Debug("so deep, skip this key") + continue + } + openlogging.Info(fmt.Sprintf("%v", curKV)) + var groupExist bool + var labelGroup *model.KVResponse + for _, labelGroup = range kvResp { + if reflect.DeepEqual(labelGroup.LabelDoc.Labels, curKV.Labels) { + groupExist = true + labelGroup.Data = append(labelGroup.Data, curKV) + break + } + } + if !groupExist { + labelGroup = &model.KVResponse{ + LabelDoc: &model.LabelDoc{ + Labels: curKV.Labels, + }, + Data: []*model.KVDoc{curKV}, + } + openlogging.Debug("add new label group") + } + kvResp = append(kvResp, labelGroup) } - if len(kvs) == 0 { + if len(kvResp) == 0 { return nil, ErrKeyNotExists } - return kvs, nil + return kvResp, nil } } diff --git a/server/dao/kv.go b/server/dao/kv.go index 1f7eccf..c48774b 100644 --- a/server/dao/kv.go +++ b/server/dao/kv.go @@ -99,7 +99,7 @@ func (s *MongodbService) KVExist(ctx context.Context, domain, key string, option return primitive.NilObjectID, ErrTooMany } - return kvs[0].ID, nil + return kvs[0].Data[0].ID, nil } } diff --git a/server/dao/kv_test.go b/server/dao/kv_test.go index 7e2a5eb..efbe6d4 100644 --- a/server/dao/kv_test.go +++ b/server/dao/kv_test.go @@ -96,7 +96,7 @@ var _ = Describe("Kv mongodb service", func() { "app": "mall", }), dao.WithExactLabels()) It("should be 1s", func() { - Expect(kvs1[0].Value).Should(Equal(beforeKV.Value)) + Expect(kvs1[0].Data[0].Value).Should(Equal(beforeKV.Value)) }) afterKV, err := s.CreateOrUpdate(context.Background(), "default", &model.KVDoc{ Key: "timeout", @@ -119,19 +119,33 @@ var _ = Describe("Kv mongodb service", func() { "app": "mall", }), dao.WithExactLabels()) It("should be 3s", func() { - Expect(kvs[0].Value).Should(Equal(afterKV.Value)) + Expect(kvs[0].Data[0].Value).Should(Equal(afterKV.Value)) }) }) }) Describe("greedy find by kv and labels", func() { - Context("with labels app ", func() { + Context("with labels app,depth is 1 ", func() { kvs, err := s.FindKV(context.Background(), "default", dao.WithKey("timeout"), dao.WithLabels(map[string]string{ "app": "mall", })) It("should not return err", func() { Expect(err).Should(BeNil()) }) + It("should has 2 records", func() { + Expect(len(kvs)).Should(Equal(2)) + }) + + }) + Context("with labels app,depth is 2 ", func() { + kvs, err := s.FindKV(context.Background(), "default", dao.WithKey("timeout"), + dao.WithLabels(map[string]string{ + "app": "mall", + }), + dao.WithDepth(2)) + It("should not return err", func() { + Expect(err).Should(BeNil()) + }) It("should has 3 records", func() { Expect(len(kvs)).Should(Equal(3)) }) diff --git a/server/dao/options.go b/server/dao/options.go index e1af384..7e33798 100644 --- a/server/dao/options.go +++ b/server/dao/options.go @@ -19,6 +19,7 @@ package dao type FindOptions struct { ExactLabels bool + Depth int Key string Labels map[string]string LabelID string @@ -55,6 +56,13 @@ func WithLabelID(label string) FindOption { } } +//WithDepth if you use greedy match this can specify the match depth +func WithDepth(d int) FindOption { + return func(o *FindOptions) { + o.Depth = d + } +} + //WithOutLabelField will clear all labels attributes in kv doc func WithOutLabelField() FindOption { return func(o *FindOptions) { diff --git a/server/resource/v1/common.go b/server/resource/v1/common.go index 2293b72..eca9fef 100644 --- a/server/resource/v1/common.go +++ b/server/resource/v1/common.go @@ -24,22 +24,36 @@ import ( "github.com/apache/servicecomb-kie/pkg/model" "github.com/go-chassis/go-chassis/server/restful" "github.com/go-mesh/openlogging" + "strconv" ) const ( - TenantHeaderParam = "X-Domain-Name" + HeaderTenant = "X-Domain-Name" + FindExact = "exact" FindMany = "greedy" MsgDomainMustNotBeEmpty = "domain must not be empty" MsgIllegalFindPolicy = "value of header " + common.HeaderMatch + " can be greedy or exact" MsgIllegalLabels = "label's value can not be empty, " + "label can not be duplicated, please check your query parameters" + MsgIllegalDepth = "X-Depth must be number" ErrIDMustNotEmpty = "must supply id if you want to remove key" ) func ReadDomain(context *restful.Context) interface{} { return context.ReadRestfulRequest().Attribute("domain") } +func ReadFindDepth(context *restful.Context) (int, error) { + d := context.ReadRestfulRequest().HeaderParameter(common.HeaderDepth) + if d == "" { + return 1, nil + } + depth, err := strconv.Atoi(d) + if err != nil { + return 0, err + } + return depth, nil +} func ReadMatchPolicy(context *restful.Context) string { policy := context.ReadRestfulRequest().HeaderParameter(common.HeaderMatch) if policy == "" { diff --git a/server/resource/v1/kv_resource.go b/server/resource/v1/kv_resource.go index db2d2b6..7832f23 100644 --- a/server/resource/v1/kv_resource.go +++ b/server/resource/v1/kv_resource.go @@ -91,10 +91,15 @@ func (r *KVResource) FindWithKey(context *restful.Context) { return } policy := ReadMatchPolicy(context) - var kvs []*model.KVDoc + d, err := ReadFindDepth(context) + if err != nil { + WriteErrResponse(context, http.StatusBadRequest, MsgIllegalDepth) + return + } + var kvs []*model.KVResponse switch policy { case common.MatchGreedy: - kvs, err = s.FindKV(context.Ctx, domain.(string), dao.WithKey(key), dao.WithLabels(labels)) + kvs, err = s.FindKV(context.Ctx, domain.(string), dao.WithKey(key), dao.WithLabels(labels), dao.WithDepth(d)) case common.MatchExact: kvs, err = s.FindKV(context.Ctx, domain.(string), dao.WithKey(key), dao.WithLabels(labels), dao.WithExactLabels()) @@ -138,10 +143,15 @@ func (r *KVResource) FindByLabels(context *restful.Context) { return } policy := ReadMatchPolicy(context) - var kvs []*model.KVDoc + d, err := ReadFindDepth(context) + if err != nil { + WriteErrResponse(context, http.StatusBadRequest, MsgIllegalDepth) + return + } + var kvs []*model.KVResponse switch policy { case common.MatchGreedy: - kvs, err = s.FindKV(context.Ctx, domain.(string), dao.WithLabels(labels)) + kvs, err = s.FindKV(context.Ctx, domain.(string), dao.WithLabels(labels), dao.WithDepth(d)) case common.MatchExact: kvs, err = s.FindKV(context.Ctx, domain.(string), dao.WithLabels(labels), dao.WithExactLabels()) @@ -199,7 +209,7 @@ func (r *KVResource) URLPatterns() []restful.Route { ParamType: goRestful.PathParameterKind, }, { DataType: "string", - Name: TenantHeaderParam, + Name: HeaderTenant, ParamType: goRestful.HeaderParameterKind, Desc: "set kv to other tenant", }, { @@ -230,7 +240,7 @@ func (r *KVResource) URLPatterns() []restful.Route { ParamType: goRestful.PathParameterKind, }, { DataType: "string", - Name: TenantHeaderParam, + Name: HeaderTenant, ParamType: goRestful.HeaderParameterKind, }, { DataType: "string", @@ -257,7 +267,7 @@ func (r *KVResource) URLPatterns() []restful.Route { Parameters: []*restful.Parameters{ { DataType: "string", - Name: TenantHeaderParam, + Name: HeaderTenant, ParamType: goRestful.HeaderParameterKind, }, { DataType: "string", @@ -283,7 +293,7 @@ func (r *KVResource) URLPatterns() []restful.Route { Parameters: []*restful.Parameters{ { DataType: "string", - Name: TenantHeaderParam, + Name: HeaderTenant, ParamType: goRestful.HeaderParameterKind, }, { DataType: "string",