This is an automated email from the ASF dual-hosted git repository. liujiapeng pushed a commit to branch override-by-env in repository https://gitbox.apache.org/repos/asf/skywalking-satellite.git
commit 01f8d56d2da6c1628e4315b5e428d661bead2974 Author: Evan <[email protected]> AuthorDate: Sun Jan 24 17:35:48 2021 +0800 override by env --- configs/satellite_config.yaml | 34 ++-- internal/satellite/config/loader.go | 6 +- internal/satellite/config/override_by_env.go | 114 ++++++++++++ internal/satellite/config/override_by_env_test.go | 203 ++++++++++++++++++++++ internal/satellite/config/satellite_config.go | 4 +- 5 files changed, 341 insertions(+), 20 deletions(-) diff --git a/configs/satellite_config.yaml b/configs/satellite_config.yaml index fc48810..4426808 100644 --- a/configs/satellite_config.yaml +++ b/configs/satellite_config.yaml @@ -15,27 +15,31 @@ # limitations under the License. # +# The logger configuration. logger: - log_pattern: "%time [%level][%field] - %msg" - time_pattern: "2006-01-02 15:04:05.000" - level: "info" + log_pattern: ${SATELLITE_LOGGER_LOG_PATTERN:%time [%level][%field] - %msg} + time_pattern: ${SATELLITE_LOGGER_TIME_PATTERN:2006-01-02 15:04:05.000} + level: ${SATELLITE_LOGGER_LEVEL:info} +# The Satellite self telemetry configuration. telemetry: - cluster: cluster1 - service: service1 - instance: instance1 + cluster: ${SATELLITE_TELEMETRY_CLUSTER:default-cluster} + service: ${SATELLITE_TELEMETRY_SERVICE:default-service} + instance: ${SATELLITE_TELEMETRY_SERVICE:default-instance} +# The sharing plugins referenced by the specific plugins in the different pipes. sharing: common_config: pipe_name: sharing clients: - plugin_name: "kafka-client" - brokers: 127.0.0.1:9092 - version: 2.1.1 + brokers: ${SATELLITE_KAFKA_CLIENT_BROKERS:127.0.0.1:9092} + version: ${SATELLITE_KAFKA_VERSION:"2.1.1"} servers: - plugin_name: "grpc-server" - plugin_name: "prometheus-server" - address: ":8090" + address: ${SATELLITE_PROMETHEUS_ADDRESS:":8090"} +# The working pipes. pipes: - common_config: pipe_name: pipe1 @@ -45,18 +49,18 @@ pipes: plugin_name: "grpc-nativelog-receiver" queue: plugin_name: "mmap-queue" - segment_size: 524288 - max_in_mem_segments: 6 + segment_size: ${SATELLITE_MMAP_QUEUE_SIZE:524288} + max_in_mem_segments: ${SATELLITE_MMAP_QUEUE_MAX_IN_MEM_SEGMENTS:6} queue_dir: "pipe1-log-grpc-receiver-queue" processor: filters: sender: fallbacker: plugin_name: none-fallbacker - flush_time: 1000 - max_buffer_size: 200 - min_flush_events: 100 + flush_time: ${SATELLITE_PIPE1_SENDER_FLUSH_TIME:1000} + max_buffer_size: ${SATELLITE_PIPE1_SENDER_MAX_BUFFER_SIZE:200} + min_flush_events: ${SATELLITE_PIPE1_SENDER_MIN_FLUSH_EVENTS:100} client_name: kafka-client forwarders: - plugin_name: nativelog-kafka-forwarder - topic: log-topic \ No newline at end of file + topic: ${SATELLITE_NATIVELOG-TOPIC:log-topic} \ No newline at end of file diff --git a/internal/satellite/config/loader.go b/internal/satellite/config/loader.go index 21c92f8..11c6032 100644 --- a/internal/satellite/config/loader.go +++ b/internal/satellite/config/loader.go @@ -61,14 +61,14 @@ func load(configPath string) (*SatelliteConfig, error) { return nil, err } v := viper.New() - v.AutomaticEnv() - v.SetEnvPrefix("satellite") - v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) v.SetConfigType("yaml") cfg := SatelliteConfig{} if err := v.ReadConfig(bytes.NewReader(content)); err != nil { return nil, err } + if err := overrideConfigByEnv(v); err != nil { + return nil, fmt.Errorf("cannot override value by env config: %v", err) + } if err := v.Unmarshal(&cfg); err != nil { return nil, err } diff --git a/internal/satellite/config/override_by_env.go b/internal/satellite/config/override_by_env.go new file mode 100644 index 0000000..88982ae --- /dev/null +++ b/internal/satellite/config/override_by_env.go @@ -0,0 +1,114 @@ +// Licensed to Apache Software Foundation (ASF) under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Apache Software Foundation (ASF) licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package config + +import ( + "os" + "regexp" + "strconv" + "strings" + + "github.com/spf13/viper" +) + +// ${ENV:defaultVal} pattern is supported to override the default value by the env config. +// And the value would be converted to the different types, such as int, float64 ana string. +// If you want to keep it as the string value, please decorate it with `"`. +const RegularExpression = "${_|[A-Z]+:.*}" + +func overrideConfigByEnv(v *viper.Viper) error { + regex, err := regexp.Compile(RegularExpression) + if err != nil { + return err + } + overrideCfg := overrideMapStringInterface(v.AllSettings(), regex) + for key, val := range overrideCfg { + v.Set(key, val) + } + return nil +} + +func overrideMapStringInterface(cfg map[string]interface{}, regex *regexp.Regexp) map[string]interface{} { + res := make(map[string]interface{}) + for key, val := range cfg { + switch val.(type) { + case string: + res[key] = overrideString(val.(string), regex) + case []interface{}: + res[key] = overrideSlice(val.([]interface{}), regex) + case map[string]interface{}: + res[key] = overrideMapStringInterface(val.(map[string]interface{}), regex) + case map[interface{}]interface{}: + res[key] = overrideMapInterfaceInterface(val.(map[interface{}]interface{}), regex) + default: + res[key] = val + } + } + return res +} + +func overrideSlice(m []interface{}, regex *regexp.Regexp) []interface{} { + res := make([]interface{}, 0) + for _, val := range m { + switch val.(type) { + case map[string]interface{}: + res = append(res, overrideMapStringInterface(val.(map[string]interface{}), regex)) + case map[interface{}]interface{}: + res = append(res, overrideMapInterfaceInterface(val.(map[interface{}]interface{}), regex)) + } + } + return res +} + +func overrideMapInterfaceInterface(m map[interface{}]interface{}, regex *regexp.Regexp) interface{} { + cfg := make(map[string]interface{}) + for key, val := range m { + cfg[key.(string)] = val + } + return overrideMapStringInterface(cfg, regex) +} + +func overrideString(s string, regex *regexp.Regexp) interface{} { + if regex.MatchString(s) { + index := strings.Index(s, ":") + envName := s[2:index] + defaultVal := s[index+1 : len(s)-1] + envVal := os.Getenv(envName) + if envVal != "" { + defaultVal = envVal + } + return parseVal(defaultVal) + } + return s +} + +func parseVal(val string) interface{} { + if intVal, err := strconv.Atoi(val); err == nil { + return intVal + } else if floatVal, err := strconv.ParseFloat(val, 64); err == nil { + return floatVal + } else if strings.EqualFold(val, "true") { + return true + } else if strings.EqualFold(val, "false") { + return false + } else if strings.HasPrefix(val, "\"") && strings.HasSuffix(val, "\"") { + return val[1 : len(val)-1] + } else { + return val + } +} diff --git a/internal/satellite/config/override_by_env_test.go b/internal/satellite/config/override_by_env_test.go new file mode 100644 index 0000000..32fd59d --- /dev/null +++ b/internal/satellite/config/override_by_env_test.go @@ -0,0 +1,203 @@ +// Licensed to Apache Software Foundation (ASF) under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Apache Software Foundation (ASF) licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package config + +import ( + "os" + "reflect" + "regexp" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func Test_overrideString(t *testing.T) { + type args struct { + s string + env string + regex string + } + tests := []struct { + name string + args args + want interface{} + }{ + { + name: "override_string", + args: args{ + s: "${TEST_OVERRIDE_STRING:test_str}", + env: "TEST_OVERRIDE_STRING=test_override_string", + regex: RegularExpression, + }, + want: "test_override_string", + }, + { + name: "no_override_string", + args: args{ + s: "${TEST_NO_OVERRIDE_STRING:test_str}", + env: "", + regex: RegularExpression, + }, + want: "test_str", + }, + { + name: "no_override_false", + args: args{ + s: "${TEST_NO_OVERRIDE_FALSE:false}", + env: "", + regex: RegularExpression, + }, + want: false, + }, + { + name: "no_override_true", + args: args{ + s: "${TEST_NO_OVERRIDE_TRUE:true}", + env: "", + regex: RegularExpression, + }, + want: true, + }, + { + name: "override_boolean", + args: args{ + s: "${TEST_OVERRIDE_BOOLEAN:true}", + env: "TEST_OVERRIDE_BOOLEAN=false", + regex: RegularExpression, + }, + want: false, + }, + { + name: "no_override_int", + args: args{ + s: "${TEST_OVERRIDE_INT:10}", + env: "", + regex: RegularExpression, + }, + want: 10, + }, + { + name: "override_int", + args: args{ + s: "${TEST_OVERRIDE_INT:10}", + env: "TEST_OVERRIDE_INT=15", + regex: RegularExpression, + }, + want: 15, + }, + { + name: "override_float", + args: args{ + s: "${TEST_OVERRIDE_FLOAT:10.5}", + env: "TEST_OVERRIDE_FLOAT=15.7", + regex: RegularExpression, + }, + want: 15.7, + }, + } + for _, tt := range tests { + if tt.args.env != "" { + envArr := strings.Split(tt.args.env, "=") + err := os.Setenv(envArr[0], envArr[1]) + if err != nil { + t.Fatalf("cannot set the env %s config: %v", tt.args.env, err) + } + } + regex, err := regexp.Compile(tt.args.regex) + if err != nil { + t.Fatalf("cannot generate the regular expression: %s", tt.args.regex) + } + + t.Run(tt.name, func(t *testing.T) { + if got := overrideString(tt.args.s, regex); !reflect.DeepEqual(got, tt.want) { + t.Errorf("overrideString() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_overrideMapStringInterface(t *testing.T) { + type args struct { + cfg map[string]interface{} + env map[string]string + } + tests := []struct { + name string + args args + want map[string]interface{} + }{ + { + name: "test-overrideByEnv", + args: args{ + cfg: map[string]interface{}{ + "stringKey": "${OVERRIDE_STRING_KEY:stringKey}", + "intKey": "${OVERRIDE_INT_KEY:10}", + "boolKey": "${OVERRIDE_BOOL_KEY:false}", + "mapKey": map[string]interface{}{ + "mapStringKey": "${OVERRIDE_STRING_KEY:stringKey}", + "mapIntKey": "${OVERRIDE_INT_KEY:10}", + "mapBoolKey": "${OVERRIDE_BOOL_KEY:false}", + "mapInterfaceKey": map[interface{}]interface{}{ + "mapinterfaceStringKey": "${OVERRIDE_STRING_KEY:stringKey}", + "mapinterfaceIntKey": "${OVERRIDE_INT_KEY:10}", + "mapinterfaceBoolKey": "${OVERRIDE_BOOL_KEY:false}", + }, + }, + }, + env: map[string]string{ + "OVERRIDE_STRING_KEY": "env-string", + "OVERRIDE_INT_KEY": "100", + "OVERRIDE_BOOL_KEY": "true", + }, + }, + want: map[string]interface{}{ + "stringKey": "env-string", + "intKey": 100, + "boolKey": true, + "mapKey": map[string]interface{}{ + "mapStringKey": "env-string", + "mapIntKey": 100, + "mapBoolKey": true, + "mapInterfaceKey": map[string]interface{}{ + "mapinterfaceStringKey": "env-string", + "mapinterfaceIntKey": 100, + "mapinterfaceBoolKey": true, + }, + }, + }, + }, + } + + for _, tt := range tests { + for k, v := range tt.args.env { + err := os.Setenv(k, v) + if err != nil { + t.Fatalf("cannot set the env config{%s=%s}: %v", k, v, err) + } + + } + t.Run(tt.name, func(t *testing.T) { + regex, _ := regexp.Compile(RegularExpression) + got := overrideMapStringInterface(tt.args.cfg, regex) + if !cmp.Equal(got, tt.want) { + t.Errorf("overrideConfigByEnv() got = %v, want = %v", got, tt.want) + } + }) + } +} diff --git a/internal/satellite/config/satellite_config.go b/internal/satellite/config/satellite_config.go index 3033692..8a999a0 100644 --- a/internal/satellite/config/satellite_config.go +++ b/internal/satellite/config/satellite_config.go @@ -30,9 +30,9 @@ import ( // SatelliteConfig is to initialize Satellite. type SatelliteConfig struct { Logger *log.LoggerConfig `mapstructure:"logger"` - Pipes []*PipeConfig `mapstructure:"pipes"` - Sharing *SharingConfig `mapstructure:"sharing"` Telemetry *telemetry.Config `mapstructure:"telemetry"` + Sharing *SharingConfig `mapstructure:"sharing"` + Pipes []*PipeConfig `mapstructure:"pipes"` } // SharingConfig contains some plugins,which could be shared by every namespace. That is useful to reduce resources cost.
