This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-satellite.git
The following commit(s) were added to refs/heads/main by this push:
new 456e87a override config by env (#20)
456e87a is described below
commit 456e87aa3014a4121ff0bdcc83877432b29dba8f
Author: Evan <[email protected]>
AuthorDate: Wed Jan 27 15:53:25 2021 +0800
override config by env (#20)
---
configs/satellite_config.yaml | 34 ++--
internal/satellite/config/loader.go | 6 +-
internal/satellite/config/override_by_env.go | 114 +++++++++++++
internal/satellite/config/override_by_env_test.go | 194 ++++++++++++++++++++++
internal/satellite/config/satellite_config.go | 4 +-
5 files changed, 332 insertions(+), 20 deletions(-)
diff --git a/configs/satellite_config.yaml b/configs/satellite_config.yaml
index fc48810..323e945 100644
--- a/configs/satellite_config.yaml
+++ b/configs/satellite_config.yaml
@@ -15,27 +15,31 @@
# limitations under the License.
#
+# The logger configuration.
logger:
- log_pattern: "%time [%level][%field] - %msg"
- time_pattern: "2006-01-02 15:04:05.000"
- level: "info"
+ log_pattern: ${SATELLITE_LOGGER_LOG_PATTERN:%time [%level][%field] - %msg}
+ time_pattern: ${SATELLITE_LOGGER_TIME_PATTERN:2006-01-02 15:04:05.000}
+ level: ${SATELLITE_LOGGER_LEVEL:info}
+# The Satellite self telemetry configuration.
telemetry:
- cluster: cluster1
- service: service1
- instance: instance1
+ cluster: ${SATELLITE_TELEMETRY_CLUSTER:default-cluster}
+ service: ${SATELLITE_TELEMETRY_SERVICE:default-service}
+ instance: ${SATELLITE_TELEMETRY_SERVICE:default-instance}
+# The sharing plugins referenced by the specific plugins in the different
pipes.
sharing:
common_config:
pipe_name: sharing
clients:
- plugin_name: "kafka-client"
- brokers: 127.0.0.1:9092
- version: 2.1.1
+ brokers: ${SATELLITE_KAFKA_CLIENT_BROKERS:127.0.0.1:9092}
+ version: ${SATELLITE_KAFKA_VERSION:"2.1.1"}
servers:
- plugin_name: "grpc-server"
- plugin_name: "prometheus-server"
- address: ":8090"
+ address: ${SATELLITE_PROMETHEUS_ADDRESS:":8090"}
+# The working pipes.
pipes:
- common_config:
pipe_name: pipe1
@@ -45,18 +49,18 @@ pipes:
plugin_name: "grpc-nativelog-receiver"
queue:
plugin_name: "mmap-queue"
- segment_size: 524288
- max_in_mem_segments: 6
+ segment_size: ${SATELLITE_MMAP_QUEUE_SIZE:524288}
+ max_in_mem_segments: ${SATELLITE_MMAP_QUEUE_MAX_IN_MEM_SEGMENTS:6}
queue_dir: "pipe1-log-grpc-receiver-queue"
processor:
filters:
sender:
fallbacker:
plugin_name: none-fallbacker
- flush_time: 1000
- max_buffer_size: 200
- min_flush_events: 100
+ flush_time: ${SATELLITE_PIPE1_SENDER_FLUSH_TIME:1000}
+ max_buffer_size: ${SATELLITE_PIPE1_SENDER_MAX_BUFFER_SIZE:200}
+ min_flush_events: ${SATELLITE_PIPE1_SENDER_MIN_FLUSH_EVENTS:100}
client_name: kafka-client
forwarders:
- plugin_name: nativelog-kafka-forwarder
- topic: log-topic
\ No newline at end of file
+ topic: ${SATELLITE_NATIVELOG-TOPIC:log-topic}
diff --git a/internal/satellite/config/loader.go
b/internal/satellite/config/loader.go
index 21c92f8..11c6032 100644
--- a/internal/satellite/config/loader.go
+++ b/internal/satellite/config/loader.go
@@ -61,14 +61,14 @@ func load(configPath string) (*SatelliteConfig, error) {
return nil, err
}
v := viper.New()
- v.AutomaticEnv()
- v.SetEnvPrefix("satellite")
- v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
v.SetConfigType("yaml")
cfg := SatelliteConfig{}
if err := v.ReadConfig(bytes.NewReader(content)); err != nil {
return nil, err
}
+ if err := overrideConfigByEnv(v); err != nil {
+ return nil, fmt.Errorf("cannot override value by env config:
%v", err)
+ }
if err := v.Unmarshal(&cfg); err != nil {
return nil, err
}
diff --git a/internal/satellite/config/override_by_env.go
b/internal/satellite/config/override_by_env.go
new file mode 100644
index 0000000..6ae678f
--- /dev/null
+++ b/internal/satellite/config/override_by_env.go
@@ -0,0 +1,114 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package config
+
+import (
+ "os"
+ "regexp"
+ "strconv"
+ "strings"
+
+ "github.com/spf13/viper"
+)
+
+// ${ENV:defaultVal} pattern is supported to override the default value by the
env config.
+// And the value would be converted to the different types, such as int,
float64 ana string.
+// If you want to keep it as the string value, please decorate it with `"`.
+const RegularExpression = "${_|[A-Z]+:.*}"
+
+func overrideConfigByEnv(v *viper.Viper) error {
+ regex, err := regexp.Compile(RegularExpression)
+ if err != nil {
+ return err
+ }
+ overrideCfg := overrideMapStringInterface(v.AllSettings(), regex)
+ for key, val := range overrideCfg {
+ v.Set(key, val)
+ }
+ return nil
+}
+
+func overrideMapStringInterface(cfg map[string]interface{}, regex
*regexp.Regexp) map[string]interface{} {
+ res := make(map[string]interface{})
+ for key, val := range cfg {
+ switch val := val.(type) {
+ case string:
+ res[key] = overrideString(val, regex)
+ case []interface{}:
+ res[key] = overrideSlice(val, regex)
+ case map[string]interface{}:
+ res[key] = overrideMapStringInterface(val, regex)
+ case map[interface{}]interface{}:
+ res[key] = overrideMapInterfaceInterface(val, regex)
+ default:
+ res[key] = val
+ }
+ }
+ return res
+}
+
+func overrideSlice(m []interface{}, regex *regexp.Regexp) []interface{} {
+ res := make([]interface{}, 0)
+ for _, val := range m {
+ switch val := val.(type) {
+ case map[string]interface{}:
+ res = append(res, overrideMapStringInterface(val,
regex))
+ case map[interface{}]interface{}:
+ res = append(res, overrideMapInterfaceInterface(val,
regex))
+ }
+ }
+ return res
+}
+
+func overrideMapInterfaceInterface(m map[interface{}]interface{}, regex
*regexp.Regexp) interface{} {
+ cfg := make(map[string]interface{})
+ for key, val := range m {
+ cfg[key.(string)] = val
+ }
+ return overrideMapStringInterface(cfg, regex)
+}
+
+func overrideString(s string, regex *regexp.Regexp) interface{} {
+ if regex.MatchString(s) {
+ index := strings.Index(s, ":")
+ envName := s[2:index]
+ defaultVal := s[index+1 : len(s)-1]
+ envVal := os.Getenv(envName)
+ if envVal != "" {
+ defaultVal = envVal
+ }
+ return parseVal(defaultVal)
+ }
+ return s
+}
+
+func parseVal(val string) interface{} {
+ if intVal, err := strconv.Atoi(val); err == nil {
+ return intVal
+ } else if floatVal, err := strconv.ParseFloat(val, 64); err == nil {
+ return floatVal
+ } else if strings.EqualFold(val, "true") {
+ return true
+ } else if strings.EqualFold(val, "false") {
+ return false
+ } else if strings.HasPrefix(val, "\"") && strings.HasSuffix(val, "\"") {
+ return val[1 : len(val)-1]
+ } else {
+ return val
+ }
+}
diff --git a/internal/satellite/config/override_by_env_test.go
b/internal/satellite/config/override_by_env_test.go
new file mode 100644
index 0000000..972579c
--- /dev/null
+++ b/internal/satellite/config/override_by_env_test.go
@@ -0,0 +1,194 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package config
+
+import (
+ "os"
+ "reflect"
+ "regexp"
+ "strings"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+)
+
+func Test_overrideString(t *testing.T) {
+ type args struct {
+ expression string
+ env string
+ }
+ tests := []struct {
+ name string
+ args args
+ want interface{}
+ }{
+ {
+ name: "override_string",
+ args: args{
+ expression: "${TEST_OVERRIDE_STRING:test_str}",
+ env:
"TEST_OVERRIDE_STRING=test_override_string",
+ },
+ want: "test_override_string",
+ },
+ {
+ name: "no_override_string",
+ args: args{
+ expression:
"${TEST_NO_OVERRIDE_STRING:test_str}",
+ },
+ want: "test_str",
+ },
+ {
+ name: "no_override_false",
+ args: args{
+ expression: "${TEST_NO_OVERRIDE_FALSE:false}",
+ },
+ want: false,
+ },
+ {
+ name: "no_override_true",
+ args: args{
+ expression: "${TEST_NO_OVERRIDE_TRUE:true}",
+ },
+ want: true,
+ },
+ {
+ name: "override_boolean",
+ args: args{
+ expression: "${TEST_OVERRIDE_BOOLEAN:true}",
+ env: "TEST_OVERRIDE_BOOLEAN=false",
+ },
+ want: false,
+ },
+ {
+ name: "no_override_int",
+ args: args{
+ expression: "${TEST_OVERRIDE_INT:10}",
+ },
+ want: 10,
+ },
+ {
+ name: "override_int",
+ args: args{
+ expression: "${TEST_OVERRIDE_INT:10}",
+ env: "TEST_OVERRIDE_INT=15",
+ },
+ want: 15,
+ },
+ {
+ name: "override_float",
+ args: args{
+ expression: "${TEST_OVERRIDE_FLOAT:10.5}",
+ env: "TEST_OVERRIDE_FLOAT=15.7",
+ },
+ want: 15.7,
+ },
+ {
+ name: "no_override_force_string",
+ args: args{
+ expression:
"${TEST_NO_OVERRIDE_FORCE_STRING:\"10.5\"}",
+ },
+ want: "10.5",
+ },
+ }
+ for _, tt := range tests {
+ if tt.args.env != "" {
+ envArr := strings.Split(tt.args.env, "=")
+ if err := os.Setenv(envArr[0], envArr[1]); err != nil {
+ t.Fatalf("cannot set the env %s config: %v",
tt.args.env, err)
+ }
+ }
+ regex, err := regexp.Compile(RegularExpression)
+ if err != nil {
+ t.Fatalf("cannot generate the regular expression: %s",
RegularExpression)
+ }
+ t.Run(tt.name, func(t *testing.T) {
+ if got := overrideString(tt.args.expression, regex);
!reflect.DeepEqual(got, tt.want) {
+ t.Errorf("overrideString() = %v, want %v", got,
tt.want)
+ }
+ })
+ }
+}
+
+func Test_overrideMapStringInterface(t *testing.T) {
+ type args struct {
+ cfg map[string]interface{}
+ env map[string]string
+ }
+ tests := []struct {
+ name string
+ args args
+ want map[string]interface{}
+ }{
+ {
+ name: "test-overrideByEnv",
+ args: args{
+ cfg: map[string]interface{}{
+ "stringKey":
"${OVERRIDE_STRING_KEY:stringKey}",
+ "intKey": "${OVERRIDE_INT_KEY:10}",
+ "boolKey":
"${OVERRIDE_BOOL_KEY:false}",
+ "mapKey": map[string]interface{}{
+ "mapStringKey":
"${OVERRIDE_STRING_KEY:stringKey}",
+ "mapIntKey":
"${OVERRIDE_INT_KEY:10}",
+ "mapBoolKey":
"${OVERRIDE_BOOL_KEY:false}",
+ "mapInterfaceKey":
map[interface{}]interface{}{
+
"mapinterfaceStringKey": "${OVERRIDE_STRING_KEY:stringKey}",
+ "mapinterfaceIntKey":
"${OVERRIDE_INT_KEY:10}",
+ "mapinterfaceBoolKey":
"${OVERRIDE_BOOL_KEY:false}",
+ },
+ },
+ },
+ env: map[string]string{
+ "OVERRIDE_STRING_KEY": "env-string",
+ "OVERRIDE_INT_KEY": "100",
+ "OVERRIDE_BOOL_KEY": "true",
+ },
+ },
+ want: map[string]interface{}{
+ "stringKey": "env-string",
+ "intKey": 100,
+ "boolKey": true,
+ "mapKey": map[string]interface{}{
+ "mapStringKey": "env-string",
+ "mapIntKey": 100,
+ "mapBoolKey": true,
+ "mapInterfaceKey":
map[string]interface{}{
+ "mapinterfaceStringKey":
"env-string",
+ "mapinterfaceIntKey": 100,
+ "mapinterfaceBoolKey": true,
+ },
+ },
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ for k, v := range tt.args.env {
+ err := os.Setenv(k, v)
+ if err != nil {
+ t.Fatalf("cannot set the env config{%s=%s}:
%v", k, v, err)
+ }
+ }
+ t.Run(tt.name, func(t *testing.T) {
+ regex, _ := regexp.Compile(RegularExpression)
+ got := overrideMapStringInterface(tt.args.cfg, regex)
+ if !cmp.Equal(got, tt.want) {
+ t.Errorf("overrideConfigByEnv() got = %v, want
= %v", got, tt.want)
+ }
+ })
+ }
+}
diff --git a/internal/satellite/config/satellite_config.go
b/internal/satellite/config/satellite_config.go
index 3033692..2828285 100644
--- a/internal/satellite/config/satellite_config.go
+++ b/internal/satellite/config/satellite_config.go
@@ -37,15 +37,15 @@ type SatelliteConfig struct {
// SharingConfig contains some plugins,which could be shared by every
namespace. That is useful to reduce resources cost.
type SharingConfig struct {
- SharingCommonConfig config.CommonFields `mapstructure:"common_config"`
Clients []plugin.Config `mapstructure:"clients"`
Servers []plugin.Config `mapstructure:"servers"`
+ SharingCommonConfig config.CommonFields `mapstructure:"common_config"`
}
// PipeConfig initializes the different module in different namespace.
type PipeConfig struct {
- PipeCommonConfig config.CommonFields
`mapstructure:"common_config"`
Gatherer *gatherer.GathererConfig `mapstructure:"gatherer"`
+ PipeCommonConfig config.CommonFields
`mapstructure:"common_config"`
Processor *processor.ProcessorConfig `mapstructure:"processor"`
Sender *sender.SenderConfig `mapstructure:"sender"`
}
