starsz commented on code in PR #2650:
URL: https://github.com/apache/apisix-dashboard/pull/2650#discussion_r1007508802
##########
api/cmd/root.go:
##########
@@ -81,3 +89,59 @@ func manageAPI() error {
}
return nil
}
+
+func etcdConnectionChecker() context.CancelFunc {
+ ctx, cancel := context.WithCancel(context.TODO())
+ unavailableTimes := 0
+
+ go func() {
+ etcdClient := storage.GenEtcdStorage().GetClient()
+ for {
+ select {
+ case <-time.Tick(10 * time.Second):
+ sCtx, sCancel :=
context.WithTimeout(context.TODO(), 5*time.Second)
Review Comment:
```suggestion
sCtx, sCancel := context.WithTimeout(ctx,
5*time.Second)
```
##########
api/cmd/root.go:
##########
@@ -81,3 +89,59 @@ func manageAPI() error {
}
return nil
}
+
+func etcdConnectionChecker() context.CancelFunc {
+ ctx, cancel := context.WithCancel(context.TODO())
+ unavailableTimes := 0
+
+ go func() {
+ etcdClient := storage.GenEtcdStorage().GetClient()
+ for {
+ select {
+ case <-time.Tick(10 * time.Second):
+ sCtx, sCancel :=
context.WithTimeout(context.TODO(), 5*time.Second)
+ err := etcdClient.Sync(sCtx)
+ sCancel()
+ if err != nil {
+ unavailableTimes++
+ log.Errorf("etcd connection loss
detected, times: %d", unavailableTimes)
+ continue
+ }
+
+ // After multiple failures, the connection is
restored
+ if unavailableTimes >= 1 {
+ log.Warnf("etcd connection recovered,
but after several connection losses, reinitializing stores, times: %d",
unavailableTimes)
+ unavailableTimes = 0
+
+ // When this happens, force a full
re-initialization of the store
+ store.RangeStore(func(key store.HubKey,
store *store.GenericStore) bool {
+ log.Warnf("etcd store
reinitializing: resource: %s", key)
+ if err := store.Init(); err !=
nil {
+ log.Errorf("etcd store
reinitialize failed: resource: %s", key)
Review Comment:
```suggestion
log.Errorf("etcd store
reinitialize failed: resource: %s, error: %s", key, err)
```
##########
api/test/shell/cli_test.sh:
##########
@@ -478,7 +477,39 @@ stop_dashboard() {
# check response header with custom header
run curl -i http://127.0.0.1:9000
-[ $(echo "$output" | grep -c "X-Frame-Options: test") -eq '1' ]
+ [ $(echo "$output" | grep -c "X-Frame-Options: test") -eq '1' ]
+
+ stop_dashboard 6
+}
+
+#16
+@test "Check etcd auto re-watch" {
+ recover_conf
+ clean_logfile
+
+ # change log level
+ if [[ $KERNEL = "Darwin" ]]; then
+ sed -i "" 's/level: warn/level: info/' ${CONF_FILE}
+ else
+ sed -i 's/level: warn/level: info/' ${CONF_FILE}
+ fi
+
+ start_dashboard 15
+
+ [ "$(grep -c "etcd connection is fine" ${LOG_FILE})" -ge '1' ]
+
+ run docker stop etcd
+
+ sleep 30
+
+ [ "$(grep -c "etcd connection loss detected" ${LOG_FILE})" -ge '1' ]
+
+ run docker start etcd
+
+ sleep 20
+
+ [ "$(grep -c "etcd connection recovered" ${LOG_FILE})" -ge '1' ]
+ [ "$(grep -c "etcd store reinitializing" ${LOG_FILE})" -ge '1' ]
Review Comment:
Better to add e2e test to ensure the data is not loss.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
