(dubbo-kubernetes) branch master updated: [horus] Fix action repeat (#374)

[zhongxjian]

This is an automated email from the ASF dual-hosted git repository.

zhongxjian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/dubbo-kubernetes.git


The following commit(s) were added to refs/heads/master by this push:
     new 49facbd4 [horus] Fix action repeat (#374)
49facbd4 is described below

commit 49facbd49a291d0bc583ab8eba22380ce99733ed
Author: mfordjody <11638...@qq.com>
AuthorDate: Tue Sep 24 14:23:48 2024 +0800

    [horus] Fix action repeat (#374)
---
 app/horus/core/horuser/action.go   |  4 ++++
 app/horus/core/horuser/downtime.go |  4 ++--
 app/horus/core/horuser/recovery.go |  5 ++++-
 app/horus/core/horuser/restart.go  | 10 ++++++++--
 deploy/horus/horus.yaml            |  2 +-
 5 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/app/horus/core/horuser/action.go b/app/horus/core/horuser/action.go
index 85ea066e..53a7ba89 100644
--- a/app/horus/core/horuser/action.go
+++ b/app/horus/core/horuser/action.go
@@ -75,6 +75,10 @@ func (h *Horuser) UnCordon(nodeName, clusterName string) 
(err error) {
        }
 
        node.Spec.Unschedulable = false
+       if !node.Spec.Unschedulable {
+               klog.Infof("Node %v is already uncordoned.", nodeName)
+               return nil
+       }
 
        ctxSecond, cancelSecond := h.GetK8sContext()
        defer cancelSecond()
diff --git a/app/horus/core/horuser/downtime.go 
b/app/horus/core/horuser/downtime.go
index e3f730ef..126c3a9d 100644
--- a/app/horus/core/horuser/downtime.go
+++ b/app/horus/core/horuser/downtime.go
@@ -107,7 +107,7 @@ func (h *Horuser) DownTimeNodes(clusterName, addr string) {
                WithDownNodeIPs[node] = str
        }
 
-       WithDownNodeIPsMsg := fmt.Sprintf("\n【%s】\n【集群：%v】\n【宕机：%v", 
h.cc.NodeDownTime.DingTalk.Title, clusterName, len(WithDownNodeIPs))
+       WithDownNodeIPsMsg := fmt.Sprintf("\n【%s】\n【集群：%v】\n【宕机：%v】", 
h.cc.NodeDownTime.DingTalk.Title, clusterName, len(WithDownNodeIPs))
        newfound := 0
        for nodeName, nodeIP := range WithDownNodeIPs {
                today := time.Now().Format("2006-01-02")
@@ -129,7 +129,7 @@ func (h *Horuser) DownTimeNodes(clusterName, addr string) {
                }
                newfound++
                if newfound > 0 {
-                       klog.Infof("NodeDownTimeCheckOnCluster get 
toNodeNameips \n【结果:%v】\n 【集群:%v】\n 【总数:%v】\n 【细节:%v】\n", WithDownNodeIPsMsg, 
clusterName, len(nodeIP), nodeName)
+                       klog.Infof("NodeDownTimeCheckOnCluster get 
toNodeNameips \n【集群:%v】\n 【总数:%v】\n 【细节:%v】\n", clusterName, len(nodeIP), 
nodeName)
                        alert.DingTalkSend(h.cc.NodeDownTime.DingTalk, 
WithDownNodeIPsMsg)
                }
                WithDownNodeIPsMsg += fmt.Sprintf("node:%v ip:%v", nodeName, 
nodeIP)
diff --git a/app/horus/core/horuser/recovery.go 
b/app/horus/core/horuser/recovery.go
index ed95bb04..4c4acd65 100644
--- a/app/horus/core/horuser/recovery.go
+++ b/app/horus/core/horuser/recovery.go
@@ -75,7 +75,10 @@ func (h *Horuser) recoveryNodes(n db.NodeDataInfo) {
        }
        klog.Infof("recoveryNodes check success.")
        err = h.UnCordon(n.NodeName, n.ClusterName)
-
+       if err == nil {
+               klog.Infof("Node %v is already uncordoned.", n.NodeName)
+               return
+       }
        res := "Success"
        if err != nil {
                res = fmt.Sprintf("failed:%v", err)
diff --git a/app/horus/core/horuser/restart.go 
b/app/horus/core/horuser/restart.go
index 2efe31fc..161a7ff9 100644
--- a/app/horus/core/horuser/restart.go
+++ b/app/horus/core/horuser/restart.go
@@ -51,10 +51,16 @@ func (h *Horuser) RestartOrRepair(ctx context.Context) {
 }
 
 func (h *Horuser) TryRestart(node db.NodeDataInfo) {
-       msg := fmt.Sprintf("\n【节点强制重启】\n 节点:%v\n 日期:%v\n 集群:%v\n", 
node.NodeName, node.FirstDate, node.ClusterName)
        err := h.UnCordon(node.NodeName, node.ClusterName)
+       if err == nil {
+               klog.Infof("Node %v is already uncordoned.", node.NodeName)
+               return
+       }
+       msg := fmt.Sprintf("\n【节点尝试重启】\n 节点:%v\n 日期:%v\n 集群:%v\n", 
node.NodeName, node.FirstDate, node.ClusterName)
        if err != nil {
-               msg = fmt.Sprintf("\n【节点强制重启失败：%v】\n", err)
+               msg = fmt.Sprintf("\n【节点重启失败：%v】\n", err)
+       } else {
+               msg = fmt.Sprintf("\n【节点重启成功】\n 节点:%v\n 日期:%v\n 集群:%v\n", 
node.NodeName, node.FirstDate, node.ClusterName)
        }
        alert.DingTalkSend(h.cc.NodeDownTime.DingTalk, msg)
        pass, err := node.RestartMarker()
diff --git a/deploy/horus/horus.yaml b/deploy/horus/horus.yaml
index cf0af232..53d6cc37 100644
--- a/deploy/horus/horus.yaml
+++ b/deploy/horus/horus.yaml
@@ -92,7 +92,7 @@ nodeDownTime:
 
 podAbnormal:
   enabled: false
-  checkIntervalSecond:5
+  checkIntervalSecond: 5
   checkDoubleSecond: 10
   labelSelector: "app.kubernetes.io/name=horus"
   fieldSelector: "status.phase != Running"