This is an automated email from the ASF dual-hosted git repository.
zhongxjian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/dubbo-kubernetes.git
The following commit(s) were added to refs/heads/master by this push:
new facd4443 [horus] Cordon logic repair (#371)
facd4443 is described below
commit facd44431d6579335e4670ade2a6282551c2eaf2
Author: mfordjody <[email protected]>
AuthorDate: Mon Sep 23 18:02:07 2024 +0800
[horus] Cordon logic repair (#371)
fix typo
---
app/horus/core/horuser/action.go | 10 +++++++---
app/horus/core/horuser/downtime.go | 5 +++--
app/horus/core/horuser/modular.go | 3 +++
app/horus/core/horuser/recovery.go | 2 +-
app/horus/core/horuser/restart.go | 8 ++++++--
5 files changed, 20 insertions(+), 8 deletions(-)
diff --git a/app/horus/core/horuser/action.go b/app/horus/core/horuser/action.go
index f994119a..99282c40 100644
--- a/app/horus/core/horuser/action.go
+++ b/app/horus/core/horuser/action.go
@@ -43,15 +43,19 @@ func (h *Horuser) Cordon(nodeName, clusterName, moduleName
string) (err error) {
annotations["dubbo.apache.org/disable-by"] = "horus"
node.Spec.Unschedulable = true
+ if node.Spec.Unschedulable {
+ klog.Infof("Node %v is already cordoned.", nodeName)
+ return nil
+ }
ctxSecond, cancelSecond := h.GetK8sContext()
defer cancelSecond()
node, err = kubeClient.CoreV1().Nodes().Update(ctxSecond, node,
v1.UpdateOptions{})
if err != nil {
klog.Errorf("node Cordon update err nodeName:%v
clusterName:%v", nodeName, clusterName)
- return err
+ } else {
+ klog.Infof("node Cordon success nodeName:%v clusterName:%v",
nodeName, clusterName)
}
- klog.Infof("node Cordon success nodeName:%v clusterName:%v", nodeName,
clusterName)
- return nil
+ return err
}
func (h *Horuser) UnCordon(nodeName, clusterName string) (err error) {
diff --git a/app/horus/core/horuser/downtime.go
b/app/horus/core/horuser/downtime.go
index 751eaf8b..e3f730ef 100644
--- a/app/horus/core/horuser/downtime.go
+++ b/app/horus/core/horuser/downtime.go
@@ -107,7 +107,7 @@ func (h *Horuser) DownTimeNodes(clusterName, addr string) {
WithDownNodeIPs[node] = str
}
- WithDownNodeIPsMsg := fmt.Sprintf("\n【%s】\n【集群：%v】\n【宕机：%v】\n",
h.cc.NodeDownTime.DingTalk.Title, clusterName, len(WithDownNodeIPs))
+ WithDownNodeIPsMsg := fmt.Sprintf("\n【%s】\n【集群：%v】\n【宕机：%v",
h.cc.NodeDownTime.DingTalk.Title, clusterName, len(WithDownNodeIPs))
newfound := 0
for nodeName, nodeIP := range WithDownNodeIPs {
today := time.Now().Format("2006-01-02")
@@ -115,6 +115,7 @@ func (h *Horuser) DownTimeNodes(clusterName, addr string) {
if err != nil {
klog.Errorf("Cordon node err:%v", err)
klog.Infof("clusterName:%v nodeName:%v", clusterName,
nodeName)
+ return
}
write := db.NodeDataInfo{
NodeName: nodeName,
@@ -128,7 +129,7 @@ func (h *Horuser) DownTimeNodes(clusterName, addr string) {
}
newfound++
if newfound > 0 {
- klog.Infof("NodeDownTimeCheckOnCluster get
toNodeNameips result msg:%v clusterName:%v count:%v detail:%v",
WithDownNodeIPsMsg, clusterName, len(nodeIP), nodeName)
+ klog.Infof("NodeDownTimeCheckOnCluster get
toNodeNameips \n【结果:%v】\n 【集群:%v】\n 【总数:%v】\n 【细节:%v】\n", WithDownNodeIPsMsg,
clusterName, len(nodeIP), nodeName)
alert.DingTalkSend(h.cc.NodeDownTime.DingTalk,
WithDownNodeIPsMsg)
}
WithDownNodeIPsMsg += fmt.Sprintf("node:%v ip:%v", nodeName,
nodeIP)
diff --git a/app/horus/core/horuser/modular.go
b/app/horus/core/horuser/modular.go
index c6501346..bcb6b322 100644
--- a/app/horus/core/horuser/modular.go
+++ b/app/horus/core/horuser/modular.go
@@ -87,6 +87,9 @@ func (h *Horuser) CustomizeModularNodes(clusterName,
moduleName, nodeName, ip st
return
}
err = h.Cordon(nodeName, clusterName, moduleName)
+ if err != nil {
+ return
+ }
write := db.NodeDataInfo{
NodeName: nodeName,
diff --git a/app/horus/core/horuser/recovery.go
b/app/horus/core/horuser/recovery.go
index d9137588..ed95bb04 100644
--- a/app/horus/core/horuser/recovery.go
+++ b/app/horus/core/horuser/recovery.go
@@ -74,8 +74,8 @@ func (h *Horuser) recoveryNodes(n db.NodeDataInfo) {
return
}
klog.Infof("recoveryNodes check success.")
-
err = h.UnCordon(n.NodeName, n.ClusterName)
+
res := "Success"
if err != nil {
res = fmt.Sprintf("failed:%v", err)
diff --git a/app/horus/core/horuser/restart.go
b/app/horus/core/horuser/restart.go
index dc85fd92..2efe31fc 100644
--- a/app/horus/core/horuser/restart.go
+++ b/app/horus/core/horuser/restart.go
@@ -41,7 +41,7 @@ func (h *Horuser) RestartOrRepair(ctx context.Context) {
klog.Warningf("Needs to be rebooted or fixed to zero.")
}
klog.Infof("GetRestartNodeDataInfoDate count:%v", len(nodes))
- wp := workerpool.New(30)
+ wp := workerpool.New(10)
for _, n := range nodes {
n := n
wp.Submit(func() {
@@ -51,7 +51,11 @@ func (h *Horuser) RestartOrRepair(ctx context.Context) {
}
func (h *Horuser) TryRestart(node db.NodeDataInfo) {
- msg := fmt.Sprintf("node restart or force restart node:%v date:%v
cluster:%v", node.NodeName, node.FirstDate, node.ClusterName)
+ msg := fmt.Sprintf("\n【节点强制重启】\n 节点:%v\n 日期:%v\n 集群:%v\n",
node.NodeName, node.FirstDate, node.ClusterName)
+ err := h.UnCordon(node.NodeName, node.ClusterName)
+ if err != nil {
+ msg = fmt.Sprintf("\n【节点强制重启失败：%v】\n", err)
+ }
alert.DingTalkSend(h.cc.NodeDownTime.DingTalk, msg)
pass, err := node.RestartMarker()
klog.Infof("RestartMarker result pass:%v err:%v", pass, err)
