This is an automated email from the ASF dual-hosted git repository.
zhongxjian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/dubbo-kubernetes.git
The following commit(s) were added to refs/heads/master by this push:
new 06b5e005 [horus] Organize downtime recovery logic (#445)
06b5e005 is described below
commit 06b5e005785f5246a45e88c4f3610afb56da695b
Author: mfordjody <[email protected]>
AuthorDate: Thu Oct 10 09:37:24 2024 +0800
[horus] Organize downtime recovery logic (#445)
---
app/horus/base/config/config.go | 1 +
app/horus/base/db/db.go | 16 +++++++++++--
app/horus/core/horuser/node_downtime.go | 40 ++++++++++++++++++++++++++++----
app/horus/core/horuser/node_recovery.go | 41 +++++++++++++++++++++++++++++++++
manifests/horus/horus.yaml | 4 ++--
5 files changed, 93 insertions(+), 9 deletions(-)
diff --git a/app/horus/base/config/config.go b/app/horus/base/config/config.go
index 1001e630..fd553966 100644
--- a/app/horus/base/config/config.go
+++ b/app/horus/base/config/config.go
@@ -77,6 +77,7 @@ type DowntimeConfiguration struct {
AllSystemUser string `yaml:"allSystemUser"`
AllSystemPassword string `yaml:"allSystemPassword"`
DingTalk *DingTalkConfiguration `yaml:"dingTalk"`
+ Slack *SlackConfiguration `yaml:"slack"`
}
type CleanerConfiguration struct {
diff --git a/app/horus/base/db/db.go b/app/horus/base/db/db.go
index 75a43ddd..b1afe246 100644
--- a/app/horus/base/db/db.go
+++ b/app/horus/base/db/db.go
@@ -40,8 +40,8 @@ type NodeDataInfo struct {
UpdateTime time.Time `json:"update_time" xorm:"update_time
updated"`
RecoveryMark int64 `json:"recovery_mark"
xorm:"recovery_mark"`
RecoveryQL string `json:"recovery_ql" xorm:"recovery_ql"`
- DownTimeRecoveryQL []string `json:"downtime_recovery_ql
xorm:downtime_recovery_ql"`
- DownTimeRecoveryMark int64 `json:"downtime_recovery_mark
xorm:downtime_recovery_mark"`
+ DownTimeRecoveryMark int64 `json:"downtime_recovery_mark"
xorm:"downtime_recovery_mark"`
+ DownTimeRecoveryQL string `json:"downtime_recovery_ql"
xorm:"downtime_recovery_ql"`
}
type PodDataInfo struct {
@@ -122,6 +122,11 @@ func (n *NodeDataInfo) RecoveryMarker() (bool, error) {
return n.Update()
}
+func (n *NodeDataInfo) DownTimeRecoveryMarker() (bool, error) {
+ n.DownTimeRecoveryMark = 1
+ return n.Update()
+}
+
func (n *NodeDataInfo) RestartMarker() (bool, error) {
n.Restart = 1
return n.Update()
@@ -149,6 +154,13 @@ func GetRecoveryNodeDataInfoDate(day int) ([]NodeDataInfo,
error) {
return ndi, err
}
+func GetDownTimeRecoveryNodeDataInfoDate(day int) ([]NodeDataInfo, error) {
+ var ndi []NodeDataInfo
+ session := db.Where(fmt.Sprintf("downtime_recovery_mark = 0 AND
first_date > DATE_SUB(CURDATE(), INTERVAL %d DAY)", day))
+ err := session.Find(&ndi)
+ return ndi, err
+}
+
func GetRestartNodeDataInfoDate() ([]NodeDataInfo, error) {
var ndi []NodeDataInfo
session := db.Where("restart = 0 and repair = 0 and module_name = ?",
"nodeDown")
diff --git a/app/horus/core/horuser/node_downtime.go
b/app/horus/core/horuser/node_downtime.go
index 0ddeacdb..c1ce6bd1 100644
--- a/app/horus/core/horuser/node_downtime.go
+++ b/app/horus/core/horuser/node_downtime.go
@@ -71,7 +71,8 @@ func (h *Horuser) DownTimeNodes(clusterName, addr string) {
klog.Infof("clusterName:%v\n", clusterName)
nodeDownTimeRes := make(map[string]int)
- cq := len(h.cc.NodeDownTime.AbnormalityQL)
+ aq := len(h.cc.NodeDownTime.AbnormalityQL)
+ rq := len(h.cc.NodeDownTime.AbnormalRecoveryQL)
for _, ql := range h.cc.NodeDownTime.AbnormalityQL {
ql := ql
res, err := h.InstantQuery(addr, ql, clusterName,
h.cc.NodeDownTime.PromQueryTimeSecond)
@@ -92,12 +93,34 @@ func (h *Horuser) DownTimeNodes(clusterName, addr string) {
nodeDownTimeRes[nodeName]++
}
}
+
+ for _, ql := range h.cc.NodeDownTime.AbnormalRecoveryQL {
+ ql := ql
+ res, err := h.InstantQuery(addr, ql, clusterName,
h.cc.NodeDownTime.PromQueryTimeSecond)
+ if err != nil {
+ klog.Errorf("downtimeNodes InstantQuery err:%v", err)
+ klog.Infof("clusterName:%v\n", clusterName)
+ continue
+ }
+
+ for _, v := range res {
+ v := v
+ nodeName := string(v.Metric["node"])
+ if nodeName == "" {
+ klog.Error("downtimeNodes InstantQuery nodeName
empty.")
+ klog.Infof("clusterName:%v\n metric:%v\n",
clusterName, v.Metric)
+ continue
+ }
+ nodeDownTimeRes[nodeName]++
+ }
+ }
+
WithDownNodeIPs := make(map[string]string)
for node, count := range nodeDownTimeRes {
- if count < cq {
- klog.Error("downtimeNodes node not reach threshold")
- klog.Infof("clusterName:%v\n nodeName:%v\n threshold:%v
count:%v", clusterName, node, cq, count)
+ if count < aq {
+ klog.Error("downtimeNodes not reach threshold")
+ klog.Infof("clusterName:%v\n nodeName:%v\n threshold:%v
count:%v", clusterName, node, aq, count)
continue
}
abnormalInfoSystemQL :=
fmt.Sprintf(h.cc.NodeDownTime.AbnormalInfoSystemQL, node)
@@ -142,12 +165,19 @@ func (h *Horuser) DownTimeNodes(clusterName, addr string)
{
return "", nil
}()
+ for node, count := range nodeDownTimeRes {
+ if count < rq {
+ klog.Error("downtimeNodes not reach recovery
threshold")
+ klog.Infof("clusterName:%v\n nodeName:%v\n
threshold:%v count:%v", clusterName, node, aq, count)
+ continue
+ }
+ }
+
write := db.NodeDataInfo{
NodeName: nodeName,
NodeIP: nodeIP,
ClusterName: clusterName,
ModuleName: NODE_DOWN,
- RecoveryQL: ,
}
exist, _ := write.Check()
if exist {
diff --git a/app/horus/core/horuser/node_recovery.go
b/app/horus/core/horuser/node_recovery.go
index cdda8932..91650618 100644
--- a/app/horus/core/horuser/node_recovery.go
+++ b/app/horus/core/horuser/node_recovery.go
@@ -93,3 +93,44 @@ func (h *Horuser) recoveryNodes(n db.NodeDataInfo) {
}
klog.Infof("RecoveryMarker result success:%v", success)
}
+
+func (h *Horuser) DownTimeRecoveryNodes(n db.NodeDataInfo) {
+ promAddr := h.cc.PromMultiple[n.ClusterName]
+ if promAddr == "" {
+ klog.Error("recoveryNodes promAddr by clusterName empty.")
+ klog.Infof("clusterName:%v nodeName:%v", n.ClusterName,
n.NodeName)
+ return
+ }
+ vecs, err := h.InstantQuery(promAddr, n.DownTimeRecoveryQL,
n.ClusterName, h.cc.NodeDownTime.PromQueryTimeSecond)
+ if err != nil {
+ klog.Errorf("recoveryNodes InstantQuery err:%v", err)
+ klog.Infof("recoveryQL:%v", n.DownTimeRecoveryQL)
+ return
+ }
+ if len(vecs) != 1 {
+ klog.Infof("Expected 1 result, but got:%d", len(vecs))
+ return
+ }
+ if err != nil {
+ klog.Errorf("recoveryNodes InstantQuery err:%v", err)
+ klog.Infof("recoveryQL:%v", n.DownTimeRecoveryQL)
+ return
+ }
+ klog.Info("recoveryNodes InstantQuery success.")
+
+ err = h.UnCordon(n.NodeName, n.ClusterName)
+ res := "Success"
+ if err != nil {
+ res = fmt.Sprintf("result failed:%v", err)
+ }
+ msg := fmt.Sprintf("\n【集群: %v】\n【宕机节点已达到恢复临界点】\n【已恢复调度节点:
%v】\n【处理结果：%v】\n【日期: %v】\n", n.ClusterName, n.NodeName, res, n.CreateTime)
+ alerter.DingTalkSend(h.cc.NodeDownTime.DingTalk, msg)
+ alerter.SlackSend(h.cc.NodeDownTime.Slack, msg)
+
+ success, err := n.DownTimeRecoveryMarker()
+ if err != nil {
+ klog.Errorf("DownTimeRecoveryMarker result failed err:%v", err)
+ return
+ }
+ klog.Infof("DownTimeRecoveryMarker result success:%v", success)
+}
diff --git a/manifests/horus/horus.yaml b/manifests/horus/horus.yaml
index 22eb575f..02ce3bc0 100644
--- a/manifests/horus/horus.yaml
+++ b/manifests/horus/horus.yaml
@@ -64,7 +64,7 @@ customModular:
title: "自定义通知"
nodeDownTime:
- enabled: false
+ enabled: true
intervalSecond: 15
promQueryTimeSecond: 60
abnormalityQL:
@@ -91,7 +91,7 @@ nodeDownTime:
title: "自定义通知"
podStagnationCleaner:
- enabled: true
+ enabled: false
intervalSecond: 15
doubleSecond: 60
fieldSelector: "status.phase!=Running"
