This is an automated email from the ASF dual-hosted git repository.
zhongxjian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/dubbo-kubernetes.git
The following commit(s) were added to refs/heads/master by this push:
new 68f2eb78 [horus] Fix duplicate alarms (#353)
68f2eb78 is described below
commit 68f2eb780e622321fe85245dc61104fbd9d193aa
Author: mfordjody <[email protected]>
AuthorDate: Wed Sep 18 20:27:47 2024 +0800
[horus] Fix duplicate alarms (#353)
---
app/horus/basic/db/db.go | 10 +++++-----
app/horus/core/horuser/action.go | 8 ++++++--
app/horus/core/horuser/modular.go | 37 +++++++++++++++++--------------------
app/horus/core/horuser/recovery.go | 2 +-
deploy/horus/horus.yaml | 4 ++--
5 files changed, 31 insertions(+), 30 deletions(-)
diff --git a/app/horus/basic/db/db.go b/app/horus/basic/db/db.go
index 325d136d..8040d52e 100644
--- a/app/horus/basic/db/db.go
+++ b/app/horus/basic/db/db.go
@@ -119,16 +119,16 @@ func (n *NodeDataInfo) AddOrGet() (int64, error) {
return row, err
}
-func GetRecoveryNodeDataInfoDate(day int) ([]*NodeDataInfo, error) {
- var ndi []*NodeDataInfo
+func GetRecoveryNodeDataInfoDate(day int) ([]NodeDataInfo, error) {
+ var ndi []NodeDataInfo
session := db.Where(fmt.Sprintf("recovery_mark = 0 AND first_date >
DATE_SUB(CURDATE(), INTERVAL %d DAY)", day))
err := session.Find(&ndi)
return ndi, err
}
-func GetDailyLimitNodeDataInfoDate(day, module, cluster string)
([]*NodeDataInfo, error) {
- var ndi []*NodeDataInfo
- session := db.Where(fmt.Sprintf("DATE(first_date) = '%s' AND
module_name = '%s' AND cluster_name = '%s'", day, module, cluster))
+func GetDailyLimitNodeDataInfoDate(day, module, cluster string)
([]NodeDataInfo, error) {
+ var ndi []NodeDataInfo
+ session := db.Where("DATE(first_date) = ? AND module_name = ? AND
cluster_name = ?", day, module, cluster)
err := session.Find(&ndi)
return ndi, err
}
diff --git a/app/horus/core/horuser/action.go b/app/horus/core/horuser/action.go
index f438258e..f994119a 100644
--- a/app/horus/core/horuser/action.go
+++ b/app/horus/core/horuser/action.go
@@ -21,7 +21,7 @@ import (
"k8s.io/klog/v2"
)
-func (h *Horuser) Cordon(nodeName, clusterName string) (err error) {
+func (h *Horuser) Cordon(nodeName, clusterName, moduleName string) (err error)
{
kubeClient := h.kubeClientMap[clusterName]
if kubeClient == nil {
klog.Errorf("node Cordon kubeClient by clusterName empty.")
@@ -36,9 +36,13 @@ func (h *Horuser) Cordon(nodeName, clusterName string) (err
error) {
klog.Errorf("node Cordon get err nodeName:%v clusterName:%v",
nodeName, clusterName)
return err
}
+ annotations := node.Annotations
+ if annotations == nil {
+ annotations = map[string]string{}
+ }
+ annotations["dubbo.apache.org/disable-by"] = "horus"
node.Spec.Unschedulable = true
-
ctxSecond, cancelSecond := h.GetK8sContext()
defer cancelSecond()
node, err = kubeClient.CoreV1().Nodes().Update(ctxSecond, node,
v1.UpdateOptions{})
diff --git a/app/horus/core/horuser/modular.go
b/app/horus/core/horuser/modular.go
index 45943829..df51676e 100644
--- a/app/horus/core/horuser/modular.go
+++ b/app/horus/core/horuser/modular.go
@@ -81,22 +81,13 @@ func (h *Horuser) CustomizeModularNodes(clusterName,
moduleName, nodeName, ip st
today := time.Now().Format("2006-01-02")
recoveryQL := fmt.Sprintf(h.cc.CustomModular.RecoveryQL[moduleName],
nodeName)
-
data, err := db.GetDailyLimitNodeDataInfoDate(today, moduleName,
clusterName)
if err != nil {
klog.Errorf("CustomizeModularNodes
GetDailyLimitNodeDataInfoDate err:%v", err)
return
}
- klog.Infof("%v", data)
-
- dailyLimit := h.cc.CustomModular.CordonDailyLimit[moduleName]
- if len(data) > dailyLimit {
- msg := fmt.Sprintf("【日期:%v】 【集群:%v\n】 【今日 Cordon 节点数: %v】\n
【已达到今日上限: %v】\n 【节点:%v】",
- data, clusterName, moduleName, dailyLimit, nodeName)
- alert.DingTalkSend(h.cc.CustomModular.DingTalk, msg)
- return
- }
-
+ err = h.Cordon(nodeName, clusterName, moduleName)
+
write := db.NodeDataInfo{
NodeName: nodeName,
NodeIP: ip,
@@ -106,29 +97,35 @@ func (h *Horuser) CustomizeModularNodes(clusterName,
moduleName, nodeName, ip st
FirstDate: today,
RecoveryQL: recoveryQL,
}
+
pass, _ := write.Check()
if pass {
klog.Infof("CustomizeModularNodes already existing
clusterName:%v nodeName:%v moduleName:%v", clusterName, nodeName, moduleName)
return
}
- err = h.Cordon(nodeName, clusterName)
+
+ _, err = write.AddOrGet()
+ if err != nil {
+ klog.Errorf("CustomizeModularNodes AddOrGet err:%v", err)
+ klog.Infof("moduleName:%v nodeName:%v", moduleName,
write.NodeName)
+ }
+
res := "Success"
if err != nil {
res = fmt.Sprintf("failed:%v", err)
klog.Errorf("Cordon failed:%v", res)
}
-
msg := fmt.Sprintf("\n【集群:%v】\n【发现 %s 异常已禁止调度】\n【已禁止调度节点:%v】\n 【处理结果:
%v】\n 【今日操作次数:%v】\n",
clusterName, moduleName, nodeName, res, len(data)+1)
-
- klog.Infof("Attempting to send DingTalk message: %s", msg)
alert.DingTalkSend(h.cc.CustomModular.DingTalk, msg)
- klog.Infof("DingTalk message sent")
- _, err = write.AddOrGet()
- if err != nil {
- klog.Errorf("CustomizeModularNodes AddOrGet err:%v", err)
- klog.Infof("moduleName:%v nodeName:%v", moduleName,
write.NodeName)
+ dailyLimit := h.cc.CustomModular.CordonDailyLimit[moduleName]
+ if len(data) > dailyLimit {
+ msg := fmt.Sprintf("【日期:%v】 【集群:%v\n】 【今日 Cordon 节点数: %v】\n
【已达到今日上限: %v】\n 【节点:%v】",
+ data, clusterName, len(data), dailyLimit, nodeName)
+ alert.DingTalkSend(h.cc.CustomModular.DingTalk, msg)
+ return
}
+
klog.Infof("CustomizeModularNodes AddOrGet success moduleName:%v
nodeName:%v", moduleName, write.NodeName)
}
diff --git a/app/horus/core/horuser/recovery.go
b/app/horus/core/horuser/recovery.go
index dd59c5c4..e97617e9 100644
--- a/app/horus/core/horuser/recovery.go
+++ b/app/horus/core/horuser/recovery.go
@@ -53,7 +53,7 @@ func (h *Horuser) recoveryCheck(ctx context.Context) {
wp.StopWait()
}
-func (h *Horuser) recoveryNodes(n *db.NodeDataInfo) {
+func (h *Horuser) recoveryNodes(n db.NodeDataInfo) {
addr := h.cc.PromMultiple[n.ClusterName]
if addr == "" {
klog.Errorf("recoveryNodes PromMultiple get addr empty.")
diff --git a/deploy/horus/horus.yaml b/deploy/horus/horus.yaml
index 768184fd..e68be09f 100644
--- a/deploy/horus/horus.yaml
+++ b/deploy/horus/horus.yaml
@@ -50,9 +50,9 @@ nodeRecovery:
- 15000000
customModular:
- enabled: true
+ enabled: false
cordonDailyLimit:
- filesystem_readonly: 1
+ filesystem_readonly: 3
checkQL:
filesystem_readonly: |-
node_filesystem_readonly{mountpoint="/"} != 1
