This is an automated email from the ASF dual-hosted git repository.
zhongxjian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/dubbo-kubernetes.git
The following commit(s) were added to refs/heads/master by this push:
new d0aa0a30 [horus] node downtime logic in effect (#370)
d0aa0a30 is described below
commit d0aa0a3012375c960df674f493692ea1619607ff
Author: mfordjody <[email protected]>
AuthorDate: Mon Sep 23 16:50:46 2024 +0800
[horus] node downtime logic in effect (#370)
fix typo
---
app/horus/basic/db/db.go | 2 +-
app/horus/core/horuser/downtime.go | 29 +++++++++++++----------------
deploy/horus/horus.yaml | 10 ++++++----
3 files changed, 20 insertions(+), 21 deletions(-)
diff --git a/app/horus/basic/db/db.go b/app/horus/basic/db/db.go
index 35cac1ee..699a8a04 100644
--- a/app/horus/basic/db/db.go
+++ b/app/horus/basic/db/db.go
@@ -128,7 +128,7 @@ func GetRecoveryNodeDataInfoDate(day int) ([]NodeDataInfo,
error) {
func GetRestartNodeDataInfoDate() ([]NodeDataInfo, error) {
var ndi []NodeDataInfo
- session := db.Where("restart = 0 and repair = 0 and module_name= ? ",
"node_down")
+ session := db.Where("restart = 0 and repair = 0 and module_name = ?",
"node_down")
err := session.Find(&ndi)
return ndi, err
}
diff --git a/app/horus/core/horuser/downtime.go
b/app/horus/core/horuser/downtime.go
index 8c16ad34..751eaf8b 100644
--- a/app/horus/core/horuser/downtime.go
+++ b/app/horus/core/horuser/downtime.go
@@ -28,7 +28,7 @@ import (
const (
NODE_DOWN = "node_down"
- NODE_DOWN_REASON = "failed"
+ NODE_DOWN_REASON = "node_down"
)
func (h *Horuser) DownTimeManager(ctx context.Context) error {
@@ -59,7 +59,7 @@ func (h *Horuser) DownTimeCheck(ctx context.Context) {
func (h *Horuser) DownTimeNodes(clusterName, addr string) {
klog.Infof("DownTimeNodes Query Start clusterName:%v", clusterName)
- nodeDownTimeRes := map[string]int{}
+ nodeDownTimeRes := make(map[string]int)
cq := len(h.cc.NodeDownTime.CheckQL)
for _, ql := range h.cc.NodeDownTime.CheckQL {
ql := ql
@@ -74,43 +74,40 @@ func (h *Horuser) DownTimeNodes(clusterName, addr string) {
v := v
nodeName := string(v.Metric["node"])
if nodeName == "" {
- klog.Errorf("downtimeNodes InstantQuery
nodeName empty.")
+ klog.Errorf("downtimeNodes Instant Query
nodeName empty.")
klog.Infof("clusterName:%v metrics:%v",
clusterName, v.Metric)
continue
}
nodeDownTimeRes[nodeName]++
}
}
- for node, count := range nodeDownTimeRes {
- if count < cq {
- klog.Errorf("downtimeNodes node not reach threshold.")
- klog.Infof("clusterName:%v node:%v threshold:%v
count:%v", clusterName, node, cq, count)
- continue
- }
- }
+ WithDownNodeIPs := make(map[string]string)
- WithDownNodeIPs := map[string]string{}
for node, count := range nodeDownTimeRes {
if count < cq {
klog.Errorf("downtimeNodes node not reach threshold")
klog.Infof("clusterName:%v nodeName:%v threshold:%v
count:%v", clusterName, node, cq, count)
continue
}
- NodeNameToIps := fmt.Sprintf(h.cc.NodeDownTime.NodeNameToIPs,
node)
- res, err := h.InstantQuery(NodeNameToIps, addr, clusterName,
h.cc.NodeDownTime.PromQueryTimeSecond)
+ nnti := fmt.Sprintf(h.cc.NodeDownTime.NodeNameToIPs, node)
+ res, err := h.InstantQuery(addr, nnti, clusterName,
h.cc.NodeDownTime.PromQueryTimeSecond)
+ if len(res) == 0 {
+ klog.Errorf("No results returned for query: %s", nnti)
+ continue
+ }
if err != nil {
klog.Errorf("downtimeNodes InstantQuery NodeName To IPs
empty err:%v", err)
- klog.Infof("clusterName:%v toNodeNameips:%v err:%v",
clusterName, NodeNameToIps, err)
+ klog.Infof("clusterName:%v nodeNameToIPs: %v, err:%v",
clusterName, nnti, err)
continue
}
str := ""
for _, v := range res {
- v := v
str = string(v.Metric["instance"])
}
WithDownNodeIPs[node] = str
}
- WithDownNodeIPsMsg := fmt.Sprintf("【%s】\n【集群：%v】\n【宕机：%v】\n",
h.cc.NodeDownTime.DingTalk.Title, clusterName, len(WithDownNodeIPs))
+
+ WithDownNodeIPsMsg := fmt.Sprintf("\n【%s】\n【集群：%v】\n【宕机：%v】\n",
h.cc.NodeDownTime.DingTalk.Title, clusterName, len(WithDownNodeIPs))
newfound := 0
for nodeName, nodeIP := range WithDownNodeIPs {
today := time.Now().Format("2006-01-02")
diff --git a/deploy/horus/horus.yaml b/deploy/horus/horus.yaml
index 134ac084..95c6c385 100644
--- a/deploy/horus/horus.yaml
+++ b/deploy/horus/horus.yaml
@@ -36,10 +36,10 @@ kubeMultiple:
cluster: config.1
promMultiple:
- cluster: http://192.168.15.128:32427
+ cluster: http://192.168.15.128:31505
nodeRecovery:
- enabled: true
+ enabled: false
dayNumber: 1
checkIntervalSecond: 5
promQueryTimeSecond: 60
@@ -52,7 +52,7 @@ nodeRecovery:
webhookUrl:
"https://hooks.slack.com/services/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";
customModular:
- enabled: true
+ enabled: false
cordonDailyLimit:
filesystem_readonly: 1
checkQL:
@@ -80,8 +80,10 @@ nodeDownTime:
checkQL:
- node_disk_info
- node_cpu_guest_seconds_total
- nodeNameToIPs: |-
+ nodeNameToIPs:
node_os_info{node="%s"}
+ kubeMultiple:
+ cluster: config.1
dingTalk:
webhookUrl:
"https://oapi.dingtalk.com/robot/send?access_token=37f8891e60e524013275cc01efafdb5976b81ef7269ce271b769bcd025826c12";
title: "horus 通知"
