This is an automated email from the ASF dual-hosted git repository.
zhongxjian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/dubbo-kubernetes.git
The following commit(s) were added to refs/heads/master by this push:
new cba7a82c [horus] Fixing logic errors (#369)
cba7a82c is described below
commit cba7a82ca29c55b51d99ef87bc3130e16d164107
Author: mfordjody <[email protected]>
AuthorDate: Mon Sep 23 10:14:25 2024 +0800
[horus] Fixing logic errors (#369)
---
app/horus/cmd/main.go | 2 +-
app/horus/core/horuser/downtime.go | 44 ++++++++++++++++++--------------------
app/horus/core/horuser/restart.go | 13 +++++------
deploy/horus/horus.yaml | 8 +++++--
4 files changed, 35 insertions(+), 32 deletions(-)
diff --git a/app/horus/cmd/main.go b/app/horus/cmd/main.go
index 909ad234..9d641063 100644
--- a/app/horus/cmd/main.go
+++ b/app/horus/cmd/main.go
@@ -160,7 +160,7 @@ func setupStopChanWithContext() (*WaitGroup, <-chan
struct{}) {
g.Add(func() error {
select {
case <-SignalChan:
- close(stopChan)
+ close(SignalChan)
}
return nil
})
diff --git a/app/horus/core/horuser/downtime.go
b/app/horus/core/horuser/downtime.go
index 00c67d39..8c16ad34 100644
--- a/app/horus/core/horuser/downtime.go
+++ b/app/horus/core/horuser/downtime.go
@@ -28,9 +28,7 @@ import (
const (
NODE_DOWN = "node_down"
- NODE_DOWN_REASON = "node_down_aegis"
- POWER_OFF = "POWER_OFF"
- POWER_ON = "POWER_ON"
+ NODE_DOWN_REASON = "failed"
)
func (h *Horuser) DownTimeManager(ctx context.Context) error {
@@ -60,14 +58,14 @@ func (h *Horuser) DownTimeCheck(ctx context.Context) {
}
func (h *Horuser) DownTimeNodes(clusterName, addr string) {
- klog.Infof("DownTimeNodes QueryStart clusterName:%v", clusterName)
- resMap := map[string]int{}
- checkQl := len(h.cc.NodeDownTime.CheckQL)
+ klog.Infof("DownTimeNodes Query Start clusterName:%v", clusterName)
+ nodeDownTimeRes := map[string]int{}
+ cq := len(h.cc.NodeDownTime.CheckQL)
for _, ql := range h.cc.NodeDownTime.CheckQL {
ql := ql
res, err := h.InstantQuery(addr, ql, clusterName,
h.cc.NodeDownTime.PromQueryTimeSecond)
if err != nil {
- klog.Errorf("downtimeNodes InstantQuery err:%v", err)
+ klog.Errorf("downtimeNodes Instant Query err:%v", err)
klog.Infof("clusterName:%v", clusterName)
continue
}
@@ -76,33 +74,33 @@ func (h *Horuser) DownTimeNodes(clusterName, addr string) {
v := v
nodeName := string(v.Metric["node"])
if nodeName == "" {
- klog.Errorf("downtimeNodes InstantQuery
nodeName empty")
+ klog.Errorf("downtimeNodes InstantQuery
nodeName empty.")
klog.Infof("clusterName:%v metrics:%v",
clusterName, v.Metric)
continue
}
- resMap[nodeName]++
+ nodeDownTimeRes[nodeName]++
}
}
- for node, count := range resMap {
- if count < checkQl {
- klog.Errorf("downtimeNodes node not reach threshold")
- klog.Infof("clusterName:%v node:%v threshold:%v
count:%v", clusterName, node, checkQl, count)
+ for node, count := range nodeDownTimeRes {
+ if count < cq {
+ klog.Errorf("downtimeNodes node not reach threshold.")
+ klog.Infof("clusterName:%v node:%v threshold:%v
count:%v", clusterName, node, cq, count)
continue
}
}
WithDownNodeIPs := map[string]string{}
- for node, count := range resMap {
- if count < checkQl {
+ for node, count := range nodeDownTimeRes {
+ if count < cq {
klog.Errorf("downtimeNodes node not reach threshold")
- klog.Infof("clusterName:%v nodeName:%v threshold:%v
count:%v", clusterName, node, checkQl, count)
+ klog.Infof("clusterName:%v nodeName:%v threshold:%v
count:%v", clusterName, node, cq, count)
continue
}
- toNodeNameips := fmt.Sprintf(h.cc.NodeDownTime.NodeNameToIPs,
node)
- res, err := h.InstantQuery(toNodeNameips, addr, clusterName,
h.cc.NodeDownTime.PromQueryTimeSecond)
+ NodeNameToIps := fmt.Sprintf(h.cc.NodeDownTime.NodeNameToIPs,
node)
+ res, err := h.InstantQuery(NodeNameToIps, addr, clusterName,
h.cc.NodeDownTime.PromQueryTimeSecond)
if err != nil {
klog.Errorf("downtimeNodes InstantQuery NodeName To IPs
empty err:%v", err)
- klog.Infof("clusterName:%v toNodeNameips:%v err:%v",
clusterName, toNodeNameips, err)
+ klog.Infof("clusterName:%v toNodeNameips:%v err:%v",
clusterName, NodeNameToIps, err)
continue
}
str := ""
@@ -132,6 +130,10 @@ func (h *Horuser) DownTimeNodes(clusterName, addr string) {
continue
}
newfound++
+ if newfound > 0 {
+ klog.Infof("NodeDownTimeCheckOnCluster get
toNodeNameips result msg:%v clusterName:%v count:%v detail:%v",
WithDownNodeIPsMsg, clusterName, len(nodeIP), nodeName)
+ alert.DingTalkSend(h.cc.NodeDownTime.DingTalk,
WithDownNodeIPsMsg)
+ }
WithDownNodeIPsMsg += fmt.Sprintf("node:%v ip:%v", nodeName,
nodeIP)
write.Reason = NODE_DOWN_REASON
write.FirstDate = today
@@ -141,9 +143,5 @@ func (h *Horuser) DownTimeNodes(clusterName, addr string) {
klog.Infof("cluster:%v node:%v", clusterName, nodeName)
}
klog.Infof("NodeDownTimeCheckOnCluster abnormal cordonNode
AddOrGetOne cluster:%v node:%v", clusterName, nodeName)
- if newfound > 0 {
- klog.Infof("NodeDownTimeCheckOnCluster get
toNodeNameips result msg:%v clusterName:%v count:%v detail:%v",
WithDownNodeIPsMsg, clusterName, len(nodeIP), nodeName)
- alert.DingTalkSend(h.cc.NodeDownTime.DingTalk,
WithDownNodeIPsMsg)
- }
}
}
diff --git a/app/horus/core/horuser/restart.go
b/app/horus/core/horuser/restart.go
index 67127b68..dc85fd92 100644
--- a/app/horus/core/horuser/restart.go
+++ b/app/horus/core/horuser/restart.go
@@ -35,13 +35,13 @@ func (h *Horuser) DowntimeRestartManager(ctx
context.Context) error {
func (h *Horuser) RestartOrRepair(ctx context.Context) {
nodes, err := db.GetRestartNodeDataInfoDate()
if err != nil {
- klog.Errorf("RestartOrRepair err:%v", err)
+ klog.Errorf("Restart or repair err:%v", err)
}
if len(nodes) == 0 {
klog.Warningf("Needs to be rebooted or fixed to zero.")
}
- klog.Infof("GetRestartNodeDataInfoDate.count:%v", len(nodes))
- wp := workerpool.New(10)
+ klog.Infof("GetRestartNodeDataInfoDate count:%v", len(nodes))
+ wp := workerpool.New(30)
for _, n := range nodes {
n := n
wp.Submit(func() {
@@ -51,7 +51,8 @@ func (h *Horuser) RestartOrRepair(ctx context.Context) {
}
func (h *Horuser) TryRestart(node db.NodeDataInfo) {
- msg := fmt.Sprintf("node restart or force restart.node:%v date:%v
cluster:%v", node.NodeName, node.FirstDate, node.ClusterName)
- alert.DingTalkSend(h.cc.DingTalk, msg)
- node.RestartMarker()
+ msg := fmt.Sprintf("node restart or force restart node:%v date:%v
cluster:%v", node.NodeName, node.FirstDate, node.ClusterName)
+ alert.DingTalkSend(h.cc.NodeDownTime.DingTalk, msg)
+ pass, err := node.RestartMarker()
+ klog.Infof("RestartMarker result pass:%v err:%v", pass, err)
}
diff --git a/deploy/horus/horus.yaml b/deploy/horus/horus.yaml
index 2d2229fb..134ac084 100644
--- a/deploy/horus/horus.yaml
+++ b/deploy/horus/horus.yaml
@@ -36,7 +36,7 @@ kubeMultiple:
cluster: config.1
promMultiple:
- cluster: http://192.168.15.128:31160
+ cluster: http://192.168.15.128:32427
nodeRecovery:
enabled: true
@@ -74,10 +74,14 @@ customModular:
webhookUrl:
"https://hooks.slack.com/services/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";
nodeDownTime:
- enabled: false
+ enabled: true
checkIntervalSecond: 5
promQueryTimeSecond: 60
checkQL:
+ - node_disk_info
+ - node_cpu_guest_seconds_total
+ nodeNameToIPs: |-
+ node_os_info{node="%s"}
dingTalk:
webhookUrl:
"https://oapi.dingtalk.com/robot/send?access_token=37f8891e60e524013275cc01efafdb5976b81ef7269ce271b769bcd025826c12";
title: "horus 通知"
