This is an automated email from the ASF dual-hosted git repository.
zhongxjian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/dubbo-kubernetes.git
The following commit(s) were added to refs/heads/master by this push:
new 0b726a1f [horus] Add slack alerts (#354)
0b726a1f is described below
commit 0b726a1f21c09075e9efa79de728e5f797754cfc
Author: mfordjody <[email protected]>
AuthorDate: Thu Sep 19 13:25:45 2024 +0800
[horus] Add slack alerts (#354)
---
app/horus/basic/config/file.go | 2 ++
app/horus/core/alert/slack_test.go | 2 +-
app/horus/core/horuser/modular.go | 4 +++-
app/horus/core/horuser/recovery.go | 1 +
deploy/horus/horus.yaml | 10 +++++++---
5 files changed, 14 insertions(+), 5 deletions(-)
diff --git a/app/horus/basic/config/file.go b/app/horus/basic/config/file.go
index 219a1ca7..18ae7e3a 100644
--- a/app/horus/basic/config/file.go
+++ b/app/horus/basic/config/file.go
@@ -49,6 +49,7 @@ type RecoveryConfiguration struct {
CheckIntervalSecond int `yaml:"checkIntervalSecond"`
PromQueryTimeSecond int64 `yaml:"promQueryTimeSecond"`
DingTalk *DingTalkConfiguration `yaml:"dingTalk"`
+ Slack *SlackConfiguration `yaml:"slack"`
}
type ModularConfiguration struct {
@@ -60,4 +61,5 @@ type ModularConfiguration struct {
PromQueryTimeSecond int64 `yaml:"promQueryTimeSecond"`
KubeMultiple map[string]string `yaml:"kubeMultiple"`
DingTalk *DingTalkConfiguration `yaml:"dingTalk"`
+ Slack *SlackConfiguration `yaml:"slack"`
}
diff --git a/app/horus/core/alert/slack_test.go
b/app/horus/core/alert/slack_test.go
index d254a043..48325861 100644
--- a/app/horus/core/alert/slack_test.go
+++ b/app/horus/core/alert/slack_test.go
@@ -23,7 +23,7 @@ import (
func TestSlackSend(t *testing.T) {
im := &config.SlackConfiguration{
- WebhookUrl:
"https://oapi.dingtalk.com/robot/send?access_token=aa2f3f74d7a2504653ca89b7a673707ba1d04b6d9d320c3572e5464d2f81471x";,
+ WebhookUrl:
"https://hooks.slack.com/services/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";,
}
alert.SlackSend(im, "test")
}
diff --git a/app/horus/core/horuser/modular.go
b/app/horus/core/horuser/modular.go
index df51676e..c6501346 100644
--- a/app/horus/core/horuser/modular.go
+++ b/app/horus/core/horuser/modular.go
@@ -87,7 +87,7 @@ func (h *Horuser) CustomizeModularNodes(clusterName,
moduleName, nodeName, ip st
return
}
err = h.Cordon(nodeName, clusterName, moduleName)
-
+
write := db.NodeDataInfo{
NodeName: nodeName,
NodeIP: ip,
@@ -118,12 +118,14 @@ func (h *Horuser) CustomizeModularNodes(clusterName,
moduleName, nodeName, ip st
msg := fmt.Sprintf("\n【集群:%v】\n【发现 %s 异常已禁止调度】\n【已禁止调度节点:%v】\n 【处理结果:
%v】\n 【今日操作次数:%v】\n",
clusterName, moduleName, nodeName, res, len(data)+1)
alert.DingTalkSend(h.cc.CustomModular.DingTalk, msg)
+ alert.SlackSend(h.cc.CustomModular.Slack, msg)
dailyLimit := h.cc.CustomModular.CordonDailyLimit[moduleName]
if len(data) > dailyLimit {
msg := fmt.Sprintf("【日期:%v】 【集群:%v\n】 【今日 Cordon 节点数: %v】\n
【已达到今日上限: %v】\n 【节点:%v】",
data, clusterName, len(data), dailyLimit, nodeName)
alert.DingTalkSend(h.cc.CustomModular.DingTalk, msg)
+ alert.SlackSend(h.cc.CustomModular.Slack, msg)
return
}
diff --git a/app/horus/core/horuser/recovery.go
b/app/horus/core/horuser/recovery.go
index e97617e9..d9137588 100644
--- a/app/horus/core/horuser/recovery.go
+++ b/app/horus/core/horuser/recovery.go
@@ -82,6 +82,7 @@ func (h *Horuser) recoveryNodes(n db.NodeDataInfo) {
}
msg := fmt.Sprintf("\n【集群: %v】\n【异常节点恢复调度】\n【已恢复调度节点:
%v】\n【处理结果：%v】\n【日期: %v】\n", n.ClusterName, n.NodeName, res, n.CreateTime)
alert.DingTalkSend(h.cc.NodeRecovery.DingTalk, msg)
+ alert.SlackSend(h.cc.CustomModular.Slack, msg)
pass, err := n.RecoveryMarker()
klog.Infof("RecoveryMarker result pass:%v err:%v", pass, err)
diff --git a/deploy/horus/horus.yaml b/deploy/horus/horus.yaml
index e68be09f..4fb6e780 100644
--- a/deploy/horus/horus.yaml
+++ b/deploy/horus/horus.yaml
@@ -48,11 +48,13 @@ nodeRecovery:
title: "horus 通知"
atMobiles:
- 15000000
+ slack:
+ webhookUrl:
"https://hooks.slack.com/services/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";
customModular:
- enabled: false
+ enabled: true
cordonDailyLimit:
- filesystem_readonly: 3
+ filesystem_readonly: 1
checkQL:
filesystem_readonly: |-
node_filesystem_readonly{mountpoint="/"} != 1
@@ -67,4 +69,6 @@ customModular:
webhookUrl:
"https://oapi.dingtalk.com/robot/send?access_token=37f8891e60e524013275cc01efafdb5976b81ef7269ce271b769bcd025826c12";
title: "horus 通知"
atMobiles:
- - 15000000
\ No newline at end of file
+ - 15000000
+ slack:
+ webhookUrl:
"https://hooks.slack.com/services/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";
\ No newline at end of file
