This is an automated email from the ASF dual-hosted git repository.
zhongxjian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/dubbo-kubernetes.git
The following commit(s) were added to refs/heads/master by this push:
new 1be672d4 [horus] Try to add downtime recovery (#444)
1be672d4 is described below
commit 1be672d45d5f2081aeef140c414f9fe70b383c7d
Author: mfordjody <[email protected]>
AuthorDate: Thu Oct 10 08:38:27 2024 +0800
[horus] Try to add downtime recovery (#444)
---
app/horus/base/config/config.go | 1 +
app/horus/base/db/db.go | 32 +++++++++++++++++---------------
app/horus/core/horuser/node_downtime.go | 1 +
manifests/horus/horus.yaml | 6 +++++-
4 files changed, 24 insertions(+), 16 deletions(-)
diff --git a/app/horus/base/config/config.go b/app/horus/base/config/config.go
index fdb38d1c..1001e630 100644
--- a/app/horus/base/config/config.go
+++ b/app/horus/base/config/config.go
@@ -73,6 +73,7 @@ type DowntimeConfiguration struct {
KubeMultiple map[string]string `yaml:"kubeMultiple"`
AbnormalityQL []string `yaml:"abnormalityQL"`
AbnormalInfoSystemQL string
`yaml:"abnormalInfoSystemQL"`
+ AbnormalRecoveryQL []string `yaml:"abnormalRecoveryQL"`
AllSystemUser string `yaml:"allSystemUser"`
AllSystemPassword string `yaml:"allSystemPassword"`
DingTalk *DingTalkConfiguration `yaml:"dingTalk"`
diff --git a/app/horus/base/db/db.go b/app/horus/base/db/db.go
index d552ad0d..75a43ddd 100644
--- a/app/horus/base/db/db.go
+++ b/app/horus/base/db/db.go
@@ -25,21 +25,23 @@ import (
)
type NodeDataInfo struct {
- Id int64 `json:"id"`
- NodeName string `json:"node_name" xorm:"node_name"`
- NodeIP string `json:"node_ip" xorm:"node_ip"`
- Sn string `json:"sn"`
- ClusterName string `json:"cluster_name" xorm:"cluster_name"`
- ModuleName string `json:"module_name" xorm:"module_name"`
- Reason string `json:"reason"`
- Restart int `json:"restart"`
- Repair int `json:"repair"`
- RepairTicketUrl string `json:"repair_ticket_url"
xorm:"repair_ticket_url"`
- FirstDate string `json:"first_date" xorm:"first_date"`
- CreateTime time.Time `json:"create_time" xorm:"create_time
created"`
- UpdateTime time.Time `json:"update_time" xorm:"update_time
updated"`
- RecoveryMark int64 `json:"recovery_mark" xorm:"recovery_mark"`
- RecoveryQL string `json:"recovery_ql" xorm:"recovery_ql"`
+ Id int64 `json:"id"`
+ NodeName string `json:"node_name" xorm:"node_name"`
+ NodeIP string `json:"node_ip" xorm:"node_ip"`
+ Sn string `json:"sn"`
+ ClusterName string `json:"cluster_name" xorm:"cluster_name"`
+ ModuleName string `json:"module_name" xorm:"module_name"`
+ Reason string `json:"reason"`
+ Restart int `json:"restart"`
+ Repair int `json:"repair"`
+ RepairTicketUrl string `json:"repair_ticket_url"
xorm:"repair_ticket_url"`
+ FirstDate string `json:"first_date" xorm:"first_date"`
+ CreateTime time.Time `json:"create_time" xorm:"create_time
created"`
+ UpdateTime time.Time `json:"update_time" xorm:"update_time
updated"`
+ RecoveryMark int64 `json:"recovery_mark"
xorm:"recovery_mark"`
+ RecoveryQL string `json:"recovery_ql" xorm:"recovery_ql"`
+ DownTimeRecoveryQL []string `json:"downtime_recovery_ql
xorm:downtime_recovery_ql"`
+ DownTimeRecoveryMark int64 `json:"downtime_recovery_mark
xorm:downtime_recovery_mark"`
}
type PodDataInfo struct {
diff --git a/app/horus/core/horuser/node_downtime.go
b/app/horus/core/horuser/node_downtime.go
index 68c96beb..0ddeacdb 100644
--- a/app/horus/core/horuser/node_downtime.go
+++ b/app/horus/core/horuser/node_downtime.go
@@ -147,6 +147,7 @@ func (h *Horuser) DownTimeNodes(clusterName, addr string) {
NodeIP: nodeIP,
ClusterName: clusterName,
ModuleName: NODE_DOWN,
+ RecoveryQL: ,
}
exist, _ := write.Check()
if exist {
diff --git a/manifests/horus/horus.yaml b/manifests/horus/horus.yaml
index fdac8695..22eb575f 100644
--- a/manifests/horus/horus.yaml
+++ b/manifests/horus/horus.yaml
@@ -75,6 +75,10 @@ nodeDownTime:
node_os_info{node="%s"}
allSystemUser: "zxj"
allSystemPassword: "1"
+ abnormalRecoveryQL:
+ - 100 - (avg by (node)
(rate(node_cpu_seconds_total{mode="idle",node="%s"}[5m])) * 100) < 20
+ - (avg by (node) (node_memory_MemFree_bytes{node="%s"} /
node_memory_MemTotal_bytes{node="%s"} )) * 100 < 25
+ # - node_filesystem_avail_bytes{mountpoint="/"} /
node_filesystem_size_bytes{mountpoint="/"} * 100 > 15
kubeMultiple:
cluster: config.1
dingTalk:
@@ -87,7 +91,7 @@ nodeDownTime:
title: "自定义通知"
podStagnationCleaner:
- enabled: false
+ enabled: true
intervalSecond: 15
doubleSecond: 60
fieldSelector: "status.phase!=Running"
