This is an automated email from the ASF dual-hosted git repository.
zhongxjian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/dubbo-kubernetes.git
The following commit(s) were added to refs/heads/master by this push:
new 1ea361cb [horus] Node Restart Policy Logic (#367)
1ea361cb is described below
commit 1ea361cb85959c1d8ba62f83c2d8dde275f7a2b9
Author: mfordjody <[email protected]>
AuthorDate: Mon Sep 23 08:30:23 2024 +0800
[horus] Node Restart Policy Logic (#367)
fix typo
---
app/horus/basic/db/db.go | 12 +++++++++
app/horus/core/horuser/restart.go | 57 +++++++++++++++++++++++++++++++++++++++
go.mod | 2 +-
3 files changed, 70 insertions(+), 1 deletion(-)
diff --git a/app/horus/basic/db/db.go b/app/horus/basic/db/db.go
index 8040d52e..35cac1ee 100644
--- a/app/horus/basic/db/db.go
+++ b/app/horus/basic/db/db.go
@@ -126,6 +126,13 @@ func GetRecoveryNodeDataInfoDate(day int) ([]NodeDataInfo,
error) {
return ndi, err
}
+func GetRestartNodeDataInfoDate() ([]NodeDataInfo, error) {
+ var ndi []NodeDataInfo
+ session := db.Where("restart = 0 and repair = 0 and module_name= ? ",
"node_down")
+ err := session.Find(&ndi)
+ return ndi, err
+}
+
func GetDailyLimitNodeDataInfoDate(day, module, cluster string)
([]NodeDataInfo, error) {
var ndi []NodeDataInfo
session := db.Where("DATE(first_date) = ? AND module_name = ? AND
cluster_name = ?", day, module, cluster)
@@ -137,3 +144,8 @@ func (n *NodeDataInfo) RecoveryMarker() (bool, error) {
n.RecoveryMark = 1
return n.Update()
}
+
+func (n *NodeDataInfo) RestartMarker() (bool, error) {
+ n.Restart = 1
+ return n.Update()
+}
diff --git a/app/horus/core/horuser/restart.go
b/app/horus/core/horuser/restart.go
new file mode 100644
index 00000000..67127b68
--- /dev/null
+++ b/app/horus/core/horuser/restart.go
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package horuser
+
+import (
+ "context"
+ "fmt"
+ "github.com/apache/dubbo-kubernetes/app/horus/basic/db"
+ "github.com/apache/dubbo-kubernetes/app/horus/core/alert"
+ "github.com/gammazero/workerpool"
+ "k8s.io/apimachinery/pkg/util/wait"
+ "k8s.io/klog/v2"
+ "time"
+)
+
+func (h *Horuser) DowntimeRestartManager(ctx context.Context) error {
+ go wait.UntilWithContext(ctx, h.RestartOrRepair,
time.Duration(h.cc.NodeDownTime.CheckIntervalSecond)*time.Second)
+ <-ctx.Done()
+ return nil
+}
+
+func (h *Horuser) RestartOrRepair(ctx context.Context) {
+ nodes, err := db.GetRestartNodeDataInfoDate()
+ if err != nil {
+ klog.Errorf("RestartOrRepair err:%v", err)
+ }
+ if len(nodes) == 0 {
+ klog.Warningf("Needs to be rebooted or fixed to zero.")
+ }
+ klog.Infof("GetRestartNodeDataInfoDate.count:%v", len(nodes))
+ wp := workerpool.New(10)
+ for _, n := range nodes {
+ n := n
+ wp.Submit(func() {
+ h.TryRestart(n)
+ })
+ }
+}
+
+func (h *Horuser) TryRestart(node db.NodeDataInfo) {
+ msg := fmt.Sprintf("node restart or force restart.node:%v date:%v
cluster:%v", node.NodeName, node.FirstDate, node.ClusterName)
+ alert.DingTalkSend(h.cc.DingTalk, msg)
+ node.RestartMarker()
+}
diff --git a/go.mod b/go.mod
index 75da2cdb..2e45fae2 100644
--- a/go.mod
+++ b/go.mod
@@ -38,6 +38,7 @@ require (
github.com/envoyproxy/protoc-gen-validate v1.0.2
github.com/evanphx/json-patch/v5 v5.8.0
github.com/fullstorydev/grpcurl v1.9.1
+ github.com/gammazero/workerpool v1.1.3
github.com/gin-gonic/gin v1.9.1
github.com/go-co-op/gocron v1.9.0
github.com/go-git/go-billy/v5 v5.4.1
@@ -200,7 +201,6 @@ require (
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/gammazero/deque v0.2.0 // indirect
- github.com/gammazero/workerpool v1.1.3 // indirect
github.com/gdamore/encoding v1.0.0 // indirect
github.com/gdamore/tcell/v2 v2.6.0 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
