This is an automated email from the ASF dual-hosted git repository.
ocket8888 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
The following commit(s) were added to refs/heads/master by this push:
new 66e98cb Add t3c retrying app lock (#6357)
66e98cb is described below
commit 66e98cb8cc1e8b329ff860e67905040568fccdf0
Author: Robert O Butts <[email protected]>
AuthorDate: Thu Nov 18 16:33:30 2021 -0700
Add t3c retrying app lock (#6357)
* Add t3c retrying app lock
* Remove test function newlines
Per PR Review.
* Change variable to const
Per PR Review.
* Change format strings to types
Per PR Review.
---
CHANGELOG.md | 1 +
cache-config/t3c-apply/t3c-apply.go | 15 ++-
cache-config/t3c-apply/util/util.go | 1 -
.../testing/ort-tests/t3c-lockfile_test.go | 113 +++++++++++++++++++++
4 files changed, 127 insertions(+), 3 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 27ff4ad..c087f6e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,7 @@ The format is based on [Keep a
Changelog](http://keepachangelog.com/en/1.0.0/).
- Added definition for `heartbeat.polling.interval` for CDN Traffic Monitor
config in API documentation.
- New `pkg` script options, `-h`, `-s`, `-S`, and `-L`.
- Added `Invalidation Type` (REFRESH or REFETCH) for invalidating content to
Traffic Portal.
+- cache config t3c-apply retrying when another t3c-apply is running.
- IMS warnings to Content Invalidation requests in Traffic Portal and
documentation.
- [#6032](https://github.com/apache/trafficcontrol/issues/6032) Add t3c
setting mode 0600 for secure files
diff --git a/cache-config/t3c-apply/t3c-apply.go
b/cache-config/t3c-apply/t3c-apply.go
index 992cf80..b685145 100644
--- a/cache-config/t3c-apply/t3c-apply.go
+++ b/cache-config/t3c-apply/t3c-apply.go
@@ -59,6 +59,10 @@ func runSysctl(cfg config.Cfg) {
}
}
+const LockFilePath = "/var/run/t3c.lock"
+const LockFileRetryInterval = time.Second
+const LockFileRetryTimeout = time.Minute
+
func main() {
var syncdsUpdate torequest.UpdateStatus
var lock util.FileLock
@@ -69,9 +73,16 @@ func main() {
} else if cfg == (config.Cfg{}) { // user used the --help option
os.Exit(Success)
}
- if !lock.GetLock("/var/run/t3c.lock") {
- os.Exit(AlreadyRunning)
+
+ log.Infoln("Trying to acquire app lock")
+ for lockStart := time.Now(); !lock.GetLock(LockFilePath); {
+ if time.Since(lockStart) > LockFileRetryTimeout {
+ log.Errorf("Failed to get app lock after %v seconds,
another instance is running, exiting without running\n",
int(LockFileRetryTimeout/time.Second))
+ os.Exit(AlreadyRunning)
+ }
+ time.Sleep(LockFileRetryInterval)
}
+ log.Infoln("Acquired app lock")
if cfg.UseGit == config.UseGitYes {
err := util.EnsureConfigDirIsGitRepo(cfg)
diff --git a/cache-config/t3c-apply/util/util.go
b/cache-config/t3c-apply/util/util.go
index 7f8c36d..4a10aee 100644
--- a/cache-config/t3c-apply/util/util.go
+++ b/cache-config/t3c-apply/util/util.go
@@ -81,7 +81,6 @@ func (f *FileLock) GetLock(lockFile string) bool {
return false
}
if !f.is_locked { // another process is running.
- log.Errorf("Another t3c process is already running, try again
later\n")
return false
}
diff --git a/cache-config/testing/ort-tests/t3c-lockfile_test.go
b/cache-config/testing/ort-tests/t3c-lockfile_test.go
new file mode 100644
index 0000000..2dce73d
--- /dev/null
+++ b/cache-config/testing/ort-tests/t3c-lockfile_test.go
@@ -0,0 +1,113 @@
+package orttest
+
+/*
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+import (
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/apache/trafficcontrol/cache-config/testing/ort-tests/tcdata"
+)
+
+func TestLockfile(t *testing.T) {
+ testName := "TestLockfile"
+ t.Logf("------------- Starting " + testName + " ---------------")
+ tcd.WithObjs(t, []tcdata.TCObj{
+ tcdata.CDNs, tcdata.Types, tcdata.Tenants, tcdata.Parameters,
+ tcdata.Profiles, tcdata.ProfileParameters, tcdata.Statuses,
+ tcdata.Divisions, tcdata.Regions, tcdata.PhysLocations,
+ tcdata.CacheGroups, tcdata.Servers, tcdata.Topologies,
+ tcdata.DeliveryServices}, func() {
+
+ const hostName = `atlanta-edge-03`
+ const fileName = `records.config`
+
+ firstOut := ""
+ firstCode := 0
+ wg := sync.WaitGroup{}
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ t.Logf("TestLockFile first t3c starting %s", time.Now())
+ firstOut, firstCode = t3cUpdateReload(hostName,
"badass")
+ t.Logf("TestLockFile first t3c finished %s", time.Now())
+ }()
+
+ time.Sleep(time.Millisecond * 100) // sleep long enough to
ensure the concurrent t3c starts
+ t.Logf("TestLockFile second t3c starting %s", time.Now())
+ out, code := t3cUpdateReload(hostName, "badass")
+ t.Logf("TestLockFile second t3c finished %s", time.Now())
+ if code != 0 {
+ t.Fatalf("second t3c apply badass failed: output
'''%s''' code %d", out, code)
+ }
+
+ wg.Wait()
+ if firstCode != 0 {
+ t.Fatalf("first t3c apply badass failed: output
'''%s''' code %d", firstOut, firstCode)
+ }
+
+ outStr := string(out)
+ outLines := strings.Split(outStr, "\n")
+ acquireStartLine := ""
+ acquireEndLine := ""
+ for _, line := range outLines {
+ if strings.Contains(line, "Trying to acquire app lock")
{
+ acquireStartLine = line
+ } else if strings.Contains(line, "Acquired app lock") {
+ acquireEndLine = line
+ }
+ if acquireStartLine != "" && acquireEndLine != "" {
+ break
+ }
+ }
+
+ if acquireStartLine == "" || acquireEndLine == "" {
+ t.Fatalf("t3c apply output expected to contain 'Trying
to acquire app lock' and 'Acquired app lock', actual: '''%s'''", out)
+ }
+
+ acquireStart := parseLogLineTime(acquireStartLine)
+ if acquireStart == nil {
+ t.Fatalf("t3c apply acquire line failed to parse time,
line '" + acquireStartLine + "'")
+ }
+ acquireEnd := parseLogLineTime(acquireEndLine)
+ if acquireEnd == nil {
+ t.Fatalf("t3c apply acquire line failed to parse time,
line '" + acquireEndLine + "'")
+ }
+
+ minDiff := time.Second * 1 // checking the file lock should
never take 1s, so that's enough to verify it was hit
+ if diff := acquireEnd.Sub(*acquireStart); diff < minDiff {
+ t.Fatalf("t3c apply expected time to acquire while
another t3c is running to be at least %s, actual %s start line '%s' end line
'%s'", minDiff, diff, acquireStartLine, acquireEndLine)
+ }
+
+ t.Logf(testName + " succeeded")
+ })
+ t.Logf("------------- End of " + testName + " ---------------")
+}
+
+func parseLogLineTime(line string) *time.Time {
+ fields := strings.Fields(line)
+ if len(fields) < 3 {
+ return nil
+ }
+ timeStr := fields[2]
+ timeStr = strings.TrimSuffix(timeStr, ":")
+ tm, err := time.Parse(time.RFC3339Nano, timeStr)
+ if err != nil {
+ return nil
+ }
+ return &tm
+}
