This is an automated email from the ASF dual-hosted git repository.
rawlin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
The following commit(s) were added to refs/heads/master by this push:
new c95689e Adding access.log to Traffic Monitor (#6620)
c95689e is described below
commit c95689eef090c36df50ce4dd4b8f3909ce968d45
Author: Srijeet Chatterjee <[email protected]>
AuthorDate: Mon Mar 14 16:20:35 2022 -0600
Adding access.log to Traffic Monitor (#6620)
* Adding access.log to Traffic Monitor
* code review fixes
* fix nil ptr, time format
* code review
* fix debug
---
.github/actions/tm-integration-tests/entrypoint.sh | 1 +
CHANGELOG.md | 1 +
.../roles/traffic-monitor/defaults/main.yml | 1 +
.../templates/traffic_monitor.cfg.j2 | 1 +
.../traffic_monitor/traffic_monitor.cfg | 1 +
infrastructure/cdn-in-a-box/variables.env | 1 +
lib/go-log/log.go | 32 +++++++++++------
traffic_monitor/build/traffic_monitor.logrotate | 10 ++++++
traffic_monitor/conf/traffic_monitor.cfg | 1 +
traffic_monitor/config/config.go | 15 ++++++++
traffic_monitor/config/config_test.go | 2 ++
traffic_monitor/datareq/datareq.go | 41 +++++++++++++++++++---
.../tests/_integration/tm/Dockerfile_run.sh | 1 +
traffic_monitor/traffic_monitor.go | 14 ++++++++
14 files changed, 108 insertions(+), 14 deletions(-)
diff --git a/.github/actions/tm-integration-tests/entrypoint.sh
b/.github/actions/tm-integration-tests/entrypoint.sh
index 676e2ae..a9dad03 100755
--- a/.github/actions/tm-integration-tests/entrypoint.sh
+++ b/.github/actions/tm-integration-tests/entrypoint.sh
@@ -129,6 +129,7 @@ cat > ./traffic_monitor.cfg <<- EOF
"max_events": 200,
"health_flush_interval_ms": 20,
"stat_flush_interval_ms": 20,
+ "log_location_access": null,
"log_location_event": null,
"log_location_error": "err.log",
"log_location_warning": "warn.log",
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3ff373d..b21fa30 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@ The format is based on [Keep a
Changelog](http://keepachangelog.com/en/1.0.0/).
- Traffic Monitor config option `distributed_polling` which enables the
ability for Traffic Monitor to poll a subset of the CDN and divide into "local
peer groups" and "distributed peer groups". Traffic Monitors in the same group
are local peers, while Traffic Monitors in other groups are distibuted peers.
Each TM group polls the same set of cachegroups and gets availability data for
the other cachegroups from other TM groups. This allows each TM to be
responsible for polling a subset of [...]
- Added support for a new Traffic Ops GLOBAL profile parameter --
`tm_query_status_override` -- to override which status of Traffic Monitors to
query (default: ONLINE).
- Traffic Router: Add support for `file`-protocol URLs for the
`geolocation.polling.url` for the Geolocation database.
+- Traffic Monitor: Add support for `access.log` to TM.
- Added functionality for login to provide a Bearer token and for that token
to be later used for authorization.
### Fixed
diff --git a/infrastructure/ansible/roles/traffic-monitor/defaults/main.yml
b/infrastructure/ansible/roles/traffic-monitor/defaults/main.yml
index 5915623..022f99c 100644
--- a/infrastructure/ansible/roles/traffic-monitor/defaults/main.yml
+++ b/infrastructure/ansible/roles/traffic-monitor/defaults/main.yml
@@ -33,6 +33,7 @@ tm_log_location_error: "{{ tm_log_dir }}/traffic_monitor.log"
tm_log_location_warning: "{{ tm_log_dir }}/traffic_monitor.log"
tm_log_location_info: "null"
tm_log_location_debug: "null"
+tm_log_location_access: "{{ tm_log_dir }}/access.log"
tm_log_location_event: "{{ tm_log_dir }}/event.log"
tm_stat_flush_interval_ms: 20
tm_serve_read_timeout_ms: 10000
diff --git
a/infrastructure/ansible/roles/traffic-monitor/templates/traffic_monitor.cfg.j2
b/infrastructure/ansible/roles/traffic-monitor/templates/traffic_monitor.cfg.j2
index 2e3aa42..37c9021 100644
---
a/infrastructure/ansible/roles/traffic-monitor/templates/traffic_monitor.cfg.j2
+++
b/infrastructure/ansible/roles/traffic-monitor/templates/traffic_monitor.cfg.j2
@@ -18,6 +18,7 @@
"max_events": {{ tm_max_events }},
"health_flush_interval_ms": {{ tm_health_flush_interval_ms }},
"stat_flush_interval_ms": {{ tm_stat_flush_interval_ms }},
+ "log_location_access": "{{ tm_log_location_access }}",
"log_location_event": "{{ tm_log_location_event }}",
"log_location_error": "{{ tm_log_location_error }}",
"log_location_warning": "{{ tm_log_location_warning }}",
diff --git a/infrastructure/cdn-in-a-box/traffic_monitor/traffic_monitor.cfg
b/infrastructure/cdn-in-a-box/traffic_monitor/traffic_monitor.cfg
index c2b1604..d36e7e6 100644
--- a/infrastructure/cdn-in-a-box/traffic_monitor/traffic_monitor.cfg
+++ b/infrastructure/cdn-in-a-box/traffic_monitor/traffic_monitor.cfg
@@ -6,6 +6,7 @@
"max_events": 200,
"health_flush_interval_ms": 20,
"stat_flush_interval_ms": 20,
+ "log_location_access": "$TM_LOG_ACCESS",
"log_location_event": "$TM_LOG_EVENT",
"log_location_error": "$TM_LOG_ERROR",
"log_location_warning": "$TM_LOG_WARNING",
diff --git a/infrastructure/cdn-in-a-box/variables.env
b/infrastructure/cdn-in-a-box/variables.env
index e283f99..e9ef966 100644
--- a/infrastructure/cdn-in-a-box/variables.env
+++ b/infrastructure/cdn-in-a-box/variables.env
@@ -73,6 +73,7 @@ TM_PORT=80
[email protected]
TM_PASSWORD=jhdslvhdfsuklvfhsuvlhs
TM_USER=tmon
+TM_LOG_ACCESS=stdout
TM_LOG_EVENT=stdout
TM_LOG_ERROR=stdout
TM_LOG_WARNING=stdout
diff --git a/lib/go-log/log.go b/lib/go-log/log.go
index 8484472..0b53a1e 100644
--- a/lib/go-log/log.go
+++ b/lib/go-log/log.go
@@ -29,16 +29,18 @@ import (
)
var (
- Debug *log.Logger
- Info *log.Logger
- Warning *log.Logger
- Error *log.Logger
- Event *log.Logger
- debugCloser io.Closer
- infoCloser io.Closer
- warnCloser io.Closer
- errCloser io.Closer
- eventCloser io.Closer
+ Debug *log.Logger
+ Info *log.Logger
+ Warning *log.Logger
+ Error *log.Logger
+ Event *log.Logger
+ Access *log.Logger
+ debugCloser io.Closer
+ infoCloser io.Closer
+ warnCloser io.Closer
+ errCloser io.Closer
+ eventCloser io.Closer
+ accessCloser io.Closer
)
func initLogger(logger **log.Logger, oldLogCloser *io.Closer, newLogWriter
io.WriteCloser, logPrefix string, logFlags int) {
@@ -84,6 +86,10 @@ func Init(eventW, errW, warnW, infoW, debugW io.WriteCloser)
{
initLogger(&Event, &eventCloser, eventW, EventPrefix, EventFlags)
}
+func InitAccess(accessW io.WriteCloser) {
+ initLogger(&Access, &accessCloser, accessW, EventPrefix, EventFlags)
+}
+
// Logf should generally be avoided, use the built-in Init or InitCfg and
Errorf, Warnln, etc functions instead.
// It logs to the given logger, in the same format as the standard log
functions.
// This should only be used in rare and unusual circumstances when the
standard loggers and functions can't.
@@ -122,6 +128,12 @@ func eventTime(t time.Time) float64 {
return float64(t.Unix()) + (float64(t.Nanosecond()) / 1e9)
}
+func Accessln(v ...interface{}) {
+ if Access != nil {
+ Access.Println(v...)
+ }
+}
+
// event log entries (TM event.log, TR access.log, etc)
func Eventf(t time.Time, format string, v ...interface{}) {
if Event == nil {
diff --git a/traffic_monitor/build/traffic_monitor.logrotate
b/traffic_monitor/build/traffic_monitor.logrotate
index b52c013..ae3ca3f 100644
--- a/traffic_monitor/build/traffic_monitor.logrotate
+++ b/traffic_monitor/build/traffic_monitor.logrotate
@@ -34,3 +34,13 @@
rotate 5
copytruncate
}
+
+/opt/traffic_monitor/var/log/access.log {
+ compress
+ maxage 30
+ missingok
+ nomail
+ size 10M
+ rotate 5
+ copytruncate
+}
diff --git a/traffic_monitor/conf/traffic_monitor.cfg
b/traffic_monitor/conf/traffic_monitor.cfg
index 11cc255..8380c2c 100644
--- a/traffic_monitor/conf/traffic_monitor.cfg
+++ b/traffic_monitor/conf/traffic_monitor.cfg
@@ -8,6 +8,7 @@
"distributed_polling": false,
"health_flush_interval_ms": 20,
"stat_flush_interval_ms": 20,
+ "log_location_access": "/opt/traffic_monitor/var/log/access.log",
"log_location_event": "/opt/traffic_monitor/var/log/event.log",
"log_location_error":
"/opt/traffic_monitor/var/log/traffic_monitor.log",
"log_location_warning":
"/opt/traffic_monitor/var/log/traffic_monitor.log",
diff --git a/traffic_monitor/config/config.go b/traffic_monitor/config/config.go
index 3a76fe8..98a4d9b 100644
--- a/traffic_monitor/config/config.go
+++ b/traffic_monitor/config/config.go
@@ -22,6 +22,8 @@ package config
import (
"encoding/json"
"errors"
+ "fmt"
+ "io"
"io/ioutil"
"strings"
"time"
@@ -112,6 +114,7 @@ type Config struct {
LogLocationInfo string `json:"log_location_info"`
LogLocationDebug string `json:"log_location_debug"`
LogLocationEvent string `json:"log_location_event"`
+ LogLocationAccess string
`json:"log_location_access"`
ServeReadTimeout time.Duration `json:"-"`
ServeWriteTimeout time.Duration `json:"-"`
StaticFileDir string `json:"static_file_dir"`
@@ -132,6 +135,17 @@ func (c Config) WarningLog() log.LogLocation { return
log.LogLocation(c.LogLocat
func (c Config) InfoLog() log.LogLocation { return
log.LogLocation(c.LogLocationInfo) }
func (c Config) DebugLog() log.LogLocation { return
log.LogLocation(c.LogLocationDebug) }
func (c Config) EventLog() log.LogLocation { return
log.LogLocation(c.LogLocationEvent) }
+func (c Config) AccessLog() log.LogLocation { return
log.LogLocation(c.LogLocationAccess) }
+
+func GetAccessLogWriter(cfg Config) (io.WriteCloser, error) {
+ accessLoc := cfg.AccessLog()
+
+ accessW, err := log.GetLogWriter(accessLoc)
+ if err != nil {
+ return nil, fmt.Errorf("getting log access writer %v: %v",
accessLoc, err)
+ }
+ return accessW, nil
+}
// DefaultConfig is the default configuration for the application, if no
configuration file is given, or if a given config setting doesn't exist in the
config file.
var DefaultConfig = Config{
@@ -149,6 +163,7 @@ var DefaultConfig = Config{
LogLocationInfo: LogLocationNull,
LogLocationDebug: LogLocationNull,
LogLocationEvent: LogLocationStdout,
+ LogLocationAccess: LogLocationNull,
ServeReadTimeout: 10 * time.Second,
ServeWriteTimeout: 10 * time.Second,
StaticFileDir: StaticFileDir,
diff --git a/traffic_monitor/config/config_test.go
b/traffic_monitor/config/config_test.go
index 935aab3..d2bc667 100644
--- a/traffic_monitor/config/config_test.go
+++ b/traffic_monitor/config/config_test.go
@@ -34,6 +34,7 @@ const exampleTMConfig = `
"stat_flush_interval_ms": 1000,
"stat_polling": false,
"distributed_polling": true,
+ "log_location_access": "access.log",
"log_location_event": "event.log",
"log_location_error": "error.log",
"log_location_warning": "warning.log",
@@ -62,6 +63,7 @@ const exampleBadTMConfig = `
"stat_flush_interval_ms": 1000,
"stat_polling": true,
"distributed_polling": true,
+ "log_location_access": "access.log",
"log_location_event": "event.log",
"log_location_error": "error.log",
"log_location_warning": "warning.log",
diff --git a/traffic_monitor/datareq/datareq.go
b/traffic_monitor/datareq/datareq.go
index c339b4a..42244d6 100644
--- a/traffic_monitor/datareq/datareq.go
+++ b/traffic_monitor/datareq/datareq.go
@@ -31,6 +31,7 @@ import (
"github.com/apache/trafficcontrol/lib/go-log"
"github.com/apache/trafficcontrol/lib/go-rfc"
+ "github.com/apache/trafficcontrol/lib/go-util"
"github.com/apache/trafficcontrol/traffic_monitor/config"
"github.com/apache/trafficcontrol/traffic_monitor/health"
"github.com/apache/trafficcontrol/traffic_monitor/peer"
@@ -286,24 +287,56 @@ func WrapAgeErr(errorCount threadsafe.Uint, f func()
([]byte, time.Time, error),
}
}
+func accessLogTime(t time.Time) float64 {
+ return float64(t.UnixMilli()) / 1000.0
+}
+
+func accessLogStr(
+ timestamp time.Time, // prefix
+ remoteAddress string, // chi
+ reqMethod string, // cqhm
+ reqPath string, // url
+ reqRawQuery string,
+ statusCode int, // pssc
+ respSize int, // b
+ reqServeTimeMs int, // ttms
+ userAgent string, // uas
+) string {
+ return fmt.Sprintf("%.3f chi=%s cqhm=%s url=\"%s?%s\" pssc=%d b=%d
ttms=%d uas=\"%s\"",
+ accessLogTime(timestamp),
+ remoteAddress,
+ reqMethod,
+ reqPath,
+ reqRawQuery,
+ statusCode,
+ respSize,
+ reqServeTimeMs,
+ userAgent)
+}
+
// WrapUnpolledCheck wraps an http.HandlerFunc, returning ServiceUnavailable
if all caches have't been polled; else, calling the wrapped func. Once all
caches have been polled, we never return a 503 again, even if the CRConfig has
been changed and new, unpolled caches exist. This is because, before those new
caches existed in the CRConfig, they weren't being routed to, so it doesn't
break anything to continue not routing to them until they're polled, while
still serving polled caches as [...]
func wrapUnpolledCheck(unpolledCaches threadsafe.UnpolledCaches, errorCount
threadsafe.Uint, f http.HandlerFunc) http.HandlerFunc {
polledAll := false
polledLocal := false
return func(w http.ResponseWriter, r *http.Request) {
+ start := time.Now()
+ iw := &util.Interceptor{W: w}
+ defer func() {
+ log.Accessln(accessLogStr(time.Now(), r.RemoteAddr,
r.Method, r.URL.Path, r.URL.RawQuery, iw.Code, iw.ByteCount,
int(time.Now().Sub(start)/time.Millisecond), r.UserAgent()))
+ }()
if !polledAll || !polledLocal {
polledAll = !unpolledCaches.Any()
polledLocal = !unpolledCaches.AnyDirectlyPolled()
rawOrLocal := r.URL.Query().Has("raw") ||
r.URL.Query().Has("local")
if (!rawOrLocal && !polledAll) || (rawOrLocal &&
!polledLocal) {
HandleErr(errorCount, r.URL.EscapedPath(),
fmt.Errorf("service still starting, some caches unpolled: %v",
unpolledCaches.UnpolledCaches()))
- w.WriteHeader(http.StatusServiceUnavailable)
- log.Write(w, []byte("Service Unavailable"),
r.URL.EscapedPath())
+ iw.WriteHeader(http.StatusServiceUnavailable)
+ log.Write(iw, []byte("Service Unavailable"),
r.URL.EscapedPath())
return
}
}
- w.Header().Set(rfc.PermissionsPolicy, "interest-cohort=()")
- f(w, r)
+ iw.Header().Set(rfc.PermissionsPolicy, "interest-cohort=()")
+ f(iw, r)
}
}
diff --git a/traffic_monitor/tests/_integration/tm/Dockerfile_run.sh
b/traffic_monitor/tests/_integration/tm/Dockerfile_run.sh
index b762f99..e895b48 100755
--- a/traffic_monitor/tests/_integration/tm/Dockerfile_run.sh
+++ b/traffic_monitor/tests/_integration/tm/Dockerfile_run.sh
@@ -45,6 +45,7 @@ init() {
"max_events": 200,
"health_flush_interval_ms": 20,
"stat_flush_interval_ms": 20,
+ "log_location_access":
"/opt/traffic_monitor/var/log/access.log",
"log_location_event":
"/opt/traffic_monitor/var/log/event.log",
"log_location_error":
"/opt/traffic_monitor/var/log/traffic_monitor.log",
"log_location_warning":
"/opt/traffic_monitor/var/log/traffic_monitor.log",
diff --git a/traffic_monitor/traffic_monitor.go
b/traffic_monitor/traffic_monitor.go
index 23d5c18..5d31bc5 100644
--- a/traffic_monitor/traffic_monitor.go
+++ b/traffic_monitor/traffic_monitor.go
@@ -38,6 +38,15 @@ var GitRevision = "No Git Revision Specified. Please build
with '-X main.GitRevi
// BuildTimestamp is the time the app was built. The app SHOULD always be
built with this set via the `-X` flag.
var BuildTimestamp = "No Build Timestamp Specified. Please build with '-X
main.BuildTimestamp=`date +'%Y-%M-%dT%H:%M:%S'`"
+func InitAccessCfg(cfg config.Config) error {
+ accessW, err := config.GetAccessLogWriter(cfg)
+ if err != nil {
+ return err
+ }
+ log.InitAccess(accessW)
+ return nil
+}
+
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
@@ -68,6 +77,11 @@ func main() {
os.Exit(1)
}
+ if err := InitAccessCfg(cfg); err != nil {
+ fmt.Printf("Error starting service: failed to create access log
writer: %v\n", err)
+ os.Exit(1)
+ }
+
if cfg.ShortHostnameOverride != "" {
staticData.Hostname = cfg.ShortHostnameOverride
}
