This is an automated email from the ASF dual-hosted git repository.
liuhan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-rover.git
The following commit(s) were added to refs/heads/main by this push:
new 159e9cd Add Network profiling documentation (#49)
159e9cd is described below
commit 159e9cdc211bb3846c93e6fdb244a898ae86b9f6
Author: mrproliu <[email protected]>
AuthorDate: Mon Aug 29 17:43:15 2022 +0800
Add Network profiling documentation (#49)
---
docs/en/setup/configuration/profiling.md | 45 ++++++++++++++++++++++++++++++++
pkg/profiling/task/network/analyzer.go | 14 +++++-----
pkg/profiling/task/network/context.go | 6 ++---
pkg/profiling/task/network/metrics.go | 45 +++++++++++++++++++++++---------
pkg/profiling/task/network/runner.go | 2 +-
5 files changed, 89 insertions(+), 23 deletions(-)
diff --git a/docs/en/setup/configuration/profiling.md
b/docs/en/setup/configuration/profiling.md
index 8513b31..78d6fa6 100644
--- a/docs/en/setup/configuration/profiling.md
+++ b/docs/en/setup/configuration/profiling.md
@@ -39,6 +39,51 @@ Also, the following protocol are supported for analyzing
using OpenSSL library,
5. Kafka
6. DNS
+#### Metrics
+
+Network profiling uses metrics send data to the backend service.
+
+##### Data Type
+
+The network profiling has customized the following two types of metrics to
represent the network data:
+1. **Counter**: Records the total number of data in a certain period of time.
Each counter containers the following data:
+ 1. **Count**: The count of the execution.
+ 2. **Bytes**: The package size of the execution.
+ 3. **Exe Time**: The consumed time(nanosecond) of the execution.
+2. **Histogram**: Records the distribution of the data in the bucket.
+
+##### Labels
+
+Each metric contains the following labels to identify the process relationship:
+
+| Name | Type | Description |
+|------|------|-------------|
+|client_process_id or server_process_id| string | The ID of the current
process, which is determined by the role of the current process in the
connection as server or client. |
+|client_local or server_local| boolean | The remote process is a local
process. |
+|client_address or server_address| string | The remote process address. ex:
`IP:port`. |
+|side| enum | The current process is either "client" or "server" in this
connection. |
+|protocol| string | Identification the protocol based on the package data
content. |
+|is_ssl| bool | Is the current connection using SSL. |
+
+##### Data
+
+Based on the above two data types, the following metrics are provided.
+
+| Name | Type| Unit | Description |
+|------|-----|------|-------------|
+|write|Counter|nanosecond|The socket write counter|
+|read|Counter|nanosecond|The socket read counter|
+|write RTT|Counter|microsecond|The socket write RTT counter|
+|connect|Counter|nanosecond|The socket connect/accept with other server/client
counter|
+|close|Counter|nanosecond|The socket close counter|
+|retransmit|Counter|nanosecond|The socket retransmit package counter|
+|drop|Counter|nanosecond|The socket drop package counter|
+|write RTT|Histogram|microsecond|The socket write RTT execute time histogram|
+|write execute time|Histogram|nanosecond|The socket write data execute time
histogram|
+|read execute time|Histogram|nanosecond|The socket read data execute time
histogram|
+|connect execute time|Histogram|nanosecond|The socket connect/accept with
other server/client execute time histogram|
+|close execute time|Histogram|nanosecond|The socket close execute time
histogram|
+
## Configuration
| Name | Default | Environment Key | Description |
diff --git a/pkg/profiling/task/network/analyzer.go
b/pkg/profiling/task/network/analyzer.go
index 581da98..8eb69bc 100644
--- a/pkg/profiling/task/network/analyzer.go
+++ b/pkg/profiling/task/network/analyzer.go
@@ -179,11 +179,11 @@ func (t *TrafficAnalyzer)
generateOrCombineTraffic(traffic *ProcessTraffic, con
CloseCounter: NewSocketDataCounter(),
RetransmitCounter: NewSocketDataCounter(),
DropCounter: NewSocketDataCounter(),
- WriteRTTHistogram: NewSocketDataHistogram(),
- WriteExeTimeHistogram: NewSocketDataHistogram(),
- ReadExeTimeHistogram: NewSocketDataHistogram(),
- ConnectExeTimeHistogram: NewSocketDataHistogram(),
- CloseExeTimeHistogram: NewSocketDataHistogram(),
+ WriteRTTHistogram:
NewSocketDataHistogram(HistogramDataUnitUS),
+ WriteExeTimeHistogram:
NewSocketDataHistogram(HistogramDataUnitNS),
+ ReadExeTimeHistogram:
NewSocketDataHistogram(HistogramDataUnitNS),
+ ConnectExeTimeHistogram:
NewSocketDataHistogram(HistogramDataUnitNS),
+ CloseExeTimeHistogram:
NewSocketDataHistogram(HistogramDataUnitNS),
}
}
if len(traffic.LocalProcesses) == 0 && len(con.LocalProcesses) > 0 {
@@ -401,8 +401,8 @@ func (t *TrafficAnalyzer)
findRemotePidWhenMeshEnvironment(con *ConnectionContex
}
continue
}
- // if current is mesh application, and remote address is not
local and dns, them it's must be sent to the MESH_DP
- if localProcess.Entity().Layer == layerMeshApp &&
con.RemotePort != 53 &&
+ // if current is mesh application, them it's must be sent to
the MESH_DP
+ if localProcess.Entity().Layer == layerMeshApp &&
len(t.localAddresses[con.RemoteIP]) == 0 &&
!tools.IsLocalHostAddress(con.RemoteIP) {
if envoyPid :=
t.findSameInstanceMeshDP(localProcess.Entity()); envoyPid != 0 {
log.Debugf("found in the mesh data plane,
remote ip: %s", con.RemoteIP)
diff --git a/pkg/profiling/task/network/context.go
b/pkg/profiling/task/network/context.go
index 644d220..42e7364 100644
--- a/pkg/profiling/task/network/context.go
+++ b/pkg/profiling/task/network/context.go
@@ -535,9 +535,9 @@ func (c *Context) newConnectionContext(conID, randomID
uint64, pid, fd uint32, p
WriteCounter: NewSocketDataCounterWithHistory(),
ReadCounter: NewSocketDataCounterWithHistory(),
WriteRTTCounter: NewSocketDataCounterWithHistory(),
- WriteRTTHistogram: NewSocketDataHistogramWithHistory(),
- WriteExeTimeHistogram: NewSocketDataHistogramWithHistory(),
- ReadExeTimeHistogram: NewSocketDataHistogramWithHistory(),
+ WriteRTTHistogram:
NewSocketDataHistogramWithHistory(HistogramDataUnitUS),
+ WriteExeTimeHistogram:
NewSocketDataHistogramWithHistory(HistogramDataUnitNS),
+ ReadExeTimeHistogram:
NewSocketDataHistogramWithHistory(HistogramDataUnitNS),
}
}
diff --git a/pkg/profiling/task/network/metrics.go
b/pkg/profiling/task/network/metrics.go
index 388be01..a52d5e3 100644
--- a/pkg/profiling/task/network/metrics.go
+++ b/pkg/profiling/task/network/metrics.go
@@ -81,17 +81,24 @@ func (c *SocketDataCounterWithHistory) CalculateIncrease()
*SocketDataCounter {
}
}
-// SocketHistogramBuckets means the histogram bucket: 0ms, 0.01ms, 0.05ms,
0.1ms, 0.5ms, 1ms, 1.2ms, 1.5ms, 1.7ms, 2ms,
+// SocketHistogramBucketsNs means the histogram bucket: 0ms, 0.01ms, 0.05ms,
0.1ms, 0.5ms, 1ms, 1.2ms, 1.5ms, 1.7ms, 2ms,
// 2.5ms, 3ms, 5ms, 7ms, 10ms, 13ms, 16ms, 20ms, 25ms, 30ms, 35ms, 40ms, 45ms,
50ms, 70ms, 100ms, 150ms,
// 200ms, 300ms, 500ms, 1s, 2s, 3s, 5s
-// value unit: us
-var SocketHistogramBuckets = []float64{0, 10, 50, 100, 500, 1000, 1200, 1500,
1700, 2000,
+// value unit: ns
+var SocketHistogramBucketsNs = []float64{0, 10000, 50000, 100000, 500000,
1000000, 1200000, 1500000, 1700000, 2000000,
+ 2500000, 3000000, 5000000, 7000000, 10000000, 13000000, 16000000,
20000000, 25000000, 30000000, 35000000, 40000000,
+ 45000000, 50000000, 70000000, 100000000, 150000000, 200000000,
300000000, 500000000, 1000000000, 2000000000,
+ 3000000000, 5000000000}
+
+// SocketHistogramBucketsUs same with SocketHistogramBucketsNs, but the value
unit: us
+var SocketHistogramBucketsUs = []float64{0, 10, 50, 100, 500, 1000, 1200,
1500, 1700, 2000,
2500, 3000, 5000, 7000, 10000, 13000, 16000, 20000, 25000, 30000,
35000, 40000,
45000, 50000, 70000, 100000, 150000, 200000, 300000, 500000, 1000000,
2000000,
3000000, 5000000}
-var SocketHistogramBucketsCount = len(SocketHistogramBuckets)
+var SocketHistogramBucketsCount = len(SocketHistogramBucketsNs)
type SocketDataHistogram struct {
+ Unit HistogramDataUnit
Buckets map[uint64]uint32
}
@@ -113,13 +120,13 @@ func (h *SocketDataHistogram) Increase(other
*SocketDataHistogram) {
func (h *SocketDataHistogram) IncreaseByValue(val uint64) {
floatVal := float64(val)
- for inx, curVal := range SocketHistogramBuckets {
+ for inx, curVal := range SocketHistogramBucketsNs {
if inx > 0 && curVal > floatVal {
h.Buckets[uint64(inx-1)]++
return
}
}
- h.Buckets[uint64(len(SocketHistogramBuckets)-1)]++
+ h.Buckets[uint64(len(SocketHistogramBucketsNs)-1)]++
}
func (h *SocketDataHistogram) NotEmpty() bool {
@@ -131,25 +138,33 @@ func (h *SocketDataHistogram) NotEmpty() bool {
return false
}
-func NewSocketDataHistogram() *SocketDataHistogram {
+func NewSocketDataHistogram(unit HistogramDataUnit) *SocketDataHistogram {
buckets := make(map[uint64]uint32, SocketHistogramBucketsCount)
for i := 0; i < SocketHistogramBucketsCount; i++ {
buckets[uint64(i)] = 0
}
return &SocketDataHistogram{
+ Unit: unit,
Buckets: buckets,
}
}
+type HistogramDataUnit int
+
+const (
+ HistogramDataUnitNS HistogramDataUnit = 1
+ HistogramDataUnitUS HistogramDataUnit = 2
+)
+
type SocketDataHistogramWithHistory struct {
Pre *SocketDataHistogram
Cur *SocketDataHistogram
}
-func NewSocketDataHistogramWithHistory() *SocketDataHistogramWithHistory {
+func NewSocketDataHistogramWithHistory(unit HistogramDataUnit)
*SocketDataHistogramWithHistory {
return &SocketDataHistogramWithHistory{
- Pre: NewSocketDataHistogram(),
- Cur: NewSocketDataHistogram(),
+ Pre: NewSocketDataHistogram(unit),
+ Cur: NewSocketDataHistogram(unit),
}
}
@@ -163,7 +178,7 @@ func (h *SocketDataHistogramWithHistory)
UpdateToCurrent(bucket uint64, val uint
}
func (h *SocketDataHistogramWithHistory) CalculateIncrease()
*SocketDataHistogram {
- histogram := NewSocketDataHistogram()
+ histogram := NewSocketDataHistogram(h.Cur.Unit)
var increaseVal uint32
for curK, curV := range h.Cur.Buckets {
if increaseVal = curV - h.Pre.Buckets[curK]; increaseVal > 0 {
@@ -298,8 +313,14 @@ func (r *ProcessTraffic) appendHistogramValue(metrics
[]*v3.MeterData, metricsPr
if bucketInx >= SocketHistogramBucketsCount {
bucketInx = SocketHistogramBucketsCount - 1
}
+ var buckets []float64
+ if histogram.Unit == HistogramDataUnitUS {
+ buckets = SocketHistogramBucketsUs
+ } else {
+ buckets = SocketHistogramBucketsNs
+ }
values = append(values, &v3.MeterBucketValue{
- Bucket: SocketHistogramBuckets[bucketInx],
+ Bucket: buckets[bucketInx],
Count: int64(count),
})
}
diff --git a/pkg/profiling/task/network/runner.go
b/pkg/profiling/task/network/runner.go
index f085ffd..c623bfd 100644
--- a/pkg/profiling/task/network/runner.go
+++ b/pkg/profiling/task/network/runner.go
@@ -237,7 +237,7 @@ func (r *Runner) logTheMetricsConnections(traffices
[]*ProcessTraffic) {
traffic.RemoteIP, traffic.RemotePort,
traffic.RemotePid)
}
side := traffic.ConnectionRole.String()
- log.Debugf("connection analyze result: %s : %s -> %s, protocol:
%s, is SSL: %t, read: %d bytes/%d, write: %d bytes/%d",
+ log.Debugf("connection analyze result: %s : %s -> %s, protocol:
%s, is SSL: %t, write: %d bytes/%d, read: %d bytes/%d",
side, localInfo, remoteInfo, traffic.Protocol.String(),
traffic.IsSSL, traffic.WriteCounter.Bytes, traffic.WriteCounter.Count,
traffic.ReadCounter.Bytes, traffic.ReadCounter.Count)
}
