This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
The following commit(s) were added to refs/heads/main by this push:
new e91924c2 Run stress test with the new column-based measure (#369)
e91924c2 is described below
commit e91924c2314209844c60b45f9293cd888c10796f
Author: Gao Hongtao <[email protected]>
AuthorDate: Fri Jan 12 09:11:43 2024 +0800
Run stress test with the new column-based measure (#369)
Signed-off-by: Gao Hongtao <[email protected]>
---
.gitignore | 1 +
banyand/observability/meter_prom.go | 5 +-
dist/LICENSE | 1 +
.../license-github.com-montanaflynn-stats.txt | 21 +++
go.mod | 1 +
go.sum | 2 +
pkg/test/helpers/system.go | 69 ++++++++
pkg/test/metric/analysis.go | 182 +++++++++++++++++++++
pkg/test/metric/collection.go | 143 ++++++++++++++++
pkg/test/metric/metric.go | 91 +++++++++++
.gitignore => test/stress/cases/istio/Makefile | 59 +++----
test/stress/cases/istio/cpu.prof | 0
test/stress/cases/istio/heap.prof | 0
test/stress/cases/istio/istio_suite_test.go | 53 ++----
test/stress/cases/istio/report.md | 144 ++++++++--------
test/stress/cases/istio/testdata/metrics/data.csv | 157 ++++++++++++++++++
.../cases/istio/testdata/metrics/intermediate.csv | 9 +
.../stress/cases/istio/testdata/metrics/result.csv | 16 ++
18 files changed, 801 insertions(+), 153 deletions(-)
diff --git a/.gitignore b/.gitignore
index 6b001b0e..6c93a6d8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,6 +29,7 @@ target
# Test binary, build with `go test -c`
*.test
+*.log
# Ginkgo test report
*.report
diff --git a/banyand/observability/meter_prom.go
b/banyand/observability/meter_prom.go
index bd861b4e..f8674ab9 100644
--- a/banyand/observability/meter_prom.go
+++ b/banyand/observability/meter_prom.go
@@ -25,6 +25,7 @@ import (
grpcprom
"github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/apache/skywalking-banyandb/pkg/meter"
@@ -40,8 +41,8 @@ var (
)
func init() {
- reg.MustRegister(prometheus.NewGoCollector())
-
reg.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
+ reg.MustRegister(collectors.NewGoCollector())
+
reg.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
mux.Handle("/metrics", promhttp.HandlerFor(
reg,
promhttp.HandlerOpts{},
diff --git a/dist/LICENSE b/dist/LICENSE
index 8c336175..43c34b3a 100644
--- a/dist/LICENSE
+++ b/dist/LICENSE
@@ -332,6 +332,7 @@ MIT licenses
github.com/mattn/go-colorable v0.1.13 MIT
github.com/mattn/go-isatty v0.0.20 MIT
github.com/mitchellh/mapstructure v1.5.0 MIT
+ github.com/montanaflynn/stats v0.7.1 MIT
github.com/onsi/ginkgo/v2 v2.13.2 MIT
github.com/onsi/gomega v1.30.0 MIT
github.com/pelletier/go-toml/v2 v2.1.1 MIT
diff --git a/dist/licenses/license-github.com-montanaflynn-stats.txt
b/dist/licenses/license-github.com-montanaflynn-stats.txt
new file mode 100644
index 00000000..3162cb1a
--- /dev/null
+++ b/dist/licenses/license-github.com-montanaflynn-stats.txt
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014-2023 Montana Flynn (https://montanaflynn.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/go.mod b/go.mod
index 4a9f0d19..dabad54e 100644
--- a/go.mod
+++ b/go.mod
@@ -97,6 +97,7 @@ require (
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd //
indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
+ github.com/montanaflynn/stats v0.7.1
github.com/mschoch/smat v0.2.0 // indirect
github.com/pelletier/go-toml/v2 v2.1.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 //
indirect
diff --git a/go.sum b/go.sum
index eacdceeb..851a20be 100644
--- a/go.sum
+++ b/go.sum
@@ -240,6 +240,8 @@ github.com/modern-go/concurrent
v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod
h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2
h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod
h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/montanaflynn/stats v0.7.1
h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
+github.com/montanaflynn/stats v0.7.1/go.mod
h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
github.com/mschoch/smat v0.2.0/go.mod
h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
diff --git a/pkg/test/helpers/system.go b/pkg/test/helpers/system.go
new file mode 100644
index 00000000..746e2d39
--- /dev/null
+++ b/pkg/test/helpers/system.go
@@ -0,0 +1,69 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package helpers
+
+import (
+ "fmt"
+
+ "github.com/shirou/gopsutil/v3/cpu"
+ "github.com/shirou/gopsutil/v3/host"
+ "github.com/shirou/gopsutil/v3/mem"
+)
+
+// CPUInfo returns the CPU information of the current machine.
+func CPUInfo() string {
+ info, err := cpu.Info()
+ if err != nil {
+ return fmt.Sprintf("failed to get CPU info: %v", err)
+ }
+
+ var result string
+ for _, cpuInfo := range info {
+ result += fmt.Sprintf("CPU: %v, VendorID: %v, Family: %v,
Model: %v, Cores: %v\n",
+ cpuInfo.ModelName, cpuInfo.VendorID, cpuInfo.Family,
cpuInfo.Model, cpuInfo.Cores)
+ }
+ return result
+}
+
+// MemoryInfo returns the memory information of the current machine.
+func MemoryInfo() string {
+ v, err := mem.VirtualMemory()
+ if err != nil {
+ return fmt.Sprintf("failed to get memory info: %v", err)
+ }
+
+ // convert to MB
+ total := v.Total / 1024 / 1024
+ available := v.Available / 1024 / 1024
+ used := v.Used / 1024 / 1024
+
+ result := fmt.Sprintf("Total: %v MB, Available: %v MB, Used: %v MB\n",
total, available, used)
+ return result
+}
+
+// OSInfo returns the OS information of the current machine.
+func OSInfo() string {
+ info, err := host.Info()
+ if err != nil {
+ return fmt.Sprintf("failed to get OS info: %v", err)
+ }
+
+ result := fmt.Sprintf("OS: %v, Platform: %v, Platform Family: %v,
Platform Version: %v\n",
+ info.OS, info.Platform, info.PlatformFamily,
info.PlatformVersion)
+ return result
+}
diff --git a/pkg/test/metric/analysis.go b/pkg/test/metric/analysis.go
new file mode 100644
index 00000000..e94f5d33
--- /dev/null
+++ b/pkg/test/metric/analysis.go
@@ -0,0 +1,182 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package metric
+
+import (
+ "encoding/csv"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strconv"
+ "time"
+
+ "github.com/montanaflynn/stats"
+)
+
+func analyzeMetrics() {
+ file, err := os.Open(filepath.Join(rootPath, metricsFile))
+ if err != nil {
+ fmt.Println("Error opening metrics file:", err)
+ return
+ }
+ defer file.Close()
+
+ reader := csv.NewReader(file)
+ records, err := reader.ReadAll()
+ if err != nil {
+ fmt.Println("Error reading metrics file:", err)
+ return
+ }
+ if reader == nil {
+ fmt.Println("No records found in metrics file.")
+ return
+ }
+
+ // Transpose the records to handle column-based data.
+ transposed := transpose(records)
+
+ // Open the intermediate file.
+ intermediateFile, err := os.OpenFile(filepath.Join(rootPath,
intermediateFile), os.O_CREATE|os.O_WRONLY, 0o644)
+ if err != nil {
+ fmt.Println("Error opening intermediate file:", err)
+ return
+ }
+ defer intermediateFile.Close()
+ if err = intermediateFile.Truncate(0); err != nil {
+ fmt.Println("Error truncating intermediate file:", err)
+ return
+ }
+
+ // Open the results file.
+ resultsFile, err := os.OpenFile(filepath.Join(rootPath, resultFile),
os.O_CREATE|os.O_WRONLY, 0o644)
+ if err != nil {
+ fmt.Println("Error opening results file:", err)
+ return
+ }
+ defer resultsFile.Close()
+ if err = resultsFile.Truncate(0); err != nil {
+ fmt.Println("Error truncating results file:", err)
+ return
+ }
+ // Write the header to the results file.
+ writeHeader(resultsFile)
+
+ for i, record := range transposed {
+ // Convert the records to a slice of floats for analysis.
+ data := make([]float64, len(record))
+ for j, r := range record {
+ data[j] = atof(r) // Convert the string to a float.
+ }
+ if _, ok := rateMetrics[metricsToCollect[i]]; ok {
+ // Calculate the rate of the metric.
+ data, err = rate(data, 5*time.Minute)
+ if err != nil {
+ fmt.Println("Error calculating rate:", err)
+ return
+ }
+ // Write the rates to the intermediate file.
+ intermediateWriter := csv.NewWriter(intermediateFile)
+ defer intermediateWriter.Flush()
+ row := []string{metricsToCollect[i], "rate[5m]"}
+ for _, r := range data {
+ row = append(row, strconv.FormatFloat(r, 'f',
-1, 64))
+ }
+ if err = intermediateWriter.Write(row); err != nil {
+ fmt.Println("Error writing to intermediate
file:", err)
+ return
+ }
+ }
+
+ // Calculate the statistics.
+ min, _ := stats.Min(data)
+ max, _ := stats.Max(data)
+ mean, _ := stats.Mean(data)
+ median, _ := stats.Median(data)
+ p90, _ := stats.Percentile(data, 90)
+ p95, _ := stats.Percentile(data, 95)
+ p98, _ := stats.Percentile(data, 98)
+ p99, _ := stats.Percentile(data, 99)
+
+ // Write the results to another file and print them to the
console.
+ writeResults(resultsFile, metricsToCollect[i], min, max, mean,
median, p90, p95, p98, p99)
+ }
+}
+
+func writeHeader(file *os.File) {
+ header := "Metric Name, Min, Max, Mean, Median, P90, P95, P98, P99\n"
+ _, err := file.WriteString(header)
+ if err != nil {
+ fmt.Println("Error writing to results file:", err)
+ return
+ }
+}
+
+func writeResults(file *os.File, metricName string, min, max, mean, median,
p90, p95, p98, p99 float64) {
+ results := fmt.Sprintf("%s, %f, %f, %f, %f, %f, %f, %f, %f\n",
+ metricName, min, max, mean, median, p90, p95, p98, p99)
+
+ _, err := file.WriteString(results)
+ if err != nil {
+ fmt.Println("Error writing to results file:", err)
+ return
+ }
+
+ fmt.Print(results)
+}
+
+func transpose(slice [][]string) [][]string {
+ xl := len(slice[0])
+ yl := len(slice)
+ result := make([][]string, xl)
+ for i := range result {
+ result[i] = make([]string, yl)
+ }
+ for i, row := range slice {
+ for j, col := range row {
+ result[j][i] = col
+ }
+ }
+ return result
+}
+
+func atof(s string) float64 {
+ value, err := strconv.ParseFloat(s, 64)
+ if err != nil {
+ panic(err)
+ }
+ return value
+}
+
+const interval = 15 * time.Second
+
+func rate(metricData []float64, timeWindow time.Duration) ([]float64, error) {
+ pointsInWindow := int(timeWindow.Seconds() / interval.Seconds())
+
+ if len(metricData) < pointsInWindow {
+ return nil, fmt.Errorf("not enough data points to calculate
rate")
+ }
+
+ rates := make([]float64, len(metricData)-pointsInWindow+1)
+ for i := 0; i <= len(metricData)-pointsInWindow; i++ {
+ start := metricData[i]
+ end := metricData[i+pointsInWindow-1]
+ rates[i] = (end - start) /
float64(pointsInWindow*int(interval.Seconds()))
+ }
+
+ return rates, nil
+}
diff --git a/pkg/test/metric/collection.go b/pkg/test/metric/collection.go
new file mode 100644
index 00000000..30941d24
--- /dev/null
+++ b/pkg/test/metric/collection.go
@@ -0,0 +1,143 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package metric
+
+import (
+ "encoding/csv"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "path/filepath"
+ "slices"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/shirou/gopsutil/v3/disk"
+)
+
+func collectMetrics(path string, closeCh <-chan struct{}) {
+ file, err := os.OpenFile(filepath.Join(rootPath, metricsFile),
os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
+ if err != nil {
+ panic(err)
+ }
+ defer file.Close()
+ if err = file.Truncate(0); err != nil {
+ panic(err)
+ }
+ for {
+ metrics, err := scrape()
+ if err != nil {
+ fmt.Println("Error scraping metrics:", err)
+ select {
+ case <-closeCh:
+ return
+ case <-time.After(5 * time.Second):
+ }
+ continue
+ }
+ usage, err := disk.Usage(path)
+ if err != nil {
+ fmt.Println("Error getting disk usage:", err)
+ } else {
+ metrics[diskUsed] = float64(usage.Used)
+ metrics[diskTotal] = float64(usage.Total)
+ }
+ ioMetrics, err := disk.IOCounters("sda")
+ if err != nil {
+ fmt.Println("Error getting disk IO:", err)
+ } else {
+ metrics[readCount] = float64(ioMetrics["sda"].ReadCount)
+ metrics[mergedReadCount] =
float64(ioMetrics["sda"].MergedReadCount)
+ metrics[writeCount] =
float64(ioMetrics["sda"].WriteCount)
+ metrics[mergedWriteCount] =
float64(ioMetrics["sda"].MergedWriteCount)
+ metrics[readBytes] = float64(ioMetrics["sda"].ReadBytes)
+ metrics[writeBytes] =
float64(ioMetrics["sda"].WriteBytes)
+ metrics[ioTime] = float64(ioMetrics["sda"].IoTime)
+ metrics[weightedIO] =
float64(ioMetrics["sda"].WeightedIO)
+ }
+
+ select {
+ case <-closeCh:
+ return
+ default:
+ }
+
+ err = writeMetrics(file, metricsToCollect, metrics)
+ if err != nil {
+ fmt.Println("Error writing metrics:", err)
+ }
+ select {
+ case <-closeCh:
+ return
+ case <-time.After(15 * time.Second):
+ }
+ }
+}
+
+func scrape() (map[string]float64, error) {
+ resp, err := http.Get("http://localhost:2121/metrics";)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, err
+ }
+ return parseMetrics(string(body)), nil
+}
+
+func writeMetrics(file *os.File, header []string, metrics map[string]float64)
error {
+ writer := csv.NewWriter(file)
+ defer writer.Flush()
+
+ record := make([]string, len(header))
+ for i, h := range header {
+ record[i] = fmt.Sprintf("%f", metrics[h])
+ }
+
+ return writer.Write(record)
+}
+
+func parseMetrics(metrics string) map[string]float64 {
+ lines := strings.Split(metrics, "\n")
+ r := make(map[string]float64)
+ for _, line := range lines {
+ if strings.Contains(line, "#") {
+ continue
+ }
+ n, v := parseMetricValue(line)
+ if n == "" {
+ continue
+ }
+ r[n] = v
+ }
+ return r
+}
+
+func parseMetricValue(line string) (string, float64) {
+ parts := strings.Split(line, " ")
+ if !slices.Contains[[]string](metricsToCollect, parts[0]) {
+ return "", 0
+ }
+ value, _ := strconv.ParseFloat(parts[1], 64)
+ return parts[0], value
+}
diff --git a/pkg/test/metric/metric.go b/pkg/test/metric/metric.go
new file mode 100644
index 00000000..ccd11352
--- /dev/null
+++ b/pkg/test/metric/metric.go
@@ -0,0 +1,91 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Package metric provides functions for collecting and analyzing metrics for
stress testing.
+package metric
+
+import (
+ "os"
+ "time"
+)
+
+const (
+ rootPath = "testdata/metrics"
+ metricsFile = "data.csv"
+ resultFile = "result.csv"
+ intermediateFile = "intermediate.csv"
+
+ // disk usage metrics.
+ diskUsed = "disk_used"
+ diskTotal = "disk_total"
+
+ // io metrics.
+ readCount = "read_count"
+ mergedReadCount = "merged_read_count"
+ writeCount = "write_count"
+ mergedWriteCount = "merged_write_count"
+ readBytes = "read_bytes"
+ writeBytes = "write_bytes"
+ ioTime = "io_time"
+ weightedIO = "weighted_io"
+)
+
+var (
+ metricsToCollect []string
+ rateMetrics map[string]struct{}
+)
+
+var (
+ memStats = []string{"go_memstats_alloc_bytes",
"go_memstats_heap_inuse_bytes", "go_memstats_sys_bytes",
"go_memstats_stack_inuse_bytes"}
+ cpuStats = []string{"process_cpu_seconds_total"}
+ diskStats = []string{diskUsed, diskTotal}
+ ioStats = []string{readCount, mergedReadCount, writeCount,
mergedWriteCount, readBytes, writeBytes, ioTime, weightedIO}
+)
+
+func init() {
+ if err := os.MkdirAll(rootPath, 0o755); err != nil {
+ panic(err)
+ }
+ metricsToCollect = append(metricsToCollect, memStats...)
+ metricsToCollect = append(metricsToCollect, cpuStats...)
+ metricsToCollect = append(metricsToCollect, diskStats...)
+ metricsToCollect = append(metricsToCollect, ioStats...)
+ rateMetrics = make(map[string]struct{})
+ for _, v := range cpuStats {
+ rateMetrics[v] = struct{}{}
+ }
+ for _, v := range ioStats {
+ rateMetrics[v] = struct{}{}
+ }
+}
+
+// Start starts the metric collection.
+func Start(path string) chan struct{} {
+ closeCh := make(chan struct{})
+ go func() {
+ select {
+ case <-closeCh:
+ return
+ case <-time.After(10 * time.Second):
+ }
+ defer func() {
+ analyzeMetrics()
+ }()
+ collectMetrics(path, closeCh)
+ }()
+ return closeCh
+}
diff --git a/.gitignore b/test/stress/cases/istio/Makefile
similarity index 57%
copy from .gitignore
copy to test/stress/cases/istio/Makefile
index 6b001b0e..5d2a9517 100644
--- a/.gitignore
+++ b/test/stress/cases/istio/Makefile
@@ -14,48 +14,27 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
-# Binaries for programs and plugins
-*.exe
-*.exe~
-*.dll
-*.so
-*.dylib
-bin
-include
-/build
-target
-/tmp
-
-# Test binary, build with `go test -c`
-*.test
-
-# Ginkgo test report
-*.report
-
-# Output of the go coverage tool, specifically when used with LiteIDE
-*.out
-
-# editor and IDE paraphernalia
-.idea
-*.swp
-*.swo
-*~
-.vscode
-
-.DS_Store
-.env
-.run
+istio_mk_path := $(abspath $(lastword $(MAKEFILE_LIST)))
+istio_mk_dir := $(dir $(istio_mk_path))
-# mock files
-*mock.go
-*mock_test.go
+root_dir := $(istio_mk_dir)../../../..
-# snky cache
-.dccache
+include $(root_dir)/scripts/build/version.mk
+include $(root_dir)/scripts/build/base.mk
+include $(root_dir)/scripts/build/ginkgo.mk
-# okteto
-.stignore
+.PHONY: profile
+profile: $(GINKGO)
+ $(GINKGO) -v --tags="prometheus" \
+ --blockprofile prof/block.prof \
+ --cpuprofile prof/cpu.prof \
+ --memprofile prof/mem.prof \
+ --mutexprofile prof/mutex.prof \
+ ./... 2 > $(root_dir)/test.log
-# profile result
-*.prof
+.PHONY: test
+test: $(GINKGO)
+ $(GINKGO) -v --tags="prometheus" \
+ ./... 2 > $(root_dir)/test.log
\ No newline at end of file
diff --git a/test/stress/cases/istio/cpu.prof b/test/stress/cases/istio/cpu.prof
deleted file mode 100644
index e69de29b..00000000
diff --git a/test/stress/cases/istio/heap.prof
b/test/stress/cases/istio/heap.prof
deleted file mode 100644
index e69de29b..00000000
diff --git a/test/stress/cases/istio/istio_suite_test.go
b/test/stress/cases/istio/istio_suite_test.go
index 9b3d4cb0..a191a2fc 100644
--- a/test/stress/cases/istio/istio_suite_test.go
+++ b/test/stress/cases/istio/istio_suite_test.go
@@ -25,7 +25,6 @@ import (
"io"
"os"
"path/filepath"
- "runtime/pprof"
"testing"
"time"
@@ -45,6 +44,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/test"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
"github.com/apache/skywalking-banyandb/pkg/test/helpers"
+ "github.com/apache/skywalking-banyandb/pkg/test/metric"
"github.com/apache/skywalking-banyandb/pkg/test/setup"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
)
@@ -54,41 +54,6 @@ func TestIstio(t *testing.T) {
g.RunSpecs(t, "Istio Suite", g.Label("integration", "slow"))
}
-var (
- cpuProfileFile *os.File
- heapProfileFile *os.File
-)
-
-var _ = g.BeforeSuite(func() {
- // Create CPU profile file
- var err error
- cpuProfileFile, err = os.Create("cpu.prof")
- gomega.Expect(err).NotTo(gomega.HaveOccurred())
-
- // Start CPU profiling
- err = pprof.StartCPUProfile(cpuProfileFile)
- gomega.Expect(err).NotTo(gomega.HaveOccurred())
-
- // Create heap profile file
- heapProfileFile, err = os.Create("heap.prof")
- gomega.Expect(err).NotTo(gomega.HaveOccurred())
-})
-
-var _ = g.AfterSuite(func() {
- // Stop CPU profiling
- pprof.StopCPUProfile()
-
- // Write heap profile
- err := pprof.WriteHeapProfile(heapProfileFile)
- gomega.Expect(err).NotTo(gomega.HaveOccurred())
-
- // Close profile files
- err = cpuProfileFile.Close()
- gomega.Expect(err).NotTo(gomega.HaveOccurred())
- err = heapProfileFile.Close()
- gomega.Expect(err).NotTo(gomega.HaveOccurred())
-})
-
var _ = g.Describe("Istio", func() {
g.BeforeEach(func() {
gomega.Expect(logger.Init(logger.Logging{
@@ -99,8 +64,9 @@ var _ = g.Describe("Istio", func() {
g.It("should pass", func() {
path, deferFn, err := test.NewSpace()
gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ measurePath := filepath.Join(path, "measure")
g.DeferCleanup(func() {
- printDiskUsage(path+"/measure", 5, 0)
+ printDiskUsage(measurePath, 5, 0)
deferFn()
})
var ports []int
@@ -110,13 +76,18 @@ var _ = g.Describe("Istio", func() {
path, ports,
[]setup.SchemaLoader{&preloadService{name: "oap"}},
"--logging-level", "info")
- g.DeferCleanup(closerServerFunc)
+ g.DeferCleanup(func() {
+ time.Sleep(time.Minute)
+ closerServerFunc()
+ })
gomega.Eventually(helpers.HealthCheck(addr, 10*time.Second,
10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials())),
flags.EventuallyTimeout).Should(gomega.Succeed())
bc := &clientCounter{}
conn, err := grpchelper.Conn(addr, 10*time.Second,
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithStatsHandler(bc))
gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ closeMetricCollectorCh := metric.Start(measurePath)
g.DeferCleanup(func() {
+ close(closeMetricCollectorCh)
conn.Close()
})
startTime := time.Now()
@@ -124,6 +95,12 @@ var _ = g.Describe("Istio", func() {
gomega.Expect(err).To(gomega.Succeed())
endTime := time.Now()
+ // Print machine and OS information.
+ fmt.Println("Machine and OS information:")
+ fmt.Println(" - CPU:", helpers.CPUInfo())
+ fmt.Println(" - Memory:", helpers.MemoryInfo())
+ fmt.Println(" - OS:", helpers.OSInfo())
+
fmt.Printf("written %d items in %s\n", writtenCount,
endTime.Sub(startTime).String())
fmt.Printf("throughput: %f items/s\n",
float64(writtenCount)/endTime.Sub(startTime).Seconds())
fmt.Printf("throughput(kb/s) %f\n",
float64(bc.bytesSent)/endTime.Sub(startTime).Seconds()/1024)
diff --git a/test/stress/cases/istio/report.md
b/test/stress/cases/istio/report.md
index ee76e41e..c42e1018 100644
--- a/test/stress/cases/istio/report.md
+++ b/test/stress/cases/istio/report.md
@@ -7,12 +7,19 @@
Service: 256
Instances: 2048, 8 per service
-
### Demo cluster
Traffic RPS: 4352
VUS: 8192
+### Machine Info
+
+Machine and OS information:
+
+- CPU: CPU: AMD EPYC 7B12, VendorID: AuthenticAMD, Family: 23, Model: 49,
Cores: 8
+- Memory: Total: 47176 MB
+- OS: OS: linux, Platform: ubuntu, Platform Family: debian, Platform Version:
21.04
+
## Result
written 16186000 items in 38m43.221090505s
@@ -20,94 +27,85 @@ throughput: 6967.051077 items/s
throughput(kb/s) 1782.985321
latency: 2ns
-## CPU Profile
+### Problem
+
+Memory: blockPointer.append used 22.55% of total memory, that is significantly
larger than expected.
-CPU Usage: 324%
+Disk IO: weighted_io(ms) is 504.606667, which is larger than 100ms. This means
a lot of IO operations are blocked.
+
+## CPU
+
+CPU 95th-percentile: 2.74 cores (8 cores in total)
```bash
-Showing top 10 nodes out of 300
+Showing top 10 nodes out of 291
flat flat% sum% cum cum%
- 348.58s 4.62% 4.62% 383.07s 5.08% runtime.findObject
- 272.92s 3.62% 8.24% 272.92s 3.62% runtime.memmove
- 240.53s 3.19% 11.43% 240.53s 3.19% runtime/internal/syscall.Syscall6
- 239.05s 3.17% 14.60% 239.05s 3.17% runtime.memclrNoHeapPointers
- 210.82s 2.80% 17.40% 340.64s 4.52%
github.com/klauspost/compress/zstd.(*doubleFastEncoder).Encode
- 189.80s 2.52% 19.92% 1111.75s 14.74% runtime.mallocgc
- 182.17s 2.42% 22.33% 687.47s 9.12% runtime.scanobject
- 134.93s 1.79% 24.12% 202.49s 2.69%
github.com/dgraph-io/badger/v3/table.(*MergeIterator).Value
- 116.62s 1.55% 25.67% 116.62s 1.55% runtime.nextFreeFast (inline)
- 110.73s 1.47% 27.14% 110.73s 1.47%
github.com/klauspost/compress/zstd.sequenceDecs_decodeSync_bmi2
+ 657.01s 10.54% 10.54% 657.01s 10.54% runtime/internal/syscall.Syscall6
+ 242.03s 3.88% 14.43% 457.06s 7.33%
github.com/blevesearch/vellum.registryCache.entry
+ 211.69s 3.40% 17.82% 218.18s 3.50%
github.com/klauspost/compress/zstd.(*fseEncoder).buildCTable
+ 173.92s 2.79% 20.61% 697.34s 11.19% runtime.mallocgc
+ 172.26s 2.76% 23.38% 172.75s 2.77%
github.com/blevesearch/vellum.(*builderNode).equiv (inline)
+ 164.02s 2.63% 26.01% 188.35s 3.02% runtime.findObject
+ 131.81s 2.12% 28.13% 169.79s 2.72%
github.com/klauspost/compress/zstd.(*fastEncoder).EncodeNoHist
+ 127.66s 2.05% 30.17% 127.66s 2.05% runtime.memmove
+ 123.01s 1.97% 32.15% 141.11s 2.26%
github.com/klauspost/compress/huff0.(*Scratch).huffSort
+ 99.42s 1.60% 33.74% 134.90s 2.16%
github.com/blevesearch/vellum.(*builderNodeUnfinished).lastCompiled
```
-From the top 10 list, we can see that the CPU is mainly used by `compaction`.
+From the top 10 list, we can see that the CPU is mainly used by `Syscall6`.
## Heap Profile
-Heap Size: 1.2GB
+`alloc_bytes` 95th-percentile: 968.94 MB.
+`heap_inuse_bytes` 95th-percentile: 1054.00 MB.
+`sys_bytes` 95th-percentile: 1445.43 MB.
+`stack_inuse_bytes` 95th-percentile: 18.03 MB.
```bash
-Showing top 10 nodes out of 104
+Showing top 10 nodes out of 212
flat flat% sum% cum cum%
- 690.27MB 53.22% 53.22% 690.27MB 53.22%
github.com/dgraph-io/ristretto/z.Calloc (inline)
- 172.07MB 13.27% 66.48% 172.07MB 13.27% runtime.malg
- 128MB 9.87% 76.35% 128MB 9.87%
github.com/klauspost/compress/zstd.(*fastBase).ensureHist (inline)
- 78.98MB 6.09% 82.44% 78.98MB 6.09%
github.com/dgraph-io/badger/v3/skl.newArena
- 57.51MB 4.43% 86.87% 141.71MB 10.92%
github.com/dgraph-io/badger/v3/table.(*Builder).addHelper.func1
- 36.02MB 2.78% 89.65% 177.73MB 13.70%
github.com/dgraph-io/badger/v3/table.(*Builder).addHelper
- 28.97MB 2.23% 91.88% 28.97MB 2.23% runtime/pprof.(*profMap).lookup
- 26.50MB 2.04% 93.93% 757.59MB 58.41%
github.com/dgraph-io/badger/v3/table.(*Builder).addInternal
- 8.21MB 0.63% 94.56% 8.21MB 0.63%
github.com/klauspost/compress/zstd.encoderOptions.encoder
- 4MB 0.31% 94.87% 48.50MB 3.74%
github.com/dgraph-io/badger/v3/table.(*Table).block
+ 132.68GB 22.55% 22.55% 132.68GB 22.55%
github.com/apache/skywalking-banyandb/banyand/measure.(*blockPointer).append
+ 46.53GB 7.91% 30.47% 46.53GB 7.91%
github.com/blevesearch/vellum.(*unfinishedNodes).get
+ 25.95GB 4.41% 34.88% 25.95GB 4.41% reflect.New
+ 25.34GB 4.31% 39.19% 25.34GB 4.31%
github.com/blevesearch/vellum.(*builderNodePool).Get
+ 19.88GB 3.38% 42.57% 29.48GB 5.01%
github.com/apache/skywalking-banyandb/banyand/measure.(*blockMetadata).unmarshal
+ 18.69GB 3.18% 45.74% 18.69GB 3.18%
github.com/RoaringBitmap/roaring.(*Bitmap).Iterator
+ 16.04GB 2.73% 48.47% 22.22GB 3.78%
github.com/apache/skywalking-banyandb/banyand/measure.(*blockMetadata).copyFrom
+ 15.90GB 2.70% 51.17% 15.90GB 2.70%
github.com/blevesearch/vellum.(*builderNodeUnfinished).lastCompiled
+ 13.91GB 2.36% 53.54% 13.91GB 2.36%
github.com/blugelabs/bluge/analysis.TokenFrequency
+ 13.91GB 2.36% 55.90% 13.91GB 2.36%
github.com/apache/skywalking-banyandb/banyand/measure.(*columnFamilyMetadata).resizeColumnMetadata
(inline)
```
-From the top 10 list, we can see that the memory is mainly used by write
`buffer(skl)` and `compaction(table)`.
-Especially, the compaction includes several table related operations, such as
`table.(*Builder).addHelper`,
-consumes most of the memory.
-
+`blockPointer.append` is the most memory consuming function.
## Disk Usage
```bash
-measure: 446 MB
-measure/measure-default: 272 MB
-measure/measure-default/shard-0: 272 MB
-measure/measure-default/shard-0/buffer-0: 1.4 MB
-measure/measure-default/shard-0/buffer-1: 2.8 MB
-measure/measure-default/shard-0/seg-20231015: 247 MB
-measure/measure-default/shard-0/seg-20231015/block-2023101516: 247 MB
-measure/measure-default/shard-0/seg-20231015/block-2023101516/encoded: 74 MB
-measure/measure-default/shard-0/seg-20231015/block-2023101516/lsm: 83 MB
-measure/measure-default/shard-0/seg-20231015/block-2023101516/tst: 90 MB
-measure/measure-default/shard-0/seg-20231015/index: 0 B
-measure/measure-default/shard-0/series: 21 MB
-measure/measure-default/shard-0/series/inverted: 2.9 MB
-measure/measure-default/shard-0/series/lsm: 1.0 MB
-measure/measure-default/shard-0/series/md: 17 MB
-measure/measure-minute: 173 MB
-measure/measure-minute/shard-0: 89 MB
-measure/measure-minute/shard-0/buffer-0: 2.0 MB
-measure/measure-minute/shard-0/buffer-1: 1.6 MB
-measure/measure-minute/shard-0/seg-20231015: 76 MB
-measure/measure-minute/shard-0/seg-20231015/block-2023101516: 76 MB
-measure/measure-minute/shard-0/seg-20231015/block-2023101516/encoded: 23 MB
-measure/measure-minute/shard-0/seg-20231015/block-2023101516/lsm: 26 MB
-measure/measure-minute/shard-0/seg-20231015/block-2023101516/tst: 28 MB
-measure/measure-minute/shard-0/seg-20231015/index: 0 B
-measure/measure-minute/shard-0/series: 8.7 MB
-measure/measure-minute/shard-0/series/inverted: 2.1 MB
-measure/measure-minute/shard-0/series/lsm: 1.0 MB
-measure/measure-minute/shard-0/series/md: 5.6 MB
-measure/measure-minute/shard-1: 84 MB
-measure/measure-minute/shard-1/buffer-0: 1.5 MB
-measure/measure-minute/shard-1/buffer-1: 698 kB
-measure/measure-minute/shard-1/seg-20231015: 73 MB
-measure/measure-minute/shard-1/seg-20231015/block-2023101516: 73 MB
-measure/measure-minute/shard-1/seg-20231015/block-2023101516/encoded: 19 MB
-measure/measure-minute/shard-1/seg-20231015/block-2023101516/lsm: 26 MB
-measure/measure-minute/shard-1/seg-20231015/block-2023101516/tst: 28 MB
-measure/measure-minute/shard-1/seg-20231015/index: 0 B
-measure/measure-minute/shard-1/series: 9.3 MB
-measure/measure-minute/shard-1/series/inverted: 2.7 MB
-measure/measure-minute/shard-1/series/lsm: 1.0 MB
-measure/measure-minute/shard-1/series/md: 5.6 MB
+measure: 300 MB
+measure/measure-default: 137 MB
+measure/measure-default/idx: 53 MB
+measure/measure-default/shard-0: 84 MB
+measure/measure-default/shard-0/seg-20240111: 84 MB
+measure/measure-default/shard-0/seg-20240111/000000000000208b: 84 MB
+measure/measure-minute: 162 MB
+measure/measure-minute/idx: 32 MB
+measure/measure-minute/shard-0: 65 MB
+measure/measure-minute/shard-0/seg-20240111: 65 MB
+measure/measure-minute/shard-0/seg-20240111/00000000000010f4: 65 MB
+measure/measure-minute/shard-1: 65 MB
+measure/measure-minute/shard-1/seg-20240111: 65 MB
+measure/measure-minute/shard-1/seg-20240111/000000000000109a: 65 MB
```
+
+## Disk IO
+
+| Metric | 95th-percentile per second |
+|---------------------|-----------------------------|
+| read_count | 0.856667 |
+| merged_read_count | 0.603333 |
+| write_count | 136.976667 |
+| merged_write_count | 269.993333 |
+| read_bytes | 13817.173333 |
+| write_bytes | 17439505.066667 |
+| io_time(ms) | 125.560000 |
+| weighted_io(ms) | 504.606667 |
diff --git a/test/stress/cases/istio/testdata/metrics/data.csv
b/test/stress/cases/istio/testdata/metrics/data.csv
new file mode 100644
index 00000000..fd586907
--- /dev/null
+++ b/test/stress/cases/istio/testdata/metrics/data.csv
@@ -0,0 +1,157 @@
+1047477344.000000,1076912128.000000,1149135880.000000,9109504.000000,71.290000,215987310592.000000,532622512128.000000,2579166.000000,934717.000000,24098711.000000,33261592.000000,78010393600.000000,822417236992.000000,15857708.000000,41007686.000000
+144503392.000000,250183680.000000,1171024904.000000,2916352.000000,147.110000,216135090176.000000,532622512128.000000,2579166.000000,934717.000000,24104227.000000,33274471.000000,78010393600.000000,822863467520.000000,15862236.000000,41016666.000000
+196618704.000000,292552704.000000,1171024904.000000,2981888.000000,147.960000,216105725952.000000,532622512128.000000,2579167.000000,934717.000000,24104668.000000,33274620.000000,78010397696.000000,822940627968.000000,15862436.000000,41018250.000000
+197673784.000000,293806080.000000,1171024904.000000,2981888.000000,148.050000,216105734144.000000,532622512128.000000,2579172.000000,934717.000000,24104678.000000,33274698.000000,78010475520.000000,822940988416.000000,15862488.000000,41018262.000000
+321690400.000000,404094976.000000,1171024904.000000,5603328.000000,236.420000,216214724608.000000,532622512128.000000,2579172.000000,934717.000000,24109165.000000,33286373.000000,78010475520.000000,823294436352.000000,15867428.000000,41024247.000000
+200504888.000000,296976384.000000,1171024904.000000,2949120.000000,274.500000,216153882624.000000,532622512128.000000,2579173.000000,934717.000000,24111338.000000,33290076.000000,78010479616.000000,823632253952.000000,15869172.000000,41031684.000000
+201537752.000000,298123264.000000,1171024904.000000,2949120.000000,274.590000,216153882624.000000,532622512128.000000,2579173.000000,934717.000000,24111459.000000,33290132.000000,78010479616.000000,823633011712.000000,15869208.000000,41031791.000000
+202622776.000000,299270144.000000,1171024904.000000,2949120.000000,274.680000,216153886720.000000,532622512128.000000,2579173.000000,934717.000000,24111601.000000,33290167.000000,78010479616.000000,823633761280.000000,15869244.000000,41031886.000000
+292802872.000000,389914624.000000,1171024904.000000,7503872.000000,355.060000,216269213696.000000,532622512128.000000,2579173.000000,934717.000000,24115275.000000,33300171.000000,78010479616.000000,823924450304.000000,15873508.000000,41037192.000000
+183342520.000000,291782656.000000,1171024904.000000,3407872.000000,406.840000,216163098624.000000,532622512128.000000,2579173.000000,934717.000000,24118323.000000,33305384.000000,78010479616.000000,824397960192.000000,15875992.000000,41048857.000000
+184347096.000000,292814848.000000,1171024904.000000,3407872.000000,406.930000,216163098624.000000,532622512128.000000,2579173.000000,934717.000000,24118438.000000,33305447.000000,78010479616.000000,824398713856.000000,15876024.000000,41048944.000000
+185456736.000000,293994496.000000,1171024904.000000,3440640.000000,407.020000,216163106816.000000,532622512128.000000,2579173.000000,934717.000000,24118618.000000,33306301.000000,78010479616.000000,824404116480.000000,15876336.000000,41049084.000000
+450575008.000000,527925248.000000,1171024904.000000,6520832.000000,493.580000,216343662592.000000,532622512128.000000,2579173.000000,934717.000000,24122829.000000,33316912.000000,78010479616.000000,824741639168.000000,15880916.000000,41055064.000000
+254195832.000000,335667200.000000,1171024904.000000,3866624.000000,544.280000,216182599680.000000,532622512128.000000,2579173.000000,934717.000000,24125956.000000,33321736.000000,78010479616.000000,825228895232.000000,15883100.000000,41068874.000000
+255186416.000000,336691200.000000,1171024904.000000,3866624.000000,544.360000,216182599680.000000,532622512128.000000,2579173.000000,934717.000000,24126019.000000,33321791.000000,78010479616.000000,825229394944.000000,15883128.000000,41068928.000000
+255438160.000000,337059840.000000,1171024904.000000,3866624.000000,544.460000,216182599680.000000,532622512128.000000,2579173.000000,934717.000000,24126078.000000,33321821.000000,78010479616.000000,825229767680.000000,15883152.000000,41068969.000000
+492845008.000000,544358400.000000,1171024904.000000,7962624.000000,634.530000,216316932096.000000,532622512128.000000,2579173.000000,934717.000000,24130450.000000,33332769.000000,78010479616.000000,825606484992.000000,15887836.000000,41075835.000000
+159341840.000000,297156608.000000,1171024904.000000,4227072.000000,682.610000,216191127552.000000,532622512128.000000,2579173.000000,934717.000000,24133200.000000,33336633.000000,78010479616.000000,826068956160.000000,15889828.000000,41087650.000000
+160330280.000000,298188800.000000,1171024904.000000,4227072.000000,682.690000,216191127552.000000,532622512128.000000,2579173.000000,934717.000000,24133333.000000,33336702.000000,78010479616.000000,826069812224.000000,15889856.000000,41087748.000000
+161327512.000000,299442176.000000,1171024904.000000,4227072.000000,682.790000,216191131648.000000,532622512128.000000,2579173.000000,934717.000000,24133344.000000,33336734.000000,78010479616.000000,826069988352.000000,15889880.000000,41087758.000000
+488013200.000000,553967616.000000,1171024904.000000,8323072.000000,774.290000,216305545216.000000,532622512128.000000,2579173.000000,934717.000000,24137807.000000,33347582.000000,78010479616.000000,826467251200.000000,15894648.000000,41096028.000000
+247208864.000000,372301824.000000,1171024904.000000,4653056.000000,820.760000,216212951040.000000,532622512128.000000,2579173.000000,934717.000000,24140622.000000,33351724.000000,78010479616.000000,826940556288.000000,15896816.000000,41112039.000000
+248231408.000000,373309440.000000,1171024904.000000,4653056.000000,820.860000,216212979712.000000,532622512128.000000,2579173.000000,934717.000000,24140668.000000,33351796.000000,78010479616.000000,826941064192.000000,15896840.000000,41112085.000000
+249232632.000000,374538240.000000,1171024904.000000,4685824.000000,820.950000,216212983808.000000,532622512128.000000,2579173.000000,934717.000000,24140832.000000,33351861.000000,78010479616.000000,826942055424.000000,15896872.000000,41112196.000000
+417615264.000000,488087552.000000,1171287048.000000,8716288.000000,898.070000,216326787072.000000,532622512128.000000,2579173.000000,934717.000000,24144333.000000,33361662.000000,78010479616.000000,827219571712.000000,15901056.000000,41117278.000000
+226420104.000000,362766336.000000,1171287048.000000,5013504.000000,962.480000,216250486784.000000,532622512128.000000,2579173.000000,934717.000000,24148103.000000,33368077.000000,78010479616.000000,827809596416.000000,15904056.000000,41131234.000000
+261184032.000000,390455296.000000,1171287048.000000,5046272.000000,965.050000,216210780160.000000,532622512128.000000,2579173.000000,934717.000000,24148565.000000,33368253.000000,78010479616.000000,827883713536.000000,15904276.000000,41132947.000000
+262184976.000000,391438336.000000,1171287048.000000,5046272.000000,965.140000,216210780160.000000,532622512128.000000,2579173.000000,934717.000000,24148574.000000,33368275.000000,78010479616.000000,827883840512.000000,15904304.000000,41132954.000000
+604721304.000000,646250496.000000,1171549192.000000,6815744.000000,1043.320000,216356417536.000000,532622512128.000000,2579177.000000,934717.000000,24152276.000000,33378336.000000,78010496000.000000,828158329856.000000,15908588.000000,41137926.000000
+313094112.000000,447815680.000000,1171549192.000000,5505024.000000,1104.420000,216272936960.000000,532622512128.000000,2579178.000000,934717.000000,24155771.000000,33384261.000000,78010500096.000000,828726760448.000000,15911396.000000,41152748.000000
+216737216.000000,368173056.000000,1171549192.000000,5505024.000000,1107.830000,216241926144.000000,532622512128.000000,2579178.000000,934717.000000,24156168.000000,33384402.000000,78010500096.000000,828805039104.000000,15911604.000000,41154417.000000
+217728816.000000,369238016.000000,1171549192.000000,5505024.000000,1107.920000,216241926144.000000,532622512128.000000,2579178.000000,934717.000000,24156176.000000,33384427.000000,78010500096.000000,828805174272.000000,15911624.000000,41154423.000000
+380038216.000000,504356864.000000,1171549192.000000,9011200.000000,1186.700000,216343621632.000000,532622512128.000000,2579178.000000,934717.000000,24159852.000000,33394035.000000,78010500096.000000,829086954496.000000,15915792.000000,41159679.000000
+326293912.000000,461873152.000000,1171811336.000000,5832704.000000,1247.120000,216264146944.000000,532622512128.000000,2579178.000000,934717.000000,24163452.000000,33399814.000000,78010500096.000000,829657355264.000000,15918560.000000,41176808.000000
+215114168.000000,373841920.000000,1171811336.000000,5898240.000000,1250.410000,216228913152.000000,532622512128.000000,2579178.000000,934717.000000,24163888.000000,33399939.000000,78010500096.000000,829740545024.000000,15918788.000000,41178746.000000
+216095096.000000,374923264.000000,1171811336.000000,5898240.000000,1250.490000,216228913152.000000,532622512128.000000,2579178.000000,934717.000000,24163950.000000,33400010.000000,78010500096.000000,829741089792.000000,15918820.000000,41178793.000000
+435942384.000000,538640384.000000,1171811336.000000,9404416.000000,1329.010000,216359718912.000000,532622512128.000000,2579178.000000,934717.000000,24167602.000000,33409087.000000,78010500096.000000,830015144960.000000,15923008.000000,41184164.000000
+366382888.000000,503480320.000000,1171811336.000000,6225920.000000,1396.520000,216318005248.000000,532622512128.000000,2579178.000000,934717.000000,24171556.000000,33415349.000000,78010500096.000000,830678291456.000000,15926204.000000,41204060.000000
+288063024.000000,438190080.000000,1171811336.000000,6324224.000000,1400.280000,216287191040.000000,532622512128.000000,2579178.000000,934717.000000,24171987.000000,33415495.000000,78010500096.000000,830766330880.000000,15926452.000000,41206305.000000
+289049008.000000,439255040.000000,1171811336.000000,6324224.000000,1400.370000,216287199232.000000,532622512128.000000,2579178.000000,934717.000000,24172063.000000,33415519.000000,78010500096.000000,830766752768.000000,15926476.000000,41206361.000000
+592820440.000000,648617984.000000,1171811336.000000,10944512.000000,1491.500000,216416116736.000000,532622512128.000000,2579178.000000,934717.000000,24176089.000000,33426933.000000,78010500096.000000,831081718784.000000,15931292.000000,41212274.000000
+412955272.000000,549838848.000000,1171811336.000000,6717440.000000,1552.530000,216303153152.000000,532622512128.000000,2579178.000000,934717.000000,24179957.000000,33431978.000000,78010500096.000000,831786656768.000000,15933968.000000,41235608.000000
+400606016.000000,540704768.000000,1171811336.000000,6750208.000000,1554.720000,216243875840.000000,532622512128.000000,2579178.000000,934717.000000,24180509.000000,33432136.000000,78010500096.000000,831880209408.000000,15934208.000000,41237618.000000
+401658792.000000,541786112.000000,1171811336.000000,6750208.000000,1554.810000,216243879936.000000,532622512128.000000,2579178.000000,934717.000000,24180515.000000,33432154.000000,78010500096.000000,831880307712.000000,15934224.000000,41237623.000000
+705799792.000000,755605504.000000,1171811336.000000,11239424.000000,1645.060000,216447635456.000000,532622512128.000000,2579178.000000,934717.000000,24184545.000000,33443202.000000,78010500096.000000,832175440896.000000,15939052.000000,41242770.000000
+435444016.000000,572989440.000000,1171811336.000000,7208960.000000,1697.150000,216331624448.000000,532622512128.000000,2579178.000000,934717.000000,24188047.000000,33447769.000000,78010500096.000000,832808510464.000000,15941368.000000,41263377.000000
+357952816.000000,505380864.000000,1171811336.000000,7208960.000000,1698.940000,216261554176.000000,532622512128.000000,2579178.000000,934717.000000,24188613.000000,33447900.000000,78010500096.000000,832906941440.000000,15941624.000000,41266004.000000
+358981336.000000,506413056.000000,1171811336.000000,7208960.000000,1699.020000,216261554176.000000,532622512128.000000,2579178.000000,934717.000000,24188666.000000,33447919.000000,78010500096.000000,832907236352.000000,15941656.000000,41266042.000000
+645200880.000000,712359936.000000,1171811336.000000,11304960.000000,1789.820000,216432041984.000000,532622512128.000000,2579178.000000,934717.000000,24192661.000000,33459025.000000,78010500096.000000,833216009216.000000,15946280.000000,41271750.000000
+486666376.000000,629022720.000000,1171811336.000000,7569408.000000,1844.880000,216325537792.000000,532622512128.000000,2579178.000000,934717.000000,24196310.000000,33464136.000000,78010500096.000000,833855517696.000000,15948896.000000,41289795.000000
+279969736.000000,453255168.000000,1171811336.000000,7602176.000000,1847.550000,216268791808.000000,532622512128.000000,2579178.000000,934717.000000,24196803.000000,33464283.000000,78010500096.000000,833958388736.000000,15949132.000000,41292740.000000
+280966504.000000,454287360.000000,1171811336.000000,7602176.000000,1847.630000,216268800000.000000,532622512128.000000,2579178.000000,934717.000000,24197104.000000,33465131.000000,78010500096.000000,833964315648.000000,15949464.000000,41292992.000000
+579870776.000000,726736896.000000,1171811336.000000,10485760.000000,1935.420000,216453967872.000000,532622512128.000000,2579178.000000,934717.000000,24201162.000000,33475831.000000,78010500096.000000,834261988352.000000,15953888.000000,41298526.000000
+475865016.000000,630161408.000000,1171811336.000000,7995392.000000,1992.630000,216328896512.000000,532622512128.000000,2579178.000000,934717.000000,24205045.000000,33480941.000000,78010500096.000000,834951488512.000000,15956544.000000,41325944.000000
+277482128.000000,462774272.000000,1171811336.000000,7995392.000000,1995.450000,216269619200.000000,532622512128.000000,2579178.000000,934717.000000,24205505.000000,33481060.000000,78010500096.000000,835058951168.000000,15956800.000000,41329203.000000
+278469464.000000,463814656.000000,1171811336.000000,7995392.000000,1995.530000,216269619200.000000,532622512128.000000,2579178.000000,934717.000000,24205568.000000,33481121.000000,78010500096.000000,835059463168.000000,15956828.000000,41329255.000000
+527738136.000000,645808128.000000,1171811336.000000,11534336.000000,2081.020000,216436346880.000000,532622512128.000000,2579178.000000,934717.000000,24209341.000000,33491514.000000,78010500096.000000,835349738496.000000,15961140.000000,41334868.000000
+276888040.000000,477872128.000000,1171811336.000000,8355840.000000,2143.000000,216321773568.000000,532622512128.000000,2579178.000000,934717.000000,24213436.000000,33497515.000000,78010500096.000000,836062512128.000000,15964140.000000,41360952.000000
+437751880.000000,602570752.000000,1171811336.000000,8421376.000000,2146.530000,216277540864.000000,532622512128.000000,2579178.000000,934717.000000,24213996.000000,33497636.000000,78010500096.000000,836174967808.000000,15964376.000000,41367221.000000
+438791088.000000,603660288.000000,1171811336.000000,8421376.000000,2146.630000,216277544960.000000,532622512128.000000,2579178.000000,934717.000000,24214053.000000,33497694.000000,78010500096.000000,836175438848.000000,15964408.000000,41367267.000000
+618939528.000000,709967872.000000,1171811336.000000,12058624.000000,2228.890000,216448671744.000000,532622512128.000000,2579178.000000,934717.000000,24217645.000000,33507672.000000,78010500096.000000,836453663744.000000,15968716.000000,41372305.000000
+279186032.000000,487243776.000000,1171811336.000000,8781824.000000,2293.640000,216244719616.000000,532622512128.000000,2579178.000000,934717.000000,24221695.000000,33513492.000000,78010500096.000000,837132968960.000000,15971568.000000,41393505.000000
+519075632.000000,675717120.000000,1171811336.000000,8847360.000000,2298.990000,216226930688.000000,532622512128.000000,2579178.000000,934717.000000,24221990.000000,33513597.000000,78010500096.000000,837197677568.000000,15971756.000000,41395008.000000
+520077248.000000,676741120.000000,1171811336.000000,8847360.000000,2299.070000,216226930688.000000,532622512128.000000,2579178.000000,934717.000000,24221999.000000,33513608.000000,78010500096.000000,837197759488.000000,15971780.000000,41395016.000000
+724053432.000000,808714240.000000,1171811336.000000,12353536.000000,2392.520000,216420888576.000000,532622512128.000000,2579178.000000,934717.000000,24226100.000000,33524859.000000,78010500096.000000,837513647104.000000,15976660.000000,41401007.000000
+358294152.000000,547651584.000000,1171811336.000000,9273344.000000,2444.330000,216265412608.000000,532622512128.000000,2579178.000000,934717.000000,24229468.000000,33528852.000000,78010500096.000000,838106416128.000000,15978808.000000,41415078.000000
+311804984.000000,510500864.000000,1171811336.000000,9273344.000000,2449.080000,216244969472.000000,532622512128.000000,2579178.000000,934717.000000,24229823.000000,33528996.000000,78010500096.000000,838172369920.000000,15979000.000000,41416560.000000
+312764648.000000,511590400.000000,1171811336.000000,9273344.000000,2449.160000,216244973568.000000,532622512128.000000,2579178.000000,934717.000000,24229829.000000,33529014.000000,78010500096.000000,838172468224.000000,15979016.000000,41416565.000000
+529505368.000000,696999936.000000,1171811336.000000,12779520.000000,2542.110000,216430071808.000000,532622512128.000000,2579178.000000,934717.000000,24234010.000000,33540229.000000,78010500096.000000,838497436672.000000,15983972.000000,41422781.000000
+339917808.000000,540098560.000000,1171811336.000000,9666560.000000,2601.990000,216304758784.000000,532622512128.000000,2579178.000000,934717.000000,24237386.000000,33544345.000000,78010500096.000000,839106565120.000000,15986292.000000,41442663.000000
+407897520.000000,592953344.000000,1171811336.000000,9699328.000000,2607.860000,216234901504.000000,532622512128.000000,2579178.000000,934717.000000,24238024.000000,33544605.000000,78010500096.000000,839234204672.000000,15986596.000000,41446313.000000
+408902544.000000,593977344.000000,1171811336.000000,9699328.000000,2607.960000,216234901504.000000,532622512128.000000,2579178.000000,934717.000000,24238101.000000,33544625.000000,78010500096.000000,839234618368.000000,15986620.000000,41446376.000000
+775233832.000000,854188032.000000,1171811336.000000,14286848.000000,2699.790000,216443449344.000000,532622512128.000000,2579178.000000,934717.000000,24242147.000000,33555074.000000,78010500096.000000,839550624768.000000,15991248.000000,41452032.000000
+591935272.000000,772947968.000000,1171811336.000000,10125312.000000,2758.970000,216241905664.000000,532622512128.000000,2579435.000000,934898.000000,24245810.000000,33559685.000000,78014645248.000000,840164103168.000000,15993940.000000,41468601.000000
+391461864.000000,584253440.000000,1171811336.000000,10092544.000000,2765.330000,216235085824.000000,532622512128.000000,2579435.000000,934898.000000,24246117.000000,33559808.000000,78014645248.000000,840231408640.000000,15994140.000000,41470045.000000
+392425000.000000,585351168.000000,1171811336.000000,10092544.000000,2765.410000,216235089920.000000,532622512128.000000,2579435.000000,934898.000000,24246200.000000,33560870.000000,78014645248.000000,840236114944.000000,15994192.000000,41470141.000000
+557936528.000000,717930496.000000,1171811336.000000,14123008.000000,2852.760000,216390418432.000000,532622512128.000000,2579435.000000,934898.000000,24250931.000000,33571561.000000,78014645248.000000,840533263360.000000,15998636.000000,41477217.000000
+621850232.000000,789372928.000000,1171811336.000000,10518528.000000,2915.620000,216246894592.000000,532622512128.000000,2579435.000000,934898.000000,24254616.000000,33576880.000000,78014645248.000000,841151087616.000000,16001252.000000,41493824.000000
+333379000.000000,550379520.000000,1171811336.000000,10518528.000000,2922.120000,216239259648.000000,532622512128.000000,2579435.000000,934898.000000,24254923.000000,33576990.000000,78014645248.000000,841219154944.000000,16001448.000000,41495421.000000
+334343928.000000,551493632.000000,1171811336.000000,10518528.000000,2922.210000,216239263744.000000,532622512128.000000,2579435.000000,934898.000000,24255075.000000,33577054.000000,78014645248.000000,841220056064.000000,16001480.000000,41495536.000000
+858211176.000000,904830976.000000,1171811336.000000,16121856.000000,3004.990000,216371470336.000000,532622512128.000000,2579435.000000,934898.000000,24258733.000000,33587720.000000,78014645248.000000,841496740864.000000,16005828.000000,41500442.000000
+547721224.000000,733822976.000000,1171811336.000000,10878976.000000,3073.730000,216325128192.000000,532622512128.000000,2579435.000000,934898.000000,24262148.000000,33593399.000000,78014645248.000000,842056430592.000000,16008468.000000,41516397.000000
+429676408.000000,633020416.000000,1171811336.000000,10944512.000000,3084.970000,216231440384.000000,532622512128.000000,2579435.000000,934898.000000,24263028.000000,33593774.000000,78014645248.000000,842251936768.000000,16008984.000000,41521642.000000
+430700848.000000,634077184.000000,1171811336.000000,10944512.000000,3085.060000,216231440384.000000,532622512128.000000,2579435.000000,934898.000000,24263084.000000,33593796.000000,78014645248.000000,842252264448.000000,16009020.000000,41521683.000000
+923123984.000000,967335936.000000,1172073480.000000,16220160.000000,3180.160000,216383979520.000000,532622512128.000000,2579435.000000,934898.000000,24267264.000000,33605055.000000,78014645248.000000,842584101888.000000,16014136.000000,41528417.000000
+610214232.000000,804519936.000000,1172073480.000000,11304960.000000,3241.030000,216232701952.000000,532622512128.000000,2579435.000000,934898.000000,24270464.000000,33608704.000000,78014645248.000000,843160441856.000000,16016340.000000,41546342.000000
+416499096.000000,628285440.000000,1172073480.000000,11337728.000000,3247.470000,216225935360.000000,532622512128.000000,2579435.000000,934898.000000,24270824.000000,33608838.000000,78014645248.000000,843230008320.000000,16016548.000000,41548140.000000
+417482744.000000,629350400.000000,1172073480.000000,11337728.000000,3247.560000,216225935360.000000,532622512128.000000,2579435.000000,934898.000000,24270832.000000,33608854.000000,78014645248.000000,843230106624.000000,16016568.000000,41548147.000000
+896464040.000000,948518912.000000,1172073480.000000,15138816.000000,3341.310000,216402153472.000000,532622512128.000000,2579435.000000,934898.000000,24274916.000000,33619751.000000,78014645248.000000,843530368000.000000,16021204.000000,41553600.000000
+620347856.000000,801988608.000000,1172073480.000000,11730944.000000,3405.000000,216255041536.000000,532622512128.000000,2579435.000000,934898.000000,24278215.000000,33623940.000000,78014645248.000000,844132058112.000000,16023476.000000,41571012.000000
+473314064.000000,677068800.000000,1172073480.000000,11796480.000000,3414.260000,216223866880.000000,532622512128.000000,2579435.000000,934898.000000,24278840.000000,33624219.000000,78014645248.000000,844258182144.000000,16023824.000000,41573783.000000
+474333152.000000,678182912.000000,1172073480.000000,11796480.000000,3414.340000,216223875072.000000,532622512128.000000,2579435.000000,934898.000000,24279096.000000,33625048.000000,78014645248.000000,844263810048.000000,16024176.000000,41574007.000000
+1024406976.000000,1060954112.000000,1197239304.000000,14024704.000000,3505.810000,216415698944.000000,532622512128.000000,2579435.000000,934898.000000,24283355.000000,33636669.000000,78014645248.000000,844564558848.000000,16028968.000000,41579426.000000
+676390872.000000,860880896.000000,1197239304.000000,12156928.000000,3568.490000,216247361536.000000,532622512128.000000,2579435.000000,934898.000000,24286844.000000,33640802.000000,78014645248.000000,845171852288.000000,16031420.000000,41601205.000000
+648432320.000000,838860800.000000,1197239304.000000,12222464.000000,3574.790000,216240578560.000000,532622512128.000000,2579435.000000,934898.000000,24287162.000000,33640924.000000,78014645248.000000,845242795008.000000,16031620.000000,41602861.000000
+649469704.000000,839892992.000000,1197239304.000000,12222464.000000,3574.870000,216240582656.000000,532622512128.000000,2579435.000000,934898.000000,24287228.000000,33640981.000000,78014645248.000000,845243298816.000000,16031644.000000,41602914.000000
+791933504.000000,897048576.000000,1197239304.000000,16842752.000000,3667.150000,216386162688.000000,532622512128.000000,2579435.000000,934898.000000,24291301.000000,33651981.000000,78014645248.000000,845555262464.000000,16036424.000000,41608573.000000
+659822856.000000,864133120.000000,1197239304.000000,12517376.000000,3732.590000,216325578752.000000,532622512128.000000,2579435.000000,934898.000000,24294340.000000,33655899.000000,78014645248.000000,846065882112.000000,16038728.000000,41622926.000000
+508670016.000000,730415104.000000,1197239304.000000,12550144.000000,3739.550000,216233742336.000000,532622512128.000000,2579435.000000,934898.000000,24294980.000000,33656115.000000,78014645248.000000,846209262592.000000,16039044.000000,41627836.000000
+509628544.000000,731455488.000000,1197239304.000000,12582912.000000,3739.640000,216233742336.000000,532622512128.000000,2579435.000000,934898.000000,24295049.000000,33656137.000000,78014645248.000000,846209647616.000000,16039068.000000,41627884.000000
+902410432.000000,981835776.000000,1206761496.000000,15826944.000000,3829.190000,216400343040.000000,532622512128.000000,2579435.000000,934898.000000,24298890.000000,33666687.000000,78014645248.000000,846500025344.000000,16043816.000000,41633120.000000
+693899760.000000,895844352.000000,1206761496.000000,12976128.000000,3895.350000,216437198848.000000,532622512128.000000,2579435.000000,934898.000000,24301593.000000,33670927.000000,78014645248.000000,846926517248.000000,16046072.000000,41644057.000000
+570358488.000000,789594112.000000,1206761496.000000,13008896.000000,3903.870000,216244035584.000000,532622512128.000000,2579435.000000,934898.000000,24302842.000000,33671429.000000,78014645248.000000,847182058496.000000,16046652.000000,41653274.000000
+571431264.000000,790650880.000000,1206761496.000000,13008896.000000,3903.970000,216244035584.000000,532622512128.000000,2579435.000000,934898.000000,24302849.000000,33671445.000000,78014645248.000000,847182152704.000000,16046672.000000,41653280.000000
+613321872.000000,827072512.000000,1206761496.000000,16220160.000000,3991.170000,216383057920.000000,532622512128.000000,2579435.000000,934898.000000,24306615.000000,33681523.000000,78014645248.000000,847465239552.000000,16051028.000000,41658496.000000
+651780272.000000,873021440.000000,1291781160.000000,13369344.000000,4060.190000,216403828736.000000,532622512128.000000,2579436.000000,934898.000000,24309420.000000,33686092.000000,78014649344.000000,847899857920.000000,16053404.000000,41670581.000000
+689819136.000000,915087360.000000,1291781160.000000,13402112.000000,4071.990000,216237846528.000000,532622512128.000000,2579436.000000,934898.000000,24310632.000000,33686608.000000,78014649344.000000,848157824000.000000,16053992.000000,41678022.000000
+690798696.000000,916086784.000000,1291781160.000000,13402112.000000,4072.090000,216237846528.000000,532622512128.000000,2579436.000000,934898.000000,24310699.000000,33686627.000000,78014649344.000000,848158192640.000000,16054012.000000,41678075.000000
+1068923856.000000,1108983808.000000,1291781160.000000,16908288.000000,4156.900000,216353181696.000000,532622512128.000000,2579436.000000,934898.000000,24314396.000000,33696436.000000,78014649344.000000,848450978816.000000,16058316.000000,41683312.000000
+763539400.000000,972324864.000000,1316946984.000000,13795328.000000,4229.970000,216396673024.000000,532622512128.000000,2579446.000000,934898.000000,24317363.000000,33701445.000000,78014690304.000000,848902759424.000000,16060708.000000,41696371.000000
+549209112.000000,780091392.000000,1316946984.000000,13795328.000000,4241.800000,216230424576.000000,532622512128.000000,2579447.000000,934898.000000,24318667.000000,33701999.000000,78014694400.000000,849164059648.000000,16061344.000000,41702870.000000
+550267880.000000,781131776.000000,1316946984.000000,13795328.000000,4241.880000,216230428672.000000,532622512128.000000,2579447.000000,934898.000000,24318675.000000,33702012.000000,78014694400.000000,849164145664.000000,16061372.000000,41702876.000000
+835743040.000000,965910528.000000,1316946984.000000,16875520.000000,4336.600000,216434671616.000000,532622512128.000000,2579447.000000,934898.000000,24322687.000000,33713843.000000,78014694400.000000,849457705984.000000,16066176.000000,41708085.000000
+652811072.000000,873193472.000000,1316946984.000000,14221312.000000,4398.980000,216452616192.000000,532622512128.000000,2579447.000000,934898.000000,24325240.000000,33717412.000000,78014694400.000000,849878709248.000000,16068024.000000,41721215.000000
+453676944.000000,707141632.000000,1316946984.000000,14254080.000000,4407.530000,216251072512.000000,532622512128.000000,2579447.000000,934898.000000,24326422.000000,33717868.000000,78014694400.000000,850142036992.000000,16068644.000000,41728055.000000
+454674632.000000,708182016.000000,1316946984.000000,14254080.000000,4407.620000,216251076608.000000,532622512128.000000,2579447.000000,934898.000000,24326474.000000,33717922.000000,78014694400.000000,850142471168.000000,16068672.000000,41728100.000000
+713334024.000000,885342208.000000,1321141288.000000,18317312.000000,4500.960000,216382279680.000000,532622512128.000000,2579456.000000,934898.000000,24330515.000000,33728990.000000,78014731264.000000,850461058048.000000,16073496.000000,41734104.000000
+559177800.000000,811810816.000000,1321141288.000000,14647296.000000,4562.950000,216421654528.000000,532622512128.000000,2579456.000000,934898.000000,24333155.000000,33733175.000000,78014731264.000000,850870875136.000000,16075644.000000,41745997.000000
+573617280.000000,820150272.000000,1321141288.000000,14680064.000000,4571.350000,216278470656.000000,532622512128.000000,2579456.000000,934898.000000,24334080.000000,33733503.000000,78014731264.000000,851079373824.000000,16076160.000000,41751296.000000
+574631112.000000,821207040.000000,1321141288.000000,14680064.000000,4571.440000,216278470656.000000,532622512128.000000,2579456.000000,934898.000000,24334150.000000,33733524.000000,78014731264.000000,851079750656.000000,16076188.000000,41751346.000000
+747266752.000000,929832960.000000,1321141288.000000,18677760.000000,4661.930000,216389705728.000000,532622512128.000000,2579460.000000,934898.000000,24338073.000000,33744031.000000,78014747648.000000,851375182848.000000,16080996.000000,41756633.000000
+811448192.000000,1036615680.000000,1321141288.000000,15007744.000000,4728.090000,216430583808.000000,532622512128.000000,2579461.000000,934898.000000,24340751.000000,33747978.000000,78014751744.000000,851810022400.000000,16083060.000000,41768165.000000
+781961504.000000,1017454592.000000,1321141288.000000,15073280.000000,4739.130000,216245497856.000000,532622512128.000000,2579464.000000,934898.000000,24342009.000000,33748495.000000,78014764032.000000,852080100352.000000,16083668.000000,41776505.000000
+782956496.000000,1018494976.000000,1321141288.000000,15073280.000000,4739.220000,216245497856.000000,532622512128.000000,2579464.000000,934898.000000,24342081.000000,33748515.000000,78014764032.000000,852080497664.000000,16083696.000000,41776561.000000
+843682424.000000,1005879296.000000,1360023608.000000,19136512.000000,4827.440000,216408956928.000000,532622512128.000000,2579464.000000,934898.000000,24345915.000000,33759468.000000,78014764032.000000,852380787712.000000,16088200.000000,41782027.000000
+471979424.000000,744054784.000000,1372606520.000000,15433728.000000,4895.060000,216367067136.000000,532622512128.000000,2579468.000000,934898.000000,24348878.000000,33764697.000000,78014780416.000000,852849034240.000000,16090740.000000,41795652.000000
+450793784.000000,722673664.000000,1372606520.000000,15466496.000000,4905.130000,216236085248.000000,532622512128.000000,2579470.000000,934898.000000,24349923.000000,33765072.000000,78014788608.000000,853064926208.000000,16091296.000000,41801660.000000
+451720424.000000,723746816.000000,1372606520.000000,15466496.000000,4905.220000,216236085248.000000,532622512128.000000,2579470.000000,934898.000000,24349931.000000,33765088.000000,78014788608.000000,853065024512.000000,16091316.000000,41801667.000000
+774937472.000000,990445568.000000,1372606520.000000,18382848.000000,4989.800000,216410808320.000000,532622512128.000000,2579473.000000,934898.000000,24353655.000000,33775394.000000,78014800896.000000,853330379776.000000,16095656.000000,41806506.000000
+692727160.000000,935575552.000000,1372606520.000000,15826944.000000,5057.970000,216424169472.000000,532622512128.000000,2579476.000000,934898.000000,24356737.000000,33779741.000000,78014813184.000000,853790569472.000000,16098008.000000,41819919.000000
+825238232.000000,1057636352.000000,1372606520.000000,15925248.000000,5071.140000,216258383872.000000,532622512128.000000,2579478.000000,934898.000000,24358006.000000,33780225.000000,78014821376.000000,854068589568.000000,16098648.000000,41827064.000000
+826229192.000000,1058660352.000000,1372606520.000000,15925248.000000,5071.220000,216258396160.000000,532622512128.000000,2579478.000000,934898.000000,24358252.000000,33781092.000000,78014821376.000000,854074319872.000000,16099032.000000,41827279.000000
+1051856816.000000,1130823680.000000,1372606520.000000,20119552.000000,5170.900000,216427343872.000000,532622512128.000000,2579480.000000,934898.000000,24362717.000000,33793812.000000,78014829568.000000,854408086528.000000,16104248.000000,41833482.000000
+708756888.000000,952344576.000000,1436654664.000000,16252928.000000,5227.060000,216310472704.000000,532622512128.000000,2579482.000000,934898.000000,24365667.000000,33797217.000000,78014837760.000000,854921393152.000000,16106148.000000,41848371.000000
+869190960.000000,1100349440.000000,1436654664.000000,16285696.000000,5234.820000,216250499072.000000,532622512128.000000,2579484.000000,934898.000000,24366295.000000,33797432.000000,78014845952.000000,855062127616.000000,16106504.000000,41851741.000000
+870176128.000000,1101414400.000000,1436654664.000000,16285696.000000,5234.910000,216250499072.000000,532622512128.000000,2579484.000000,934898.000000,24366350.000000,33797464.000000,78014845952.000000,855062483968.000000,16106532.000000,41851788.000000
+1137455816.000000,1207549952.000000,1449237576.000000,20152320.000000,5321.490000,216388722688.000000,532622512128.000000,2579485.000000,934898.000000,24370022.000000,33807608.000000,78014850048.000000,855348302848.000000,16110928.000000,41857441.000000
+571767048.000000,844046336.000000,1449237576.000000,16711680.000000,5391.890000,216407445504.000000,532622512128.000000,2579489.000000,934898.000000,24373154.000000,33812759.000000,78014866432.000000,855829865472.000000,16113440.000000,41869770.000000
+514990208.000000,794312704.000000,1449237576.000000,16809984.000000,5406.710000,216252874752.000000,532622512128.000000,2579489.000000,934898.000000,24374484.000000,33813262.000000,78014866432.000000,856114459648.000000,16114064.000000,41880503.000000
+515922896.000000,795394048.000000,1449237576.000000,16809984.000000,5406.800000,216252874752.000000,532622512128.000000,2579489.000000,934898.000000,24374490.000000,33813278.000000,78014866432.000000,856114549760.000000,16114084.000000,41880508.000000
+1051940632.000000,1171873792.000000,1449237576.000000,20381696.000000,5494.420000,216415571968.000000,532622512128.000000,2579489.000000,934898.000000,24378305.000000,33824101.000000,78014866432.000000,856415958016.000000,16118660.000000,41886200.000000
+580243608.000000,855883776.000000,1449237576.000000,17039360.000000,5556.650000,216394838016.000000,532622512128.000000,2579489.000000,934898.000000,24381122.000000,33828867.000000,78014866432.000000,856863790080.000000,16121036.000000,41898995.000000
+613272720.000000,879353856.000000,1449237576.000000,17104896.000000,5567.490000,216267210752.000000,532622512128.000000,2579489.000000,934898.000000,24382230.000000,33829268.000000,78014866432.000000,857089094656.000000,16121576.000000,41906410.000000
+614248792.000000,880353280.000000,1449237576.000000,17104896.000000,5567.580000,216267210752.000000,532622512128.000000,2579489.000000,934898.000000,24382237.000000,33829288.000000,78014866432.000000,857089205248.000000,16121592.000000,41906416.000000
+1252014544.000000,1309556736.000000,1449237576.000000,20840448.000000,5659.490000,216431235072.000000,532622512128.000000,2579489.000000,934898.000000,24386023.000000,33839361.000000,78014866432.000000,857377813504.000000,16126184.000000,41911971.000000
+797175408.000000,1046847488.000000,1500964952.000000,17530880.000000,5720.850000,216401350656.000000,532622512128.000000,2579489.000000,934898.000000,24388988.000000,33843529.000000,78014866432.000000,857839866880.000000,16128444.000000,41924337.000000
+752652464.000000,1001832448.000000,1500964952.000000,17563648.000000,5731.530000,216268832768.000000,532622512128.000000,2579489.000000,934898.000000,24390009.000000,33843905.000000,78014866432.000000,858067031040.000000,16128964.000000,41931964.000000
+753659376.000000,1002897408.000000,1500964952.000000,17563648.000000,5731.620000,216268836864.000000,532622512128.000000,2579489.000000,934898.000000,24390069.000000,33843964.000000,78014866432.000000,858067518464.000000,16128988.000000,41932022.000000
+1007611960.000000,1216274432.000000,1500964952.000000,21561344.000000,5824.770000,216411131904.000000,532622512128.000000,2579489.000000,934898.000000,24393973.000000,33854536.000000,78014866432.000000,858377188352.000000,16133704.000000,41937748.000000
+909896688.000000,1173528576.000000,1530325080.000000,17956864.000000,5887.440000,216354177024.000000,532622512128.000000,2579489.000000,934898.000000,24396944.000000,33858743.000000,78014866432.000000,858845758464.000000,16135996.000000,41953151.000000
+601099344.000000,887398400.000000,1530325080.000000,17956864.000000,5896.570000,216265007104.000000,532622512128.000000,2579489.000000,934898.000000,24397670.000000,33858979.000000,78014866432.000000,859011412992.000000,16136400.000000,41956770.000000
+602074544.000000,888430592.000000,1530325080.000000,17956864.000000,5896.650000,216265007104.000000,532622512128.000000,2579489.000000,934898.000000,24397798.000000,33859025.000000,78014866432.000000,859012137984.000000,16136432.000000,41956888.000000
+897101312.000000,1099210752.000000,1530587224.000000,22544384.000000,5987.690000,216399233024.000000,532622512128.000000,2579489.000000,934898.000000,24401675.000000,33869678.000000,78014866432.000000,859319858176.000000,16141116.000000,41962644.000000
+779468568.000000,1037910016.000000,1530587224.000000,18382848.000000,6050.910000,216452333568.000000,532622512128.000000,2579489.000000,934898.000000,24404550.000000,33873899.000000,78014866432.000000,859770196992.000000,16143364.000000,41975158.000000
+608176264.000000,891797504.000000,1530587224.000000,18382848.000000,6065.770000,216282607616.000000,532622512128.000000,2579489.000000,934898.000000,24405905.000000,33874388.000000,78014866432.000000,860067116032.000000,16144012.000000,41984858.000000
+609149112.000000,892903424.000000,1530587224.000000,18382848.000000,6065.860000,216282611712.000000,532622512128.000000,2579489.000000,934898.000000,24405913.000000,33874409.000000,78014866432.000000,860067234816.000000,16144028.000000,41984865.000000
+1082462264.000000,1187504128.000000,1543301208.000000,22544384.000000,6162.430000,216430489600.000000,532622512128.000000,2579489.000000,934898.000000,24409870.000000,33885905.000000,78014866432.000000,860382004224.000000,16148828.000000,41990624.000000
diff --git a/test/stress/cases/istio/testdata/metrics/intermediate.csv
b/test/stress/cases/istio/testdata/metrics/intermediate.csv
new file mode 100644
index 00000000..ff82bf9e
--- /dev/null
+++ b/test/stress/cases/istio/testdata/metrics/intermediate.csv
@@ -0,0 +1,9 @@
+weighted_io,rate[5m],266.9066666666667,264.54,312.63,312.74333333333334,293.16333333333336,285.31333333333333,331.4766666666667,336.87,319.20666666666665,296.89666666666665,346.0133333333333,351.11,331.19666666666666,302.68333333333334,359.6,365.92333333333335,343.1933333333333,321.7133333333333,387.70666666666665,395.1566666666667,367.77666666666664,334.1166666666667,411.74333333333334,418.0733333333333,401.15,371.7866666666667,434.76666666666665,443.5,427.05333333333334,396.67333333333
[...]
+io_time,rate[5m],107.24,108.04,114.6,114.50666666666666,98.14666666666666,106.28,116.16,116.77333333333333,102.65333333333334,108.65333333333334,117.90666666666667,117.56,102.36,108.97333333333333,118.10666666666667,118.78666666666666,103.28,110.6,121.16,121.90666666666667,106.09333333333333,114.92,123.76,124.45333333333333,110.56,116.65333333333334,123.64,124.4,110.22666666666667,116.28,124.30666666666667,125.02666666666667,112.24,117.76,125.85333333333334,126.6,112.73333333333333,116.4
[...]
+write_bytes,rate[5m],12175837.866666667,12012612.266666668,13333094.4,13333585.92,12158730.24,11957725.866666667,13921949.013333334,14166507.52,13197967.36,12534565.546666667,14426821.973333333,14669742.08,13545117.013333334,12860197.546666667,14759867.733333332,15035924.48,13782016,13153962.666666666,15361597.44,15654475.093333334,14331671.893333333,13803874.986666666,16151975.253333334,16460513.28,15535786.666666666,14552814.933333334,16415989.76,16743669.76,15829688.32,14964162.56,168
[...]
+read_bytes,rate[5m],286.72,286.72,273.06666666666666,13.653333333333334,13.653333333333334,0,0,0,0,54.61333333333334,68.26666666666667,68.26666666666667,68.26666666666667,68.26666666666667,68.26666666666667,68.26666666666667,68.26666666666667,68.26666666666667,68.26666666666667,68.26666666666667,68.26666666666667,68.26666666666667,68.26666666666667,68.26666666666667,68.26666666666667,68.26666666666667,68.26666666666667,68.26666666666667,13.653333333333334,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
[...]
+merged_write_count,rate[5m],250.47333333333333,243.70333333333335,257.0133333333333,256.99333333333334,218.29333333333332,238.62,259.81666666666666,260.2866666666667,227.01333333333332,243.17333333333335,262.7133333333333,260.33666666666664,225.05,240.99666666666667,260.07666666666665,260.3933333333333,224.13666666666666,241.51333333333332,262.1566666666667,262.5366666666667,226.45666666666668,250.69666666666666,267.2733333333333,267.5833333333333,234.97333333333333,250.41666666666666,26
[...]
+write_count,rate[5m],115.44333333333333,111.93333333333334,119.84666666666666,119.96666666666667,105.55666666666667,109.98333333333333,122.14666666666666,123.21333333333334,110.99666666666667,113.17666666666666,124.44333333333333,125.16666666666667,111.15666666666667,112.98666666666666,124.77666666666667,126.03333333333333,111.66666666666667,114.67333333333333,127.41,128.81,114.18666666666667,118.22333333333333,130.96333333333334,132.25666666666666,120.60666666666667,121.47333333333333,1
[...]
+merged_read_count,rate[5m],0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0.6033333333333334,0.6033333333333334,0.6033333333333334,0.6033333333333334,0.6033333333333334,0.6033333333333334,0.6033333333333334,0.6033333333333334,0.6033333333333334,0.6033333333333334,0.6033333333333334,0.6033333333333334,0.6033333333333334,0.6033333333333334,0.6033333333333334,0.6033333333333334,0.6033333333333334,0.6033333333333334,0.6033333333333
[...]
+read_count,rate[5m],0.023333333333333334,0.023333333333333334,0.02,0.0033333333333333335,0.0033333333333333335,0,0,0,0,0.013333333333333334,0.016666666666666666,0.016666666666666666,0.016666666666666666,0.016666666666666666,0.016666666666666666,0.016666666666666666,0.016666666666666666,0.016666666666666666,0.016666666666666666,0.016666666666666666,0.016666666666666666,0.016666666666666666,0.016666666666666666,0.016666666666666666,0.016666666666666666,0.016666666666666666,0.01666666666666
[...]
+process_cpu_seconds_total,rate[5m],2.0383333333333336,2.0906,2.2426666666666666,2.2426999999999997,1.9484333333333337,2.0785666666666667,2.292966666666667,2.301233333333333,2.0336,2.1216,2.3249666666666666,2.336033333333333,2.0478000000000005,2.1414000000000004,2.342533333333333,2.3531666666666666,2.0532,2.1546666666666665,2.379433333333333,2.3916333333333335,2.086933333333333,2.2358000000000002,2.4389,2.4459,2.189133333333333,2.2752666666666665,2.4403333333333337,2.446,2.185666666666666
[...]
diff --git a/test/stress/cases/istio/testdata/metrics/result.csv
b/test/stress/cases/istio/testdata/metrics/result.csv
new file mode 100644
index 00000000..00b3dbed
--- /dev/null
+++ b/test/stress/cases/istio/testdata/metrics/result.csv
@@ -0,0 +1,16 @@
+Metric Name, Min, Max, Mean, Median, P90, P95, P98, P99
+go_memstats_alloc_bytes, 144503392.000000, 1252014544.000000,
533194949.707006, 515922896.000000, 863701068.000000, 1016009468.000000,
1060432244.000000, 1109959040.000000
+go_memstats_heap_inuse_bytes, 250183680.000000, 1309556736.000000,
701603318.216560, 709967872.000000, 1042378752.000000, 1105199104.000000,
1180516352.000000, 1211912192.000000
+go_memstats_sys_bytes, 1149135880.000000, 1543301208.000000,
1250226991.337580, 1171811336.000000, 1449237576.000000, 1515645016.000000,
1530587224.000000, 1530587224.000000
+go_memstats_stack_inuse_bytes, 2916352.000000, 22544384.000000,
11434571.006369, 11337728.000000, 17956864.000000, 18907136.000000,
20611072.000000, 22052864.000000
+process_cpu_seconds_total, 1.948433, 2.784233, 2.473533, 2.480917, 2.747350,
2.762550, 2.776167, 2.780333
+disk_used, 215987310592.000000, 216453967872.000000, 216290951611.515930,
216265412608.000000, 216422912000.000000, 216435509248.000000,
216448153600.000000, 216452474880.000000
+disk_total, 532622512128.000000, 532622512128.000000, 532622512128.000000,
532622512128.000000, 532622512128.000000, 532622512128.000000,
532622512128.000000, 532622512128.000000
+read_count, 0.000000, 0.856667, 0.145556, 0.016667, 0.856667, 0.856667,
0.856667, 0.856667
+merged_read_count, 0.000000, 0.603333, 0.083068, 0.000000, 0.603333, 0.603333,
0.603333, 0.603333
+write_count, 105.556667, 139.776667, 126.035966, 125.403333, 135.776667,
136.976667, 138.206667, 138.383333
+merged_write_count, 217.376667, 273.723333, 250.497319, 254.925000,
267.046667, 269.993333, 271.331667, 272.438333
+read_bytes, 0.000000, 13817.173333, 2019.605024, 68.266667, 13817.173333,
13817.173333, 13817.173333, 13817.173333
+write_bytes, 11957725.866667, 18027383.466667, 15585415.939710,
15669131.946667, 16918766.933333, 17439505.066667, 17689251.840000,
17725385.386667
+io_time, 98.146667, 126.600000, 117.581546, 117.793333, 124.973333,
125.560000, 126.260000, 126.300000
+weighted_io, 264.540000, 536.200000, 407.836111, 410.760000, 483.176667,
504.606667, 518.133333, 522.120000
