mrproliu commented on code in PR #1142:
URL:
https://github.com/apache/skywalking-banyandb/pull/1142#discussion_r3301279222
##########
banyand/trace/test_helper.go:
##########
@@ -18,64 +18,148 @@
package trace
import (
+ "fmt"
"time"
- "github.com/apache/skywalking-banyandb/pkg/convert"
+ "google.golang.org/protobuf/types/known/timestamppb"
+
+ databasev1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
"github.com/apache/skywalking-banyandb/pkg/fs"
- pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
-// CreateTestPartForDump creates a test trace part for testing the dump tool.
-// It returns the part path (directory) and cleanup function.
-func CreateTestPartForDump(tmpPath string, fileSystem fs.FileSystem) (string,
func()) {
- now := time.Now().UnixNano()
+// Timestamp marks a DumpTag value as a ValueType=Timestamp tag carrying nanos
+// since the epoch (trace stores its per-span time in such a tag).
+type Timestamp int64
- // Create test traces with various tag types
- traces := &traces{
- traceIDs: []string{"test-trace-1", "test-trace-1",
"test-trace-2"},
- timestamps: []int64{now, now + 1000, now + 2000},
- spanIDs: []string{"span-1", "span-2", "span-3"},
- spans: [][]byte{
- []byte("span-data-1-with-content"),
- []byte("span-data-2-with-content"),
- []byte("span-data-3-with-content"),
- },
- tags: [][]*tagValue{
- {
- {tag: "service.name", valueType:
pbv1.ValueTypeStr, value: []byte("test-service")},
- {tag: "http.status", valueType:
pbv1.ValueTypeInt64, value: convert.Int64ToBytes(200)},
- {tag: "timestamp", valueType:
pbv1.ValueTypeTimestamp, value: convert.Int64ToBytes(now)},
- {tag: "tags", valueType: pbv1.ValueTypeStrArr,
valueArr: [][]byte{[]byte("tag1"), []byte("tag2")}},
- {tag: "duration", valueType:
pbv1.ValueTypeInt64, value: convert.Int64ToBytes(1234567)},
+// DumpTag describes one (flat) tag of a trace span for the dump test builder.
+// Value is a Go-native value: string, int64, []string, []int64, []byte or
+// Timestamp.
+type DumpTag struct {
+ Value any
+ Name string
+}
+
+// DumpRow is one trace span for BuildPartForDump. SeriesID is derived from the
+// tags by the parser, so it is not specified here.
+type DumpRow struct {
+ TraceID string
+ SpanID string
+ Span []byte
+ Tags []DumpTag
+ Timestamp int64
+}
+
+// BuildPartForDump writes spans into a trace part at root/<partID>, returning
the
+// part directory, the rows for verification and a cleanup func.
+func BuildPartForDump(tmpPath string, fileSystem fs.FileSystem, partID uint64,
rows []DumpRow) (string, []DumpRow, func()) {
+ ts := &traces{}
+ for i := range rows {
+ r := &rows[i]
+ ts.traceIDs = append(ts.traceIDs, r.TraceID)
+ ts.timestamps = append(ts.timestamps, r.Timestamp)
+ ts.spanIDs = append(ts.spanIDs, r.SpanID)
+ ts.spans = append(ts.spans, r.Span)
+ ts.tags = append(ts.tags, buildDumpTags(r.Tags))
+ }
+
+ mp := &memPart{}
+ mp.mustInitFromTraces(ts)
+ path := partPath(tmpPath, partID)
+ mp.mustFlush(fileSystem, path)
+
+ return path, rows, func() {}
Review Comment:
Update to using `generateDataPoints()` and `releaseDataPoints` to generate
and release the data points.
##########
banyand/measure/test_helper.go:
##########
@@ -18,95 +18,201 @@
package measure
import (
+ "fmt"
+ "path/filepath"
"time"
"github.com/apache/skywalking-banyandb/api/common"
+ databasev1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
"github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/index"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
-// CreateTestPartForDump creates a test measure part for testing the dump tool.
-// It takes a temporary path and a file system as input, generates test data
points with various tag and field types,
-// creates a memory part, flushes it to disk, and returns the path to the
created part directory.
-// Parameters:
-//
-// tmpPath: the base directory where the part will be created.
-// fileSystem: the file system to use for writing the part.
-//
-// Returns:
-//
-// The path to the created part directory and a cleanup function.
-func CreateTestPartForDump(tmpPath string, fileSystem fs.FileSystem) (string,
func()) {
- now := time.Now().UnixNano()
+// DumpTag describes one tag of a measure row for the dump test builder. Value
is
+// a Go-native value (string, int64, []string, []int64 or []byte) encoded
through
+// the production encodeTagValue path.
+type DumpTag struct {
+ Value any
+ Family string
+ Name string
+}
- // Create test data points with various tag and field types
- dps := &dataPoints{
- seriesIDs: []common.SeriesID{1, 2, 3},
- timestamps: []int64{now, now + 1000, now + 2000},
- versions: []int64{1, 2, 3},
- tagFamilies: [][]nameValues{
- {
- {
- name: "arrTag",
- values: []*nameValue{
- {name: "strArrTag", valueType:
pbv1.ValueTypeStrArr, value: nil, valueArr: [][]byte{[]byte("value1"),
[]byte("value2")}},
- {name: "intArrTag", valueType:
pbv1.ValueTypeInt64Arr, value: nil, valueArr:
[][]byte{convert.Int64ToBytes(25), convert.Int64ToBytes(30)}},
- },
- },
- {
- name: "singleTag",
- values: []*nameValue{
- {name: "strTag", valueType:
pbv1.ValueTypeStr, value: []byte("test-value"), valueArr: nil},
- {name: "intTag", valueType:
pbv1.ValueTypeInt64, value: convert.Int64ToBytes(100), valueArr: nil},
- },
- },
- },
- {
- {
- name: "singleTag",
- values: []*nameValue{
- {name: "strTag1", valueType:
pbv1.ValueTypeStr, value: []byte("tag1"), valueArr: nil},
- {name: "strTag2", valueType:
pbv1.ValueTypeStr, value: []byte("tag2"), valueArr: nil},
- },
- },
+// DumpField describes one field of a measure row. Value is int64 or float64.
+type DumpField struct {
+ Value any
+ Name string
+}
+
+// DumpRow is one measure data point for BuildPartForDump. When Entity is set
the
+// SeriesID is derived as hash(Entity) and (optionally) written to smeta.bin.
+type DumpRow struct {
+ Entity string
+ Tags []DumpTag
+ Fields []DumpField
+ SeriesID common.SeriesID
+ Timestamp int64
+ Version int64
+}
+
+// BuildPartForDump writes rows into a measure part at root/<partID>, returning
+// the part directory, the rows (with resolved SeriesID) for verification and a
+// cleanup func. When writeSeriesMeta is true a matching smeta.bin is written
for
+// every row that carries an Entity.
+func BuildPartForDump(tmpPath string, fileSystem fs.FileSystem, partID uint64,
rows []DumpRow) (string, []DumpRow, func()) {
+ dps := &dataPoints{}
+ for i := range rows {
+ r := &rows[i]
+ if r.Entity != "" {
+ r.SeriesID =
common.SeriesID(convert.Hash([]byte(r.Entity)))
+ }
+ dps.seriesIDs = append(dps.seriesIDs, r.SeriesID)
+ dps.timestamps = append(dps.timestamps, r.Timestamp)
+ dps.versions = append(dps.versions, r.Version)
+ dps.tagFamilies = append(dps.tagFamilies,
buildDumpTagFamilies(r.Tags))
+ dps.fields = append(dps.fields, buildDumpFields(r.Fields))
+ }
+
+ mp := generateMemPart()
+ mp.mustInitFromDataPoints(dps)
+ path := partPath(tmpPath, partID)
+ mp.mustFlush(fileSystem, path)
+
+ return path, rows, func() {
+ releaseMemPart(mp)
Review Comment:
Update to using `generateDataPoints()` and `releaseDataPoints` to generate
and release the data points.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
