This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
The following commit(s) were added to refs/heads/main by this push:
new f8cb0194 Remove check requiring tags in criteria to be present in
projection (#876)
f8cb0194 is described below
commit f8cb019441e09a41db5dfda6af529a96cf3130de
Author: Gao Hongtao <[email protected]>
AuthorDate: Tue Dec 2 12:37:21 2025 +0800
Remove check requiring tags in criteria to be present in projection (#876)
* Refactor query engine to eliminate unnecessary validation for criteria
tags in projections.
---
CHANGES.md | 1 +
banyand/liaison/grpc/measure.go | 2 +-
banyand/liaison/grpc/stream.go | 2 +-
banyand/liaison/grpc/trace.go | 2 +-
pkg/query/logical/criteria_tags.go | 42 +++++
pkg/query/logical/hidden_tags.go | 158 +++++++++++++++++
pkg/query/logical/hidden_tags_test.go | 186 ++++++++++++++++++++
.../measure/measure_plan_indexscan_local.go | 64 ++++++-
.../measure/measure_plan_indexscan_local_test.go | 163 ++++++++++++++++++
pkg/query/logical/stream/stream_plan_tag_filter.go | 189 ++++++++++++++++++---
.../logical/stream/stream_plan_tag_filter_test.go | 152 +++++++++++++++++
test/cases/measure/data/input/filter_hidden_tag.ql | 22 +++
.../measure/data/input/filter_hidden_tag.yaml | 35 ++++
.../data/input/index_mode_filter_hidden_tag.ql | 22 +++
.../data/input/index_mode_filter_hidden_tag.yaml | 33 ++++
.../cases/measure/data/want/filter_hidden_tag.yaml | 126 ++++++++++++++
.../data/want/index_mode_filter_hidden_tag.yaml | 38 +++++
test/cases/measure/measure.go | 2 +
test/cases/stream/data/input/filter_hidden_tag.ql | 21 +++
.../cases/stream/data/input/filter_hidden_tag.yaml | 41 +++++
test/cases/stream/data/want/filter_hidden_tag.yaml | 43 +++++
test/cases/stream/stream.go | 1 +
22 files changed, 1310 insertions(+), 35 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index 9ce68fdd..32eb43ac 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -8,6 +8,7 @@ Release Notes.
- Remove Bloom filter for dictionary-encoded tags.
- Implement BanyanDB MCP.
+- Remove check requiring tags in criteria to be present in projection
### Bug Fixes
diff --git a/banyand/liaison/grpc/measure.go b/banyand/liaison/grpc/measure.go
index 17291452..355968d0 100644
--- a/banyand/liaison/grpc/measure.go
+++ b/banyand/liaison/grpc/measure.go
@@ -95,7 +95,7 @@ func (ms *measureService) Write(measure
measurev1.MeasureService_WriteServer) er
return nil
}
if err != nil {
- if !errors.Is(err, context.DeadlineExceeded) &&
!errors.Is(err, context.Canceled) {
+ if status.Code(err) != codes.Canceled &&
status.Code(err) != codes.DeadlineExceeded {
ms.l.Error().Err(err).Stringer("written",
writeRequest).Msg("failed to receive message")
}
return err
diff --git a/banyand/liaison/grpc/stream.go b/banyand/liaison/grpc/stream.go
index d0da8c6f..4b31874b 100644
--- a/banyand/liaison/grpc/stream.go
+++ b/banyand/liaison/grpc/stream.go
@@ -194,7 +194,7 @@ func (s *streamService) Write(stream
streamv1.StreamService_WriteServer) error {
return nil
}
if err != nil {
- if !errors.Is(err, context.DeadlineExceeded) &&
!errors.Is(err, context.Canceled) {
+ if status.Code(err) != codes.Canceled &&
status.Code(err) != codes.DeadlineExceeded {
s.l.Error().Stringer("written",
writeEntity).Err(err).Msg("failed to receive message")
}
return err
diff --git a/banyand/liaison/grpc/trace.go b/banyand/liaison/grpc/trace.go
index c4989652..89125a21 100644
--- a/banyand/liaison/grpc/trace.go
+++ b/banyand/liaison/grpc/trace.go
@@ -257,7 +257,7 @@ func (s *traceService) Write(stream
tracev1.TraceService_WriteServer) error {
return nil
}
if err != nil {
- if !errors.Is(err, context.DeadlineExceeded) &&
!errors.Is(err, context.Canceled) {
+ if status.Code(err) != codes.Canceled &&
status.Code(err) != codes.DeadlineExceeded {
s.l.Error().Stringer("written",
writeEntity).Err(err).Msg("failed to receive message")
}
return err
diff --git a/pkg/query/logical/criteria_tags.go
b/pkg/query/logical/criteria_tags.go
new file mode 100644
index 00000000..2712df1a
--- /dev/null
+++ b/pkg/query/logical/criteria_tags.go
@@ -0,0 +1,42 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package logical
+
+import modelv1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+
+// CollectCriteriaTagNames walks through the provided criteria expression and
+// records every referenced tag name into the supplied map. The map acts as a
set
+// that deduplicates tag names for callers that need to know all tags used in a
+// criteria tree.
+func CollectCriteriaTagNames(criteria *modelv1.Criteria, tags
map[string]struct{}) {
+ if criteria == nil || tags == nil {
+ return
+ }
+ switch exp := criteria.GetExp().(type) {
+ case *modelv1.Criteria_Condition:
+ if cond := criteria.GetCondition(); cond != nil {
+ tags[cond.GetName()] = struct{}{}
+ }
+ case *modelv1.Criteria_Le:
+ if exp.Le == nil {
+ return
+ }
+ CollectCriteriaTagNames(exp.Le.Left, tags)
+ CollectCriteriaTagNames(exp.Le.Right, tags)
+ }
+}
diff --git a/pkg/query/logical/hidden_tags.go b/pkg/query/logical/hidden_tags.go
new file mode 100644
index 00000000..bd49bcbe
--- /dev/null
+++ b/pkg/query/logical/hidden_tags.go
@@ -0,0 +1,158 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package logical
+
+import (
+ modelv1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+)
+
+// HiddenTagSet is a set of tag names that should be stripped from query
results.
+// These are tags that were used in criteria but not requested in the
projection.
+type HiddenTagSet map[string]struct{}
+
+// NewHiddenTagSet creates a new HiddenTagSet.
+func NewHiddenTagSet() HiddenTagSet {
+ return make(map[string]struct{})
+}
+
+// Add adds a tag name to the hidden set.
+func (h HiddenTagSet) Add(tagName string) {
+ h[tagName] = struct{}{}
+}
+
+// Contains checks if a tag name is in the hidden set.
+func (h HiddenTagSet) Contains(tagName string) bool {
+ _, ok := h[tagName]
+ return ok
+}
+
+// IsEmpty returns true if the hidden set has no entries.
+func (h HiddenTagSet) IsEmpty() bool {
+ return len(h) == 0
+}
+
+// StripHiddenTags removes hidden tags from a slice of tag families.
+// It modifies the slice in place and returns the filtered result.
+// Empty tag families are removed from the result.
+func (h HiddenTagSet) StripHiddenTags(tagFamilies []*modelv1.TagFamily)
[]*modelv1.TagFamily {
+ if h.IsEmpty() || len(tagFamilies) == 0 {
+ return tagFamilies
+ }
+
+ families := tagFamilies[:0]
+ for _, tf := range tagFamilies {
+ if tf == nil {
+ continue
+ }
+ tags := tf.Tags[:0]
+ for _, tag := range tf.Tags {
+ if tag == nil {
+ continue
+ }
+ if h.Contains(tag.GetKey()) {
+ continue
+ }
+ tags = append(tags, tag)
+ }
+ if len(tags) == 0 {
+ continue
+ }
+ tf.Tags = tags
+ families = append(families, tf)
+ }
+ return families
+}
+
+// CollectHiddenCriteriaTags identifies tags used in criteria that are not in
the projection.
+// It returns a HiddenTagSet containing those tag names and an optional slice
of Tag
+// structures grouped by family for adding to the projection.
+//
+// Parameters:
+// - criteria: The query criteria containing filter conditions
+// - projectedTagNames: Set of tag names already in the projection
+// - entityDict: Map of entity tag names (these are skipped)
+// - schema: The schema to look up tag family information
+// - tagFamilyGetter: Function to get tag families from the schema
+//
+// Returns:
+// - HiddenTagSet: Set of hidden tag names
+// - [][]*Tag: Tags grouped by family to add to projection (nil if empty)
+func CollectHiddenCriteriaTags(
+ criteria *modelv1.Criteria,
+ projectedTagNames map[string]struct{},
+ entityDict map[string]int,
+ schema Schema,
+ tagFamilyGetter func() []string,
+) (HiddenTagSet, [][]*Tag) {
+ hidden := NewHiddenTagSet()
+ if criteria == nil {
+ return hidden, nil
+ }
+
+ // Collect all tag names from criteria
+ tagNames := make(map[string]struct{})
+ CollectCriteriaTagNames(criteria, tagNames)
+
+ // Group tags by family
+ familyTags := make(map[string][]*Tag)
+ var familyOrder []string
+
+ tagFamilies := tagFamilyGetter()
+
+ for tagName := range tagNames {
+ // Skip if already in projection
+ if _, ok := projectedTagNames[tagName]; ok {
+ continue
+ }
+ // Skip entity tags
+ if _, isEntity := entityDict[tagName]; isEntity {
+ continue
+ }
+
+ tagSpec := schema.FindTagSpecByName(tagName)
+ if tagSpec == nil {
+ continue
+ }
+
+ // Get the actual family name from the schema
+ if tagSpec.TagFamilyIdx < 0 || tagSpec.TagFamilyIdx >=
len(tagFamilies) {
+ continue
+ }
+ familyName := tagFamilies[tagSpec.TagFamilyIdx]
+
+ // Mark as hidden
+ hidden.Add(tagName)
+
+ // Group tags by family
+ if _, exists := familyTags[familyName]; !exists {
+ familyOrder = append(familyOrder, familyName)
+ familyTags[familyName] = make([]*Tag, 0)
+ }
+ familyTags[familyName] = append(familyTags[familyName],
NewTag(familyName, tagName))
+ }
+
+ // Convert to [][]*Tag maintaining family grouping
+ if len(familyOrder) == 0 {
+ return hidden, nil
+ }
+ extras := make([][]*Tag, 0, len(familyOrder))
+ for _, family := range familyOrder {
+ extras = append(extras, familyTags[family])
+ }
+ return hidden, extras
+}
diff --git a/pkg/query/logical/hidden_tags_test.go
b/pkg/query/logical/hidden_tags_test.go
new file mode 100644
index 00000000..82881f8d
--- /dev/null
+++ b/pkg/query/logical/hidden_tags_test.go
@@ -0,0 +1,186 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package logical
+
+import (
+ "testing"
+
+ modelv1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+)
+
+func TestHiddenTagSet(t *testing.T) {
+ t.Run("NewHiddenTagSet creates empty set", func(t *testing.T) {
+ h := NewHiddenTagSet()
+ if !h.IsEmpty() {
+ t.Error("expected new HiddenTagSet to be empty")
+ }
+ })
+
+ t.Run("Add and Contains", func(t *testing.T) {
+ h := NewHiddenTagSet()
+ h.Add("tag1")
+ h.Add("tag2")
+
+ if !h.Contains("tag1") {
+ t.Error("expected tag1 to be in set")
+ }
+ if !h.Contains("tag2") {
+ t.Error("expected tag2 to be in set")
+ }
+ if h.Contains("tag3") {
+ t.Error("expected tag3 not to be in set")
+ }
+ if h.IsEmpty() {
+ t.Error("expected set not to be empty after adding
tags")
+ }
+ })
+
+ t.Run("StripHiddenTags removes hidden tags", func(t *testing.T) {
+ h := NewHiddenTagSet()
+ h.Add("hidden1")
+ h.Add("hidden2")
+
+ families := []*modelv1.TagFamily{
+ {
+ Name: "family1",
+ Tags: []*modelv1.Tag{
+ {Key: "visible", Value:
&modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value:
"keep"}}}},
+ {Key: "hidden1", Value:
&modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value:
"drop"}}}},
+ },
+ },
+ {
+ Name: "family2",
+ Tags: []*modelv1.Tag{
+ {Key: "hidden2", Value:
&modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value:
"drop"}}}},
+ },
+ },
+ {
+ Name: "family3",
+ Tags: []*modelv1.Tag{
+ {Key: "also_visible", Value:
&modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value:
"keep"}}}},
+ },
+ },
+ }
+
+ result := h.StripHiddenTags(families)
+
+ // family2 should be removed entirely because all tags are
hidden
+ if len(result) != 2 {
+ t.Fatalf("expected 2 families after stripping, got %d",
len(result))
+ }
+
+ // Check family1
+ if result[0].Name != "family1" {
+ t.Errorf("expected first family to be 'family1', got
%s", result[0].Name)
+ }
+ if len(result[0].Tags) != 1 {
+ t.Fatalf("expected 1 tag in family1, got %d",
len(result[0].Tags))
+ }
+ if result[0].Tags[0].Key != "visible" {
+ t.Errorf("expected tag 'visible' in family1, got %s",
result[0].Tags[0].Key)
+ }
+
+ // Check family3
+ if result[1].Name != "family3" {
+ t.Errorf("expected second family to be 'family3', got
%s", result[1].Name)
+ }
+ if len(result[1].Tags) != 1 {
+ t.Fatalf("expected 1 tag in family3, got %d",
len(result[1].Tags))
+ }
+ if result[1].Tags[0].Key != "also_visible" {
+ t.Errorf("expected tag 'also_visible' in family3, got
%s", result[1].Tags[0].Key)
+ }
+ })
+
+ t.Run("StripHiddenTags with empty hidden set returns unchanged", func(t
*testing.T) {
+ h := NewHiddenTagSet()
+ families := []*modelv1.TagFamily{
+ {
+ Name: "family1",
+ Tags: []*modelv1.Tag{
+ {Key: "tag1", Value:
&modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value:
"value"}}}},
+ },
+ },
+ }
+
+ result := h.StripHiddenTags(families)
+
+ if len(result) != 1 {
+ t.Fatalf("expected 1 family, got %d", len(result))
+ }
+ if len(result[0].Tags) != 1 {
+ t.Fatalf("expected 1 tag, got %d", len(result[0].Tags))
+ }
+ })
+
+ t.Run("StripHiddenTags with nil families returns nil", func(t
*testing.T) {
+ h := NewHiddenTagSet()
+ h.Add("hidden")
+
+ result := h.StripHiddenTags(nil)
+
+ if result != nil {
+ t.Errorf("expected nil result for nil input, got %v",
result)
+ }
+ })
+
+ t.Run("StripHiddenTags handles nil tag families", func(t *testing.T) {
+ h := NewHiddenTagSet()
+ h.Add("hidden")
+
+ families := []*modelv1.TagFamily{
+ nil,
+ {
+ Name: "family1",
+ Tags: []*modelv1.Tag{
+ {Key: "visible", Value:
&modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value:
"keep"}}}},
+ },
+ },
+ }
+
+ result := h.StripHiddenTags(families)
+
+ if len(result) != 1 {
+ t.Fatalf("expected 1 family after stripping nil, got
%d", len(result))
+ }
+ })
+
+ t.Run("StripHiddenTags handles nil tags", func(t *testing.T) {
+ h := NewHiddenTagSet()
+ h.Add("hidden")
+
+ families := []*modelv1.TagFamily{
+ {
+ Name: "family1",
+ Tags: []*modelv1.Tag{
+ nil,
+ {Key: "visible", Value:
&modelv1.TagValue{Value: &modelv1.TagValue_Str{Str: &modelv1.Str{Value:
"keep"}}}},
+ },
+ },
+ }
+
+ result := h.StripHiddenTags(families)
+
+ if len(result) != 1 {
+ t.Fatalf("expected 1 family, got %d", len(result))
+ }
+ if len(result[0].Tags) != 1 {
+ t.Fatalf("expected 1 tag after stripping nil, got %d",
len(result[0].Tags))
+ }
+ })
+}
diff --git a/pkg/query/logical/measure/measure_plan_indexscan_local.go
b/pkg/query/logical/measure/measure_plan_indexscan_local.go
index c5fc25ac..08c57449 100644
--- a/pkg/query/logical/measure/measure_plan_indexscan_local.go
+++ b/pkg/query/logical/measure/measure_plan_indexscan_local.go
@@ -53,16 +53,24 @@ type unresolvedIndexScan struct {
func (uis *unresolvedIndexScan) Analyze(s logical.Schema) (logical.Plan,
error) {
projTags := make([]model.TagProjection, len(uis.projectionTags))
+ projectedTagNames := make(map[string]struct{})
var projTagsRefs [][]*logical.TagRef
if len(uis.projectionTags) > 0 {
for i := range uis.projectionTags {
for _, tag := range uis.projectionTags[i] {
projTags[i].Family = tag.GetFamilyName()
projTags[i].Names = append(projTags[i].Names,
tag.GetTagName())
+ projectedTagNames[tag.GetTagName()] = struct{}{}
}
}
+ }
+ tagRefInput := make([][]*logical.Tag, 0, len(uis.projectionTags))
+ tagRefInput = append(tagRefInput, uis.projectionTags...)
+
+ // Create tag refs for projection tags before any early returns
+ if len(tagRefInput) > 0 {
var err error
- projTagsRefs, err = s.CreateTagRef(uis.projectionTags...)
+ projTagsRefs, err = s.CreateTagRef(tagRefInput...)
if err != nil {
return nil, err
}
@@ -112,6 +120,31 @@ func (uis *unresolvedIndexScan) Analyze(s logical.Schema)
(logical.Plan, error)
// fill AnyEntry by default
entity[idx] = pbv1.AnyTagValue
}
+
+ // Collect hidden criteria tags using shared utility
+ tagFamilies := ms.measure.GetTagFamilies()
+ hiddenTags, extraTags := logical.CollectHiddenCriteriaTags(
+ uis.criteria,
+ projectedTagNames,
+ entityMap,
+ s,
+ func() []string {
+ names := make([]string, len(tagFamilies))
+ for i, tf := range tagFamilies {
+ names[i] = tf.GetName()
+ }
+ return names
+ },
+ )
+ if len(extraTags) > 0 {
+ tagRefInput = append(tagRefInput, extraTags...)
+ // Recreate tag refs with both projection tags and hidden
criteria tags
+ var err error
+ projTagsRefs, err = s.CreateTagRef(tagRefInput...)
+ if err != nil {
+ return nil, err
+ }
+ }
query, entities, _, err := inverted.BuildQuery(uis.criteria, s,
entityMap, entity)
if err != nil {
return nil, err
@@ -128,6 +161,7 @@ func (uis *unresolvedIndexScan) Analyze(s logical.Schema)
(logical.Plan, error)
query: query,
entities: entities,
groupByEntity: uis.groupByEntity,
+ hiddenTags: hiddenTags,
uis: uis,
l: logger.GetLogger("query", "measure",
uis.metadata.Group, uis.metadata.Name, "local-index"),
ec: uis.ec,
@@ -147,6 +181,7 @@ type localIndexScan struct {
order *logical.OrderBy
metadata *commonv1.Metadata
l *logger.Logger
+ hiddenTags logical.HiddenTagSet
timeRange timestamp.TimeRange
projectionTagsRefs [][]*logical.TagRef
projectionFieldsRefs []*logical.FieldRef
@@ -195,7 +230,8 @@ func (i *localIndexScan) Execute(ctx context.Context) (mit
executor.MIterator, e
return nil, fmt.Errorf("failed to query measure: %w", err)
}
return &resultMIterator{
- result: result,
+ result: result,
+ hiddenTags: i.hiddenTags,
}, nil
}
@@ -210,10 +246,16 @@ func (i *localIndexScan) Children() []logical.Plan {
}
func (i *localIndexScan) Schema() logical.Schema {
- if len(i.projectionTagsRefs) == 0 {
- return i.schema
+ schema := i.schema
+ if len(i.projectionTagsRefs) > 0 {
+ if projected := schema.ProjTags(i.projectionTagsRefs...);
projected != nil {
+ schema = projected
+ }
+ }
+ if len(i.projectionFieldsRefs) > 0 {
+ schema = schema.ProjFields(i.projectionFieldsRefs...)
}
- return
i.schema.ProjTags(i.projectionTagsRefs...).ProjFields(i.projectionFieldsRefs...)
+ return schema
}
func indexScan(startTime, endTime time.Time, metadata *commonv1.Metadata,
projectionTags [][]*logical.Tag,
@@ -232,10 +274,11 @@ func indexScan(startTime, endTime time.Time, metadata
*commonv1.Metadata, projec
}
type resultMIterator struct {
- result model.MeasureQueryResult
- err error
- current []*measurev1.DataPoint
- i int
+ result model.MeasureQueryResult
+ hiddenTags logical.HiddenTagSet
+ err error
+ current []*measurev1.DataPoint
+ i int
}
func (ei *resultMIterator) Next() bool {
@@ -276,6 +319,9 @@ func (ei *resultMIterator) Next() bool {
})
}
}
+ // Strip hidden tags from the result
+ dp.TagFamilies = ei.hiddenTags.StripHiddenTags(dp.TagFamilies)
+
for _, f := range r.Fields {
dp.Fields = append(dp.Fields,
&measurev1.DataPoint_Field{
Name: f.Name,
diff --git a/pkg/query/logical/measure/measure_plan_indexscan_local_test.go
b/pkg/query/logical/measure/measure_plan_indexscan_local_test.go
new file mode 100644
index 00000000..d83a8bf4
--- /dev/null
+++ b/pkg/query/logical/measure/measure_plan_indexscan_local_test.go
@@ -0,0 +1,163 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "testing"
+ "time"
+
+ commonv1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ "github.com/apache/skywalking-banyandb/pkg/query/logical"
+)
+
+func TestLocalIndexScanSchemaRetainsNonProjectedTags(t *testing.T) {
+ measureMeta := &databasev1.Measure{
+ Entity: &databasev1.Entity{
+ TagNames: []string{"entity_id"},
+ },
+ TagFamilies: []*databasev1.TagFamilySpec{
+ {
+ Name: "default",
+ Tags: []*databasev1.TagSpec{
+ {Name: "filter_tag", Type:
databasev1.TagType_TAG_TYPE_STRING},
+ {Name: "projected_tag", Type:
databasev1.TagType_TAG_TYPE_STRING},
+ },
+ },
+ },
+ }
+ indexRules := []*databasev1.IndexRule{
+ {
+ Metadata: &commonv1.Metadata{Name: "filter-tag-rule"},
+ Type: databasev1.IndexRule_TYPE_INVERTED,
+ Tags: []string{"filter_tag"},
+ },
+ }
+ schema, err := BuildSchema(measureMeta, indexRules)
+ if err != nil {
+ t.Fatalf("build schema: %v", err)
+ }
+
+ criteria := &modelv1.Criteria{
+ Exp: &modelv1.Criteria_Condition{
+ Condition: &modelv1.Condition{
+ Name: "filter_tag",
+ Op: modelv1.Condition_BINARY_OP_EQ,
+ Value: &modelv1.TagValue{
+ Value: &modelv1.TagValue_Str{
+ Str: &modelv1.Str{Value:
"match"},
+ },
+ },
+ },
+ },
+ }
+ metadata := &commonv1.Metadata{Name: "test", Group: "default"}
+
+ plan, err := indexScan(
+ time.Unix(0, 0),
+ time.Unix(1, 0),
+ metadata,
+ [][]*logical.Tag{logical.NewTags("default", "projected_tag")},
+ nil,
+ false,
+ criteria,
+ nil,
+ ).Analyze(schema)
+ if err != nil {
+ t.Fatalf("analyze plan: %v", err)
+ }
+
+ if plan.Schema().FindTagSpecByName("filter_tag") == nil {
+ t.Fatalf("expected schema to retain definition for filter_tag
even when it is not projected")
+ }
+}
+
+func TestLocalIndexScanSchemaProjectionInIndexMode(t *testing.T) {
+ measureMeta := &databasev1.Measure{
+ Entity: &databasev1.Entity{
+ TagNames: []string{"entity_id"},
+ },
+ TagFamilies: []*databasev1.TagFamilySpec{
+ {
+ Name: "default",
+ Tags: []*databasev1.TagSpec{
+ {Name: "entity_id", Type:
databasev1.TagType_TAG_TYPE_STRING},
+ {Name: "filter_tag", Type:
databasev1.TagType_TAG_TYPE_STRING},
+ {Name: "projected_tag", Type:
databasev1.TagType_TAG_TYPE_STRING},
+ {Name: "non_projected_tag", Type:
databasev1.TagType_TAG_TYPE_STRING},
+ },
+ },
+ },
+ IndexMode: true,
+ }
+ indexRules := []*databasev1.IndexRule{
+ {
+ Metadata: &commonv1.Metadata{Name: "filter-tag-rule"},
+ Type: databasev1.IndexRule_TYPE_INVERTED,
+ Tags: []string{"filter_tag"},
+ },
+ }
+ schema, err := BuildSchema(measureMeta, indexRules)
+ if err != nil {
+ t.Fatalf("build schema: %v", err)
+ }
+
+ criteria := &modelv1.Criteria{
+ Exp: &modelv1.Criteria_Condition{
+ Condition: &modelv1.Condition{
+ Name: "filter_tag",
+ Op: modelv1.Condition_BINARY_OP_EQ,
+ Value: &modelv1.TagValue{
+ Value: &modelv1.TagValue_Str{
+ Str: &modelv1.Str{Value:
"match"},
+ },
+ },
+ },
+ },
+ }
+ metadata := &commonv1.Metadata{Name: "test", Group: "default"}
+
+ plan, err := indexScan(
+ time.Unix(0, 0),
+ time.Unix(1, 0),
+ metadata,
+ [][]*logical.Tag{logical.NewTags("default", "projected_tag")},
+ nil,
+ false,
+ criteria,
+ nil,
+ ).Analyze(schema)
+ if err != nil {
+ t.Fatalf("analyze plan: %v", err)
+ }
+
+ projectedSchema := plan.Schema()
+
+ // Verify that projected_tag is present in the schema
+ if projectedSchema.FindTagSpecByName("projected_tag") == nil {
+ t.Fatalf("expected schema to include projected_tag")
+ }
+
+ // Verify that non_projected_tag is NOT present in the projected schema
+ // This is the key test - without the fix, projectedSchema would be the
full schema
+ // and non_projected_tag would incorrectly be present
+ if projectedSchema.FindTagSpecByName("non_projected_tag") != nil {
+ t.Fatalf("expected schema to exclude non_projected_tag when
using projection")
+ }
+}
diff --git a/pkg/query/logical/stream/stream_plan_tag_filter.go
b/pkg/query/logical/stream/stream_plan_tag_filter.go
index a80e4b6c..ee0533bf 100644
--- a/pkg/query/logical/stream/stream_plan_tag_filter.go
+++ b/pkg/query/logical/stream/stream_plan_tag_filter.go
@@ -66,31 +66,52 @@ func (uis *unresolvedTagFilter) Analyze(s logical.Schema)
(logical.Plan, error)
return nil, err
}
- projTags := make([]model.TagProjection, len(uis.projectionTags))
- if len(uis.projectionTags) > 0 {
- for i := range uis.projectionTags {
- for _, tag := range uis.projectionTags[i] {
- projTags[i].Family = tag.GetFamilyName()
- projTags[i].Names = append(projTags[i].Names,
tag.GetTagName())
+ projBuilder := newProjectionBuilder(uis.projectionTags)
+ criteriaTagNames := make(map[string]struct{})
+ logical.CollectCriteriaTagNames(uis.criteria, criteriaTagNames)
+
+ hiddenTags := logical.NewHiddenTagSet()
+ var tagFilter logical.TagFilter
+ if uis.criteria != nil {
+ var errFilter error
+ tagFilter, errFilter = logical.BuildTagFilter(uis.criteria,
entityDict, s, s, false, "")
+ if errFilter != nil {
+ return nil, errFilter
+ }
+ if tagFilter != logical.DummyFilter {
+ for tagName := range criteriaTagNames {
+ if _, isEntity := entityDict[tagName]; isEntity
{
+ continue
+ }
+ added, errAdd :=
projBuilder.AddTagFromSchema(s, tagName)
+ if errAdd != nil {
+ return nil, errAdd
+ }
+ if added {
+ hiddenTags.Add(tagName)
+ }
}
}
+ }
+
+ ctx.projectionTags = projBuilder.Model()
+ if logicalTags := projBuilder.LogicalTags(); len(logicalTags) > 0 {
var errProject error
- ctx.projTagsRefs, errProject =
s.CreateTagRef(uis.projectionTags...)
+ ctx.projTagsRefs, errProject = s.CreateTagRef(logicalTags...)
if errProject != nil {
return nil, errProject
}
}
- ctx.projectionTags = projTags
+
plan := uis.selectIndexScanner(ctx, uis.ec)
- if uis.criteria != nil {
- tagFilter, errFilter := logical.BuildTagFilter(uis.criteria,
entityDict, s, s, false, "")
- if errFilter != nil {
- return nil, errFilter
- }
- if tagFilter != logical.DummyFilter {
- // create tagFilter with a projected view
- plan = newTagFilter(s.ProjTags(ctx.projTagsRefs...),
plan, tagFilter)
+ if uis.criteria != nil && tagFilter != nil && tagFilter !=
logical.DummyFilter {
+ projSchema := s
+ if len(ctx.projTagsRefs) > 0 {
+ if projected := s.ProjTags(ctx.projTagsRefs...);
projected != nil {
+ projSchema = projected
+ }
}
+ plan = newTagFilter(projSchema, plan, tagFilter, hiddenTags)
}
return plan, err
}
@@ -144,20 +165,22 @@ var (
)
type tagFilterPlan struct {
- s logical.Schema
- parent logical.Plan
- tagFilter logical.TagFilter
+ s logical.Schema
+ parent logical.Plan
+ tagFilter logical.TagFilter
+ hiddenTags logical.HiddenTagSet
}
func (t *tagFilterPlan) Close() {
t.parent.(executor.StreamExecutable).Close()
}
-func newTagFilter(s logical.Schema, parent logical.Plan, tagFilter
logical.TagFilter) logical.Plan {
+func newTagFilter(s logical.Schema, parent logical.Plan, tagFilter
logical.TagFilter, hiddenTags logical.HiddenTagSet) logical.Plan {
return &tagFilterPlan{
- s: s,
- parent: parent,
- tagFilter: tagFilter,
+ s: s,
+ parent: parent,
+ tagFilter: tagFilter,
+ hiddenTags: hiddenTags,
}
}
@@ -178,6 +201,8 @@ func (t *tagFilterPlan) Execute(ec context.Context)
([]*streamv1.Element, error)
return nil, err
}
if ok {
+ // Strip hidden tags using shared utility
+ e.TagFamilies =
t.hiddenTags.StripHiddenTags(e.TagFamilies)
filteredElements = append(filteredElements, e)
}
}
@@ -200,3 +225,121 @@ func (t *tagFilterPlan) Children() []logical.Plan {
func (t *tagFilterPlan) Schema() logical.Schema {
return t.s
}
+
+type projectionBuilder struct {
+ familyTags map[string][]string
+ tagSet map[string]struct{}
+ familyOrder []string
+}
+
+func newProjectionBuilder(existing [][]*logical.Tag) *projectionBuilder {
+ pb := &projectionBuilder{
+ familyTags: make(map[string][]string),
+ tagSet: make(map[string]struct{}),
+ }
+ for _, tags := range existing {
+ if len(tags) == 0 {
+ continue
+ }
+ family := tags[0].GetFamilyName()
+ pb.ensureFamily(family)
+ for _, tag := range tags {
+ pb.addTag(family, tag.GetTagName())
+ }
+ }
+ return pb
+}
+
+func (pb *projectionBuilder) HasTag(tagName string) bool {
+ _, ok := pb.tagSet[tagName]
+ return ok
+}
+
+func (pb *projectionBuilder) addTag(family, tagName string) {
+ if tagName == "" || pb.HasTag(tagName) {
+ return
+ }
+ pb.ensureFamily(family)
+ pb.familyTags[family] = append(pb.familyTags[family], tagName)
+ pb.tagSet[tagName] = struct{}{}
+}
+
+func (pb *projectionBuilder) ensureFamily(family string) {
+ if _, ok := pb.familyTags[family]; ok {
+ return
+ }
+ pb.familyOrder = append(pb.familyOrder, family)
+ pb.familyTags[family] = make([]string, 0)
+}
+
+func (pb *projectionBuilder) AddTagFromSchema(s logical.Schema, tagName
string) (bool, error) {
+ if pb.HasTag(tagName) {
+ return false, nil
+ }
+ tagSpec := s.FindTagSpecByName(tagName)
+ if tagSpec == nil {
+ return false, fmt.Errorf("tag %s not defined in schema",
tagName)
+ }
+ family, ok := familyNameFromSchema(s, tagSpec)
+ if !ok {
+ return false, fmt.Errorf("tag family not found for %s", tagName)
+ }
+ pb.addTag(family, tagName)
+ return true, nil
+}
+
+func (pb *projectionBuilder) Model() []model.TagProjection {
+ if len(pb.familyOrder) == 0 {
+ return nil
+ }
+ projections := make([]model.TagProjection, 0, len(pb.familyOrder))
+ for _, family := range pb.familyOrder {
+ names := pb.familyTags[family]
+ if len(names) == 0 {
+ continue
+ }
+ proj := model.TagProjection{
+ Family: family,
+ Names: append([]string(nil), names...),
+ }
+ projections = append(projections, proj)
+ }
+ if len(projections) == 0 {
+ return nil
+ }
+ return projections
+}
+
+func (pb *projectionBuilder) LogicalTags() [][]*logical.Tag {
+ if len(pb.familyOrder) == 0 {
+ return nil
+ }
+ logicalTags := make([][]*logical.Tag, 0, len(pb.familyOrder))
+ for _, family := range pb.familyOrder {
+ names := pb.familyTags[family]
+ if len(names) == 0 {
+ continue
+ }
+ familyTags := make([]*logical.Tag, 0, len(names))
+ for _, name := range names {
+ familyTags = append(familyTags, logical.NewTag(family,
name))
+ }
+ logicalTags = append(logicalTags, familyTags)
+ }
+ if len(logicalTags) == 0 {
+ return nil
+ }
+ return logicalTags
+}
+
+func familyNameFromSchema(s logical.Schema, tagSpec *logical.TagSpec) (string,
bool) {
+ streamSchema, ok := s.(*schema)
+ if !ok || streamSchema == nil || streamSchema.stream == nil {
+ return "", false
+ }
+ tagFamilies := streamSchema.stream.GetTagFamilies()
+ if tagSpec.TagFamilyIdx < 0 || tagSpec.TagFamilyIdx >= len(tagFamilies)
{
+ return "", false
+ }
+ return tagFamilies[tagSpec.TagFamilyIdx].GetName(), true
+}
diff --git a/pkg/query/logical/stream/stream_plan_tag_filter_test.go
b/pkg/query/logical/stream/stream_plan_tag_filter_test.go
new file mode 100644
index 00000000..d2e68f63
--- /dev/null
+++ b/pkg/query/logical/stream/stream_plan_tag_filter_test.go
@@ -0,0 +1,152 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package stream
+
+import (
+ "testing"
+ "time"
+
+ commonv1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ modelv1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ streamv1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
+ "github.com/apache/skywalking-banyandb/pkg/query/logical"
+)
+
+func TestAnalyzeAddsHiddenCriteriaTags(t *testing.T) {
+ schema := mustBuildTestStreamSchema(t)
+ metadata := &commonv1.Metadata{Name: "svc", Group: "default"}
+ plan := tagFilter(
+ time.Unix(0, 0),
+ time.Unix(1, 0),
+ metadata,
+ buildEqualityCriteria("filter_tag", "match"),
+ [][]*logical.Tag{logical.NewTags("default", "projected_tag")},
+ nil,
+ )
+ resolved, err := plan.Analyze(schema)
+ if err != nil {
+ t.Fatalf("analyze tag filter: %v", err)
+ }
+
+ tfPlan, ok := resolved.(*tagFilterPlan)
+ if !ok {
+ t.Fatalf("expected tagFilterPlan, got %T", resolved)
+ }
+ if !tfPlan.hiddenTags.Contains("filter_tag") {
+ t.Fatalf("expected filter_tag to be tracked as hidden")
+ }
+
+ parent, ok := tfPlan.parent.(*localIndexScan)
+ if !ok {
+ t.Fatalf("expected localIndexScan as parent, got %T",
tfPlan.parent)
+ }
+ found := false
+ for _, family := range parent.projectionTags {
+ for _, tagName := range family.Names {
+ if tagName == "filter_tag" {
+ found = true
+ break
+ }
+ }
+ }
+ if !found {
+ t.Fatalf("expected filter_tag to be added to projection for
evaluation")
+ }
+}
+
+func TestStripHiddenTagsRemovesSensitiveValues(t *testing.T) {
+ hiddenTags := logical.NewHiddenTagSet()
+ hiddenTags.Add("hidden")
+
+ element := &streamv1.Element{
+ TagFamilies: []*modelv1.TagFamily{
+ {
+ Name: "default",
+ Tags: []*modelv1.Tag{
+ {Key: "visible", Value:
buildStringValue("keep")},
+ {Key: "hidden", Value:
buildStringValue("drop")},
+ },
+ },
+ {
+ Name: "second",
+ Tags: []*modelv1.Tag{
+ {Key: "hidden", Value:
buildStringValue("drop")},
+ },
+ },
+ },
+ }
+
+ // Use the shared utility to strip hidden tags
+ element.TagFamilies = hiddenTags.StripHiddenTags(element.TagFamilies)
+
+ if len(element.GetTagFamilies()) != 1 {
+ t.Fatalf("expected only one tag family after stripping, got
%d", len(element.GetTagFamilies()))
+ }
+ remainingTags := element.GetTagFamilies()[0].GetTags()
+ if len(remainingTags) != 1 || remainingTags[0].GetKey() != "visible" {
+ t.Fatalf("unexpected tags remaining after stripping: %+v",
remainingTags)
+ }
+}
+
+func mustBuildTestStreamSchema(t *testing.T) logical.Schema {
+ t.Helper()
+ stream := &databasev1.Stream{
+ Entity: &databasev1.Entity{
+ TagNames: []string{"entity_id"},
+ },
+ TagFamilies: []*databasev1.TagFamilySpec{
+ {
+ Name: "default",
+ Tags: []*databasev1.TagSpec{
+ {Name: "filter_tag", Type:
databasev1.TagType_TAG_TYPE_STRING},
+ {Name: "projected_tag", Type:
databasev1.TagType_TAG_TYPE_STRING},
+ },
+ },
+ },
+ }
+ schema, err := BuildSchema(stream, nil)
+ if err != nil {
+ t.Fatalf("build schema: %v", err)
+ }
+ return schema
+}
+
+func buildEqualityCriteria(tagName, value string) *modelv1.Criteria {
+ return &modelv1.Criteria{
+ Exp: &modelv1.Criteria_Condition{
+ Condition: &modelv1.Condition{
+ Name: tagName,
+ Op: modelv1.Condition_BINARY_OP_EQ,
+ Value: &modelv1.TagValue{
+ Value: &modelv1.TagValue_Str{
+ Str: &modelv1.Str{Value: value},
+ },
+ },
+ },
+ },
+ }
+}
+
+func buildStringValue(val string) *modelv1.TagValue {
+ return &modelv1.TagValue{
+ Value: &modelv1.TagValue_Str{
+ Str: &modelv1.Str{Value: val},
+ },
+ }
+}
diff --git a/test/cases/measure/data/input/filter_hidden_tag.ql
b/test/cases/measure/data/input/filter_hidden_tag.ql
new file mode 100644
index 00000000..bc532640
--- /dev/null
+++ b/test/cases/measure/data/input/filter_hidden_tag.ql
@@ -0,0 +1,22 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Test: Filter by service_id (indexed hidden tag) but only select id,
entity_id (visible tags)
+SELECT id, entity_id, total::field, value::field FROM MEASURE
service_instance_cpm_minute IN sw_metric
+TIME > '-15m'
+WHERE service_id = 'svc_1'
+
diff --git a/test/cases/measure/data/input/filter_hidden_tag.yaml
b/test/cases/measure/data/input/filter_hidden_tag.yaml
new file mode 100644
index 00000000..46d8fb69
--- /dev/null
+++ b/test/cases/measure/data/input/filter_hidden_tag.yaml
@@ -0,0 +1,35 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Test: Filter by service_id (indexed hidden tag) but only project id,
entity_id (visible tags)
+# This tests the hidden tag stripping in normal measure mode
+name: "service_instance_cpm_minute"
+groups: ["sw_metric"]
+tagProjection:
+ tagFamilies:
+ - name: "default"
+ tags: ["id", "entity_id"]
+fieldProjection:
+ names: ["total", "value"]
+criteria:
+ condition:
+ name: "service_id"
+ op: "BINARY_OP_EQ"
+ value:
+ str:
+ value: "svc_1"
+
diff --git a/test/cases/measure/data/input/index_mode_filter_hidden_tag.ql
b/test/cases/measure/data/input/index_mode_filter_hidden_tag.ql
new file mode 100644
index 00000000..e2dfc569
--- /dev/null
+++ b/test/cases/measure/data/input/index_mode_filter_hidden_tag.ql
@@ -0,0 +1,22 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Test: Filter by layer (indexed hidden tag) but only select id, service_id,
name (visible tags)
+SELECT id, service_id, name FROM MEASURE service_traffic IN index_mode
+TIME > '-15m'
+WHERE layer = 1
+
diff --git a/test/cases/measure/data/input/index_mode_filter_hidden_tag.yaml
b/test/cases/measure/data/input/index_mode_filter_hidden_tag.yaml
new file mode 100644
index 00000000..6f257245
--- /dev/null
+++ b/test/cases/measure/data/input/index_mode_filter_hidden_tag.yaml
@@ -0,0 +1,33 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Test: Filter by layer (indexed hidden tag) but only project id, service_id,
name (visible tags)
+# This tests the hidden tag stripping in index mode measure
+name: "service_traffic"
+groups: [ "index_mode" ]
+tagProjection:
+ tagFamilies:
+ - name: "default"
+ tags: [ "id", "service_id", "name" ]
+criteria:
+ condition:
+ name: "layer"
+ op: "BINARY_OP_EQ"
+ value:
+ int:
+ value: 1
+
diff --git a/test/cases/measure/data/want/filter_hidden_tag.yaml
b/test/cases/measure/data/want/filter_hidden_tag.yaml
new file mode 100644
index 00000000..7aa7c2d8
--- /dev/null
+++ b/test/cases/measure/data/want/filter_hidden_tag.yaml
@@ -0,0 +1,126 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Expected result: service_instance_cpm_minute data with service_id=svc_1
+# Note: service_id should NOT be in result (hidden tag stripped)
+# Results are sorted by sid, then by timestamp (due to DisOrder: true)
+dataPoints:
+# sid: 7941974329620232473
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "11"
+ tagFamilies:
+ - name: default
+ tags:
+ - key: id
+ value:
+ str:
+ value: "10"
+ - key: entity_id
+ value:
+ str:
+ value: entity_3
+# sid: 8355698475044176280 (earlier timestamp)
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "1"
+ tagFamilies:
+ - name: default
+ tags:
+ - key: id
+ value:
+ str:
+ value: "3"
+ - key: entity_id
+ value:
+ str:
+ value: entity_1
+# sid: 8355698475044176280 (later timestamp)
+- fields:
+ - name: total
+ value:
+ int:
+ value: "300"
+ - name: value
+ value:
+ int:
+ value: "6"
+ tagFamilies:
+ - name: default
+ tags:
+ - key: id
+ value:
+ str:
+ value: "3"
+ - key: entity_id
+ value:
+ str:
+ value: entity_1
+# sid: 11136573860836752918 (earlier timestamp)
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "9"
+ tagFamilies:
+ - name: default
+ tags:
+ - key: id
+ value:
+ str:
+ value: "5"
+ - key: entity_id
+ value:
+ str:
+ value: entity_2
+# sid: 11136573860836752918 (later timestamp)
+- fields:
+ - name: total
+ value:
+ int:
+ value: "100"
+ - name: value
+ value:
+ int:
+ value: "3"
+ tagFamilies:
+ - name: default
+ tags:
+ - key: id
+ value:
+ str:
+ value: "5"
+ - key: entity_id
+ value:
+ str:
+ value: entity_2
diff --git a/test/cases/measure/data/want/index_mode_filter_hidden_tag.yaml
b/test/cases/measure/data/want/index_mode_filter_hidden_tag.yaml
new file mode 100644
index 00000000..8fdfcaab
--- /dev/null
+++ b/test/cases/measure/data/want/index_mode_filter_hidden_tag.yaml
@@ -0,0 +1,38 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Expected result: Only service with layer=1 (service_1), layer should NOT be
in result
+dataPoints:
+- sid: "15142466043926325685"
+ tagFamilies:
+ - name: default
+ tags:
+ - key: id
+ value:
+ str:
+ value: "1"
+ - key: service_id
+ value:
+ str:
+ value: service_1
+ - key: name
+ value:
+ str:
+ value: service_name_1
+ timestamp: "2024-11-15T01:02:00Z"
+ version: "1"
+
diff --git a/test/cases/measure/measure.go b/test/cases/measure/measure.go
index 274dd3cb..5e692974 100644
--- a/test/cases/measure/measure.go
+++ b/test/cases/measure/measure.go
@@ -40,6 +40,8 @@ var (
)
var _ = g.DescribeTable("Scanning Measures", verify,
+ g.Entry("filter hidden tag projection", helpers.Args{Input:
"filter_hidden_tag", Duration: 25 * time.Minute, Offset: -20 * time.Minute,
DisOrder: true}),
+ g.Entry("index mode filter hidden tag projection", helpers.Args{Input:
"index_mode_filter_hidden_tag", Duration: 25 * time.Minute, Offset: -20 *
time.Minute}),
g.Entry("all", helpers.Args{Input: "all", Duration: 25 * time.Minute,
Offset: -20 * time.Minute}),
g.Entry("all only fields", helpers.Args{Input: "all_only_fields",
Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("the max limit", helpers.Args{Input: "all_max_limit", Want:
"all", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
diff --git a/test/cases/stream/data/input/filter_hidden_tag.ql
b/test/cases/stream/data/input/filter_hidden_tag.ql
new file mode 100644
index 00000000..960b0ca6
--- /dev/null
+++ b/test/cases/stream/data/input/filter_hidden_tag.ql
@@ -0,0 +1,21 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+SELECT trace_id, span_id FROM STREAM sw IN default
+TIME > '-15m'
+WHERE span_id = '1' AND start_time = 1622933202000000000
+
diff --git a/test/cases/stream/data/input/filter_hidden_tag.yaml
b/test/cases/stream/data/input/filter_hidden_tag.yaml
new file mode 100644
index 00000000..73ded0f3
--- /dev/null
+++ b/test/cases/stream/data/input/filter_hidden_tag.yaml
@@ -0,0 +1,41 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: "sw"
+groups: ["default"]
+projection:
+ tagFamilies:
+ - name: "searchable"
+ tags: ["trace_id", "span_id"]
+criteria:
+ le:
+ op: "LOGICAL_OP_AND"
+ left:
+ condition:
+ name: "span_id"
+ op: "BINARY_OP_EQ"
+ value:
+ str:
+ value: "1"
+ right:
+ condition:
+ name: "start_time"
+ op: "BINARY_OP_EQ"
+ value:
+ int:
+ value: 1622933202000000000
+
diff --git a/test/cases/stream/data/want/filter_hidden_tag.yaml
b/test/cases/stream/data/want/filter_hidden_tag.yaml
new file mode 100644
index 00000000..b1ac936c
--- /dev/null
+++ b/test/cases/stream/data/want/filter_hidden_tag.yaml
@@ -0,0 +1,43 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+elements:
+- elementId: "0978df79cf4ed409"
+ tagFamilies:
+ - name: searchable
+ tags:
+ - key: trace_id
+ value:
+ str:
+ value: "1"
+ - key: span_id
+ value:
+ str:
+ value: "1"
+- elementId: "4dd999e3502d728d"
+ tagFamilies:
+ - name: searchable
+ tags:
+ - key: trace_id
+ value:
+ str:
+ value: "2"
+ - key: span_id
+ value:
+ str:
+ value: "1"
+
diff --git a/test/cases/stream/stream.go b/test/cases/stream/stream.go
index 97ae14a5..c0715df0 100644
--- a/test/cases/stream/stream.go
+++ b/test/cases/stream/stream.go
@@ -67,6 +67,7 @@ var _ = g.DescribeTable("Scanning Streams", func(args
helpers.Args) {
g.Entry("global index", helpers.Args{Input: "global_index", Duration: 1
* time.Hour}),
g.Entry("multi-global index", helpers.Args{Input: "global_indices",
Duration: 1 * time.Hour}),
g.Entry("filter by non-indexed tag", helpers.Args{Input: "filter_tag",
Duration: 1 * time.Hour}),
+ g.Entry("filter hidden tag projection", helpers.Args{Input:
"filter_hidden_tag", Duration: 1 * time.Hour}),
g.Entry("filter by non-indexed tag order by duration desc with limit
3", helpers.Args{Input: "sort_duration_no_index_limit", Duration: 1 *
time.Hour}),
g.Entry("get empty result by non-indexed tag", helpers.Args{Input:
"filter_tag_empty", Duration: 1 * time.Hour, WantEmpty: true}),
g.Entry("get results by no non-index tag", helpers.Args{Input:
"filter_no_indexed", Duration: 1 * time.Hour}),
