This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
The following commit(s) were added to refs/heads/main by this push:
new fb543e79 Fix CI tasks (#391)
fb543e79 is described below
commit fb543e79a0160f65326fc6c4a9a2305b84a1d26d
Author: Gao Hongtao <hanahm...@gmail.com>
AuthorDate: Wed Feb 28 09:01:36 2024 +0800
Fix CI tasks (#391)
---
.github/workflows/flaky-test.yml | 2 +-
banyand/measure/merger_policy.go | 2 +-
bydbctl/internal/cmd/root.go | 3 +
test/docker/base-compose.yml | 2 +-
test/integration/load/load_suite_test.go | 141 ++++++++++++++++++-------------
test/stress/Makefile | 16 ++--
test/stress/docker-compose.yaml | 16 +++-
test/stress/env | 8 +-
test/stress/env.dev | 8 +-
9 files changed, 122 insertions(+), 76 deletions(-)
diff --git a/.github/workflows/flaky-test.yml b/.github/workflows/flaky-test.yml
index 0ca35b3a..3420da1d 100644
--- a/.github/workflows/flaky-test.yml
+++ b/.github/workflows/flaky-test.yml
@@ -28,5 +28,5 @@ jobs:
if: github.repository == 'apache/skywalking-banyandb'
uses: ./.github/workflows/test.yml
with:
- options: --repeat 49 --label-filter
\\(integration\\&\\&\\!slow\\)\\|\\|banyand
+ options: --repeat 49 --label-filter '(integration&&!slow)||banyand'
timeout-minutes: 40
diff --git a/banyand/measure/merger_policy.go b/banyand/measure/merger_policy.go
index cdb7e9ad..b04e26cc 100644
--- a/banyand/measure/merger_policy.go
+++ b/banyand/measure/merger_policy.go
@@ -32,7 +32,7 @@ type mergePolicy struct {
// NewDefaultMergePolicy create a MergePolicy with default parameters.
func newDefaultMergePolicy() *mergePolicy {
- return newMergePolicy(15, 1.7, math.MaxUint64)
+ return newMergePolicy(8, 1.7, math.MaxUint64)
}
func newDefaultMergePolicyForTesting() *mergePolicy {
diff --git a/bydbctl/internal/cmd/root.go b/bydbctl/internal/cmd/root.go
index 9aff040d..f9645fca 100644
--- a/bydbctl/internal/cmd/root.go
+++ b/bydbctl/internal/cmd/root.go
@@ -79,6 +79,9 @@ func init() {
func initConfig() {
if cfgFile != "" {
+ if cfgFile == "-" {
+ return
+ }
// Use config file from the flag.
viper.SetConfigFile(cfgFile)
} else {
diff --git a/test/docker/base-compose.yml b/test/docker/base-compose.yml
index 49b4e3d5..7379151c 100644
--- a/test/docker/base-compose.yml
+++ b/test/docker/base-compose.yml
@@ -21,7 +21,7 @@ services:
- 6060
command: standalone
healthcheck:
- test: ["CMD", "./bydbctl", "health", "--addr=http://banyandb:17913";]
+ test: ["CMD", "./bydbctl", "health", "--config=-",
"--addr=http://banyandb:17913";]
interval: 5s
timeout: 10s
retries: 120
diff --git a/test/integration/load/load_suite_test.go
b/test/integration/load/load_suite_test.go
index 7071bc5e..65011cd4 100644
--- a/test/integration/load/load_suite_test.go
+++ b/test/integration/load/load_suite_test.go
@@ -18,6 +18,7 @@
package integration_load_test
import (
+ "context"
"testing"
"time"
@@ -27,12 +28,14 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
+ commonv1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ modelv1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
+ streamv1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
"github.com/apache/skywalking-banyandb/pkg/grpchelper"
"github.com/apache/skywalking-banyandb/pkg/logger"
"github.com/apache/skywalking-banyandb/pkg/test/flags"
"github.com/apache/skywalking-banyandb/pkg/test/helpers"
"github.com/apache/skywalking-banyandb/pkg/test/setup"
- cases_stream "github.com/apache/skywalking-banyandb/test/cases/stream"
cases_stream_data
"github.com/apache/skywalking-banyandb/test/cases/stream/data"
)
@@ -41,64 +44,88 @@ func TestIntegrationLoad(t *testing.T) {
RunSpecs(t, "Integration Load Suite", Label("integration", "slow"))
}
-var (
- connection *grpc.ClientConn
- now time.Time
- deferFunc func()
- goods []gleak.Goroutine
-)
+var _ = Describe("Test Suite", func() {
+ var (
+ connection *grpc.ClientConn
+ now time.Time
+ deferFunc func()
+ goods []gleak.Goroutine
+ addr string
+ )
-var _ = SynchronizedBeforeSuite(func() []byte {
- goods = gleak.Goroutines()
- Expect(logger.Init(logger.Logging{
- Env: "dev",
- Level: flags.LogLevel,
- })).To(Succeed())
- var addr string
- addr, _, deferFunc = setup.Standalone()
- Eventually(
- helpers.HealthCheck(addr, 10*time.Second, 10*time.Second,
grpc.WithTransportCredentials(insecure.NewCredentials())),
- flags.EventuallyTimeout).Should(Succeed())
- conn, err := grpchelper.Conn(addr, 10*time.Second,
grpc.WithTransportCredentials(insecure.NewCredentials()))
- Expect(err).NotTo(HaveOccurred())
- days := 7
- hours := 24
- minutes := 60
- interval := 10 * time.Second
- c := time.Now()
- for i := 0; i < days; i++ {
- date := c.Add(-time.Hour * time.Duration((days-i)*24))
- for h := 0; h < hours; h++ {
- hour := date.Add(time.Hour * time.Duration(h))
- start := time.Now()
- for j := 0; j < minutes; j++ {
- n := hour.Add(time.Minute * time.Duration(j))
- ns := n.UnixNano()
- now = time.Unix(0, ns-ns%int64(time.Minute))
- // stream
- cases_stream_data.Write(conn, "data.json", now,
interval)
+ BeforeEach(func() {
+ Expect(logger.Init(logger.Logging{
+ Env: "dev",
+ Level: flags.LogLevel,
+ })).Should(Succeed())
+ goods = gleak.Goroutines()
+ addr, _, deferFunc = setup.Standalone()
+ Eventually(
+ helpers.HealthCheck(addr, 10*time.Second,
10*time.Second, grpc.WithTransportCredentials(insecure.NewCredentials())),
+ flags.EventuallyTimeout).Should(Succeed())
+ var err error
+ connection, err = grpchelper.Conn(addr, 10*time.Second,
grpc.WithTransportCredentials(insecure.NewCredentials()))
+ Expect(err).NotTo(HaveOccurred())
+ days := 7
+ hours := 24
+ minutes := 60
+ interval := 10 * time.Second
+ c := time.Now()
+ for i := 0; i < days; i++ {
+ date := c.Add(-time.Hour * time.Duration((days-i)*24))
+ for h := 0; h < hours; h++ {
+ hour := date.Add(time.Hour * time.Duration(h))
+ start := time.Now()
+ for j := 0; j < minutes; j++ {
+ n := hour.Add(time.Minute *
time.Duration(j))
+ ns := n.UnixNano()
+ now = time.Unix(0,
ns-ns%int64(time.Minute))
+ // stream
+ cases_stream_data.Write(connection,
"data.json", now, interval)
+ }
+ logger.Infof("written stream in %s took %s \n",
hour, time.Since(start))
}
- GinkgoWriter.Printf("written stream in %s took %s \n",
hour, time.Since(start))
}
- }
- Expect(conn.Close()).To(Succeed())
- return []byte(addr)
-}, func(address []byte) {
- var err error
- connection, err = grpchelper.Conn(string(address), 10*time.Second,
- grpc.WithTransportCredentials(insecure.NewCredentials()))
- cases_stream.SharedContext = helpers.SharedContext{
- Connection: connection,
- BaseTime: now,
- }
- Expect(err).NotTo(HaveOccurred())
-})
-
-var _ = SynchronizedAfterSuite(func() {
- if connection != nil {
Expect(connection.Close()).To(Succeed())
- }
-}, func() {
- deferFunc()
- Eventually(gleak.Goroutines,
flags.EventuallyTimeout).ShouldNot(gleak.HaveLeaked(goods))
+ })
+
+ It("should read data", func() {
+ var err error
+ connection, err = grpchelper.Conn(addr, 10*time.Second,
+
grpc.WithTransportCredentials(insecure.NewCredentials()))
+ Expect(err).NotTo(HaveOccurred())
+ sharedContext := helpers.SharedContext{
+ Connection: connection,
+ BaseTime: now,
+ }
+ query := &streamv1.QueryRequest{
+ Metadata: &commonv1.Metadata{
+ Name: "sw",
+ Group: "default",
+ },
+ Projection: &modelv1.TagProjection{
+ TagFamilies: []*modelv1.TagProjection_TagFamily{
+ {
+ Name: "searchable",
+ Tags: []string{"trace_id"},
+ },
+ },
+ },
+ }
+ query.TimeRange = helpers.TimeRange(helpers.Args{Input: "all",
Duration: 1 * time.Hour}, sharedContext)
+ c := streamv1.NewStreamServiceClient(sharedContext.Connection)
+ ctx := context.Background()
+ resp, err := c.Query(ctx, query)
+ Expect(err).NotTo(HaveOccurred())
+ GinkgoWriter.Printf("query result: %s elements\n",
resp.GetElements())
+ Expect(len(resp.GetElements())).To(BeNumerically(">", 0))
+ })
+
+ AfterEach(func() {
+ if connection != nil {
+ Expect(connection.Close()).To(Succeed())
+ }
+ deferFunc()
+ Eventually(gleak.Goroutines,
flags.EventuallyTimeout).ShouldNot(gleak.HaveLeaked(goods))
+ })
})
diff --git a/test/stress/Makefile b/test/stress/Makefile
index e0200152..789c4934 100644
--- a/test/stress/Makefile
+++ b/test/stress/Makefile
@@ -24,14 +24,20 @@ ifdef PROFILE
CLI_ARGS := $(CLI_ARGS) --profile $(PROFILE)
endif
+cli_env := DOCKER_BUILDKIT=1 USER_ID=$(shell id -u) GROUP_ID=$(shell id -g)
+
+.PHONY: clean
+clean:
+ rm -rf /tmp/banyandb-stress-agent
+
.PHONY: dev-up
-dev-up:
- DOCKER_BUILDKIT=1 docker compose $(CLI_ARGS) --env-file ./env.dev up
--build
+dev-up: clean
+ $(cli_env) docker compose $(CLI_ARGS) --env-file ./env.dev up --build
.PHONY: up
-up:
- DOCKER_BUILDKIT=1 docker compose $(CLI_ARGS) --env-file ./env up --build
+up: clean
+ $(cli_env) docker compose $(CLI_ARGS) --env-file ./env up --build
.PHONY: down
down:
- DOCKER_BUILDKIT=1 docker compose down
+ docker compose down
diff --git a/test/stress/docker-compose.yaml b/test/stress/docker-compose.yaml
index 1278a19e..9d28ff79 100644
--- a/test/stress/docker-compose.yaml
+++ b/test/stress/docker-compose.yaml
@@ -14,7 +14,16 @@
# limitations under the License.
services:
+ change-vol-ownership:
+ image: ubuntu
+ user: "root"
+ group_add:
+ - '${GROUP_ID}'
+ volumes:
+ - /tmp/banyandb-stress-agent:/tmp/change-ownership
+ command: chown -R ${USER_ID}:${GROUP_ID} /tmp/change-ownership
banyandb:
+ user: "${USER_ID}:${GROUP_ID}"
extends:
file: ../docker/base-compose.yml
service: banyandb
@@ -22,9 +31,7 @@ services:
dockerfile: ./docker/Dockerfile
context: ../..
volumes:
- - ../..:/app:rw,delegated
- # Uncomment below line to mount an external volume
- # - /tmp:/tmp:rw,delgated
+ - /tmp/banyandb-stress-agent:/tmp:rw,delgated
ports:
- 17913:17913
- 6060:6060
@@ -32,6 +39,9 @@ services:
networks:
- test
- monitoring
+ depends_on:
+ change-vol-ownership:
+ condition: service_completed_successfully
oap:
extends:
diff --git a/test/stress/env b/test/stress/env
index 1560b70d..25350fb1 100644
--- a/test/stress/env
+++ b/test/stress/env
@@ -22,11 +22,11 @@ SW_AGENT_GO_COMMIT=4af380c2db6243106b0fc650b6003ce3b3eb82a0
SW_AGENT_PYTHON_COMMIT=c76a6ec51a478ac91abb20ec8f22a99b8d4d6a58
SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449
SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016
-SW_KUBERNETES_COMMIT_SHA=b670c41d94a82ddefcf466d54bab5c492d88d772
-SW_ROVER_COMMIT=d956eaede57b62108b78bca48045bd09ba88e653
-SW_CTL_COMMIT=e684fae0107045fc23799146d62f04cb68bd5a3b
+SW_KUBERNETES_COMMIT_SHA=e2c61c6774cf377b23516fca6f8a1e119d3191c5
+SW_ROVER_COMMIT=fc8d074c6d34ecfee585a7097cbd5aef1ca680a5
+SW_CTL_COMMIT=6b2eb0011e38b630db6af7203db215806bd141ed
SW_OAP_COMMIT=e5c308d4358d5b02a658b8ad7f153aec99b8e63a
-SW_AGENT_E2E_SERVICE_PROVIDER_COMMIT=828e6e2f2b57a0f06bb0d507e3296d2377943d9a
+SW_AGENT_E2E_SERVICE_PROVIDER_COMMIT=cc7a2c9e97fd2c421adbe3e9c471688459a446d9
VUS=10
diff --git a/test/stress/env.dev b/test/stress/env.dev
index 69aa1ce1..6e0d3224 100644
--- a/test/stress/env.dev
+++ b/test/stress/env.dev
@@ -21,11 +21,11 @@ SW_AGENT_GO_COMMIT=4af380c2db6243106b0fc650b6003ce3b3eb82a0
SW_AGENT_PYTHON_COMMIT=c76a6ec51a478ac91abb20ec8f22a99b8d4d6a58
SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449
SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016
-SW_KUBERNETES_COMMIT_SHA=b670c41d94a82ddefcf466d54bab5c492d88d772
-SW_ROVER_COMMIT=d956eaede57b62108b78bca48045bd09ba88e653
-SW_CTL_COMMIT=e684fae0107045fc23799146d62f04cb68bd5a3b
+SW_KUBERNETES_COMMIT_SHA=e2c61c6774cf377b23516fca6f8a1e119d3191c5
+SW_ROVER_COMMIT=fc8d074c6d34ecfee585a7097cbd5aef1ca680a5
+SW_CTL_COMMIT=6b2eb0011e38b630db6af7203db215806bd141ed
SW_OAP_COMMIT=e5c308d4358d5b02a658b8ad7f153aec99b8e63a
-SW_AGENT_E2E_SERVICE_PROVIDER_COMMIT=828e6e2f2b57a0f06bb0d507e3296d2377943d9a
+SW_AGENT_E2E_SERVICE_PROVIDER_COMMIT=cc7a2c9e97fd2c421adbe3e9c471688459a446d9
VUS=1
