This is an automated email from the ASF dual-hosted git repository. hanahmily pushed a commit to branch open-block in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 11621554730ac4b880585e733df18ea03f697ae3 Author: Gao Hongtao <[email protected]> AuthorDate: Thu Nov 10 16:00:26 2022 +0000 Sync the block opening Signed-off-by: Gao Hongtao <[email protected]> --- .github/workflows/ci.yml | 3 +- .github/workflows/flaky-test.yml | 71 +++++++++++++++++++++++++ Makefile | 5 +- banyand/tsdb/block.go | 15 ++++-- banyand/tsdb/segment.go | 8 ++- banyand/tsdb/tsdb_suite_test.go | 2 +- test/integration/cold_query/query_suite_test.go | 2 +- test/integration/other/other_suite_test.go | 2 +- test/integration/query/query_suite_test.go | 2 +- 9 files changed, 95 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70d0a8a..35da044 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -158,7 +158,8 @@ jobs: - name: Generate mocks run: make generate - name: Test - run: make test-ci + run: | + TEST_CI_OPTS="--cover --covermode atomic --coverprofile=coverage.out --label-filter !slow" make test-ci - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 result: diff --git a/.github/workflows/flaky-test.yml b/.github/workflows/flaky-test.yml new file mode 100644 index 0000000..7939039 --- /dev/null +++ b/.github/workflows/flaky-test.yml @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Flaky Test + +on: + schedule: + - cron: '50 * * * *' + +env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 + +jobs: + test: + name: Test + runs-on: ubuntu-20.04 + strategy: + matrix: + tz: ["UTC", "Asia/Shanghai", "America/Los_Angeles"] + steps: + - name: Set timezone + run: sudo timedatectl set-timezone ${{ matrix.tz }} + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + - name: Cache Go Modules + uses: actions/cache@v3 + id: cache-go + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Cache tools + uses: actions/cache@v3 + id: cache-tool + with: + path: bin + key: ${{ runner.os }}-test-tool-${{ hashFiles('**version.mk') }} + restore-keys: | + ${{ runner.os }}-test-tool- + - uses: actions/setup-node@v3 + with: + node-version: 16.15 + cache: 'npm' + cache-dependency-path: ui/package-lock.json + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version-file: 'go.mod' + - name: Update dependencies + if: steps.cache-go.outputs.cache-hit != 'true' + run: GOPROXY=https://proxy.golang.org go mod download + - name: Generate mocks + run: make generate + - name: Test integration and banyand + run: TEST_CI_OPTS="--repeat 4 --label-filter (integration&&!slow)||banyand" make test-ci # run tests 4+1 rounds diff --git a/Makefile b/Makefile index 259e1da..bc46155 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,8 @@ include scripts/build/version.mk PROJECTS := ui banyand bydbctl +TEST_CI_OPTS ?= + ##@ Build targets clean: TARGET=clean @@ -70,8 +72,7 @@ test-ci: $(GINKGO) ## Run the unit tests in CI $(GINKGO) --race \ -ldflags \ "-X github.com/apache/skywalking-banyandb/pkg/test/flags.eventuallyTimeout=30s -X github.com/apache/skywalking-banyandb/pkg/test/flags.LogLevel=warn" \ - --cover --covermode atomic --coverprofile=coverage.out \ - --label-filter !slow \ + $(TEST_CI_OPTS) \ ./... ##@ Code quality targets diff --git a/banyand/tsdb/block.go b/banyand/tsdb/block.go index 9a813c0..b7f3158 100644 --- a/banyand/tsdb/block.go +++ b/banyand/tsdb/block.go @@ -143,10 +143,19 @@ func (b *block) options(ctx context.Context) { } } -func (b *block) open() (err error) { - if b.deleted.Load() { +func (b *block) openSafely() (err error) { + if b.deleted.Load() || !b.Closed() { return nil } + b.lock.Lock() + defer b.lock.Unlock() + if !b.Closed() { + return + } + return b.open() +} + +func (b *block) open() (err error) { if b.store, err = kv.OpenTimeSeriesStore( 0, path.Join(b.path, componentMain), @@ -194,7 +203,7 @@ func (b *block) delegate(ctx context.Context) (BlockDelegate, error) { b.lock.Lock() defer b.lock.Unlock() if err := b.queue.Push(ctx, blockID, func() error { - if !b.Closed() { + if b.deleted.Load() || !b.Closed() { return nil } return b.open() diff --git a/banyand/tsdb/segment.go b/banyand/tsdb/segment.go index abfcd28..fbc6fa8 100644 --- a/banyand/tsdb/segment.go +++ b/banyand/tsdb/segment.go @@ -196,10 +196,8 @@ func (bc *blockController) Current() (bucket.Reporter, error) { } return nil }(); b != nil { - if b.Closed() { - if err := b.open(); err != nil { - return nil, err - } + if err := b.openSafely(); err != nil { + return nil, err } return b, nil } @@ -390,7 +388,7 @@ func (bc *blockController) create(startTime time.Time) (*block, error) { if err != nil { return nil, err } - err = b.open() + err = b.openSafely() if err != nil { return nil, err } diff --git a/banyand/tsdb/tsdb_suite_test.go b/banyand/tsdb/tsdb_suite_test.go index 52e1157..5456e59 100644 --- a/banyand/tsdb/tsdb_suite_test.go +++ b/banyand/tsdb/tsdb_suite_test.go @@ -28,7 +28,7 @@ import ( func TestTsdb(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Tsdb Suite") + RunSpecs(t, "Tsdb Suite", Label("banyand")) } var _ = BeforeSuite(func() { diff --git a/test/integration/cold_query/query_suite_test.go b/test/integration/cold_query/query_suite_test.go index 849a059..2d9285b 100644 --- a/test/integration/cold_query/query_suite_test.go +++ b/test/integration/cold_query/query_suite_test.go @@ -40,7 +40,7 @@ import ( func TestIntegrationColdQuery(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Integration Query Cold Data Suite") + RunSpecs(t, "Integration Query Cold Data Suite", Label("integration")) } var ( diff --git a/test/integration/other/other_suite_test.go b/test/integration/other/other_suite_test.go index eaf2474..b073fc8 100644 --- a/test/integration/other/other_suite_test.go +++ b/test/integration/other/other_suite_test.go @@ -29,7 +29,7 @@ import ( func TestIntegrationOther(t *testing.T) { gm.RegisterFailHandler(g.Fail) - g.RunSpecs(t, "Integration Other Suite") + g.RunSpecs(t, "Integration Other Suite", g.Label("integration")) } var _ = g.BeforeSuite(func() { diff --git a/test/integration/query/query_suite_test.go b/test/integration/query/query_suite_test.go index 69be763..5379dbf 100644 --- a/test/integration/query/query_suite_test.go +++ b/test/integration/query/query_suite_test.go @@ -40,7 +40,7 @@ import ( func TestIntegrationQuery(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Integration Query Suite") + RunSpecs(t, "Integration Query Suite", Label("integration")) } var (
