This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
The following commit(s) were added to refs/heads/main by this push:
new 3f07c21 Improve workflows and building tools (#208)
3f07c21 is described below
commit 3f07c212797aab7e1eee107c76b7777fb2de4b92
Author: Gao Hongtao <[email protected]>
AuthorDate: Thu Nov 10 09:36:27 2022 +0800
Improve workflows and building tools (#208)
* Update workflows
* Rearrange steps of workflows
* Introduce building tools cache to throttle up the workflow
* Introduce dependency review
* Add SEGMENT_DOWNLOAD_TIMEOUT_MINS
Signed-off-by: Gao Hongtao <[email protected]>
---
.github/workflows/{go.yml => ci.yml} | 103 +++++++++++++++++++--------
.github/workflows/e2e.storage.yml | 32 +++++++--
.github/workflows/load.yml | 21 +++---
.github/workflows/publish-docker.yml | 30 ++++++--
Makefile | 2 +
api/Makefile | 15 ++--
banyand/Makefile | 1 +
banyand/tsdb/shard_test.go | 2 +
bydbctl/Makefile | 1 +
scripts/build/base.mk | 2 -
scripts/build/generate_go.mk | 2 +-
scripts/build/ginkgo.mk | 2 +-
scripts/build/license.mk | 2 +-
scripts/build/lint-bin.mk | 2 +-
scripts/build/lint.mk | 2 +-
scripts/build/{generate_go.mk => version.mk} | 23 +++---
scripts/ci/check/Makefile | 1 +
ui/Makefile | 4 ++
18 files changed, 176 insertions(+), 71 deletions(-)
diff --git a/.github/workflows/go.yml b/.github/workflows/ci.yml
similarity index 66%
rename from .github/workflows/go.yml
rename to .github/workflows/ci.yml
index 153411e..1b75422 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/ci.yml
@@ -22,23 +22,18 @@ on:
branches:
- main
+env:
+ SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
jobs:
check:
name: Check
runs-on: ubuntu-20.04
steps:
- - uses: actions/setup-node@v3
- with:
- node-version: 16.15
- - name: Install Go
- uses: actions/setup-go@v2
- with:
- go-version: 1.19
- name: Check out code into the Go module directory
- uses: actions/checkout@v2
- - name: Check License Header
- run: make license-check
- - uses: actions/cache@v3
+ uses: actions/checkout@v3
+ - name: Cache Go Modules
+ uses: actions/cache@v3
id: cache-go
with:
path: |
@@ -47,6 +42,25 @@ jobs:
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
+ - name: Cache Tools
+ uses: actions/cache@v3
+ id: cache-tool
+ with:
+ path: bin
+ key: ${{ runner.os }}-check-tool-${{ hashFiles('**version.mk') }}
+ restore-keys: |
+ ${{ runner.os }}-check-tool-
+ - uses: actions/setup-node@v3
+ with:
+ node-version: 16.15
+ cache: 'npm'
+ cache-dependency-path: ui/package-lock.json
+ - name: Install Go
+ uses: actions/setup-go@v3
+ with:
+ go-version-file: 'go.mod'
+ - name: Check License Header
+ run: make license-check
- name: Update dependencies
if: steps.cache-go.outputs.cache-hit != 'true'
run: GOPROXY=https://proxy.golang.org go mod download
@@ -62,16 +76,10 @@ jobs:
name: Build
runs-on: ubuntu-20.04
steps:
- - uses: actions/setup-node@v3
- with:
- node-version: 16.15
- - name: Install Go
- uses: actions/setup-go@v2
- with:
- go-version: 1.19
- name: Check out code into the Go module directory
- uses: actions/checkout@v2
- - uses: actions/cache@v3
+ uses: actions/checkout@v3
+ - name: Cache Go Modules
+ uses: actions/cache@v3
id: cache-go
with:
path: |
@@ -80,6 +88,23 @@ jobs:
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
+ - name: Cache Tools
+ uses: actions/cache@v3
+ id: cache-tool
+ with:
+ path: bin
+ key: ${{ runner.os }}-build-tool-${{ hashFiles('**version.mk') }}
+ restore-keys: |
+ ${{ runner.os }}-build-tool-
+ - uses: actions/setup-node@v3
+ with:
+ node-version: 16.15
+ cache: 'npm'
+ cache-dependency-path: ui/package-lock.json
+ - name: Install Go
+ uses: actions/setup-go@v3
+ with:
+ go-version-file: 'go.mod'
- name: Update dependencies
if: steps.cache-go.outputs.cache-hit != 'true'
run: GOPROXY=https://proxy.golang.org go mod download
@@ -98,16 +123,10 @@ jobs:
steps:
- name: Set timezone
run: sudo timedatectl set-timezone ${{ matrix.tz }}
- - uses: actions/setup-node@v3
- with:
- node-version: 16.15
- - name: Install Go
- uses: actions/setup-go@v2
- with:
- go-version: 1.19
- name: Check out code into the Go module directory
- uses: actions/checkout@v2
- - uses: actions/cache@v3
+ uses: actions/checkout@v3
+ - name: Cache Go Modules
+ uses: actions/cache@v3
id: cache-go
with:
path: |
@@ -116,6 +135,23 @@ jobs:
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
+ - name: Cache tools
+ uses: actions/cache@v3
+ id: cache-tool
+ with:
+ path: bin
+ key: ${{ runner.os }}-test-tool-${{ hashFiles('**version.mk') }}
+ restore-keys: |
+ ${{ runner.os }}-test-tool-
+ - uses: actions/setup-node@v3
+ with:
+ node-version: 16.15
+ cache: 'npm'
+ cache-dependency-path: ui/package-lock.json
+ - name: Install Go
+ uses: actions/setup-go@v3
+ with:
+ go-version-file: 'go.mod'
- name: Update dependencies
if: steps.cache-go.outputs.cache-hit != 'true'
run: GOPROXY=https://proxy.golang.org go mod download
@@ -125,10 +161,17 @@ jobs:
run: make test-ci
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
+ dependency-review:
+ runs-on: ubuntu-latest
+ steps:
+ - name: 'Checkout Repository'
+ uses: actions/checkout@v3
+ - name: 'Dependency Review'
+ uses: actions/dependency-review-action@v2
result:
name: Continuous Integration
runs-on: ubuntu-20.04
- needs: [check, build, test]
+ needs: [check, build, test, dependency-review]
steps:
- run: echo 'success'
diff --git a/.github/workflows/e2e.storage.yml
b/.github/workflows/e2e.storage.yml
index 08302bf..9604d0a 100644
--- a/.github/workflows/e2e.storage.yml
+++ b/.github/workflows/e2e.storage.yml
@@ -24,6 +24,7 @@ on:
env:
SW_AGENT_JDK_VERSION: 8
+ SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
concurrency:
group: e2e-storages-${{ github.event.pull_request.number || github.ref }}
@@ -50,18 +51,37 @@ jobs:
env:
TAG: ${{ github.sha }}
steps:
+ - name: Check out code into the Go module directory
+ uses: actions/checkout@v3
+ - name: Cache Go Modules
+ uses: actions/cache@v3
+ id: cache-go
+ with:
+ path: |
+ ~/.cache/go-build
+ ~/go/pkg/mod
+ key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+ restore-keys: |
+ ${{ runner.os }}-go-
+ - name: Cache Tools
+ uses: actions/cache@v3
+ id: cache-tool
+ with:
+ path: bin
+ key: ${{ runner.os }}-build-tool-${{ hashFiles('**version.mk') }}
+ restore-keys: |
+ ${{ runner.os }}-build-tool-
- uses: actions/setup-node@v3
with:
node-version: 16.15
+ cache: 'npm'
+ cache-dependency-path: ui/package-lock.json
- name: Install Go
- uses: actions/setup-go@v2
- with:
- go-version: 1.19
- - name: Check out code into the Go module directory
- uses: actions/checkout@v2
+ uses: actions/setup-go@v3
with:
- submodules: true
+ go-version-file: 'go.mod'
- name: Update dependencies
+ if: steps.cache-go.outputs.cache-hit != 'true'
run: GOPROXY=https://proxy.golang.org go mod download
- name: Generate codes
run: make generate
diff --git a/.github/workflows/load.yml b/.github/workflows/load.yml
index d3682d4..694af4d 100644
--- a/.github/workflows/load.yml
+++ b/.github/workflows/load.yml
@@ -20,6 +20,9 @@ on:
schedule:
- cron: '0 20 * * *'
+env:
+ SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
jobs:
test:
name: Load several days data
@@ -30,15 +33,8 @@ jobs:
steps:
- name: Set timezone
run: sudo timedatectl set-timezone ${{ matrix.tz }}
- - uses: actions/setup-node@v3
- with:
- node-version: 16.15
- - name: Install Go
- uses: actions/setup-go@v2
- with:
- go-version: 1.19
- name: Check out code into the Go module directory
- uses: actions/checkout@v2
+ uses: actions/checkout@v3
- uses: actions/cache@v3
id: cache-go
with:
@@ -48,6 +44,15 @@ jobs:
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
+ - uses: actions/setup-node@v3
+ with:
+ node-version: 16.15
+ cache: 'npm'
+ cache-dependency-path: ui/package-lock.json
+ - name: Install Go
+ uses: actions/setup-go@v3
+ with:
+ go-version-file: 'go.mod'
- name: Update dependencies
if: steps.cache-go.outputs.cache-hit != 'true'
run: GOPROXY=https://proxy.golang.org go mod download
diff --git a/.github/workflows/publish-docker.yml
b/.github/workflows/publish-docker.yml
index 1e6e78a..91bdeb4 100644
--- a/.github/workflows/publish-docker.yml
+++ b/.github/workflows/publish-docker.yml
@@ -24,6 +24,7 @@ on:
env:
HUB: ghcr.io/apache
+ SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
jobs:
push-banyandb-image:
@@ -38,16 +39,37 @@ jobs:
env:
TAG: ${{ github.sha }}
steps:
+ - name: Check out code into the Go module directory
+ uses: actions/checkout@v3
+ - name: Cache Go Modules
+ uses: actions/cache@v3
+ id: cache-go
+ with:
+ path: |
+ ~/.cache/go-build
+ ~/go/pkg/mod
+ key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+ restore-keys: |
+ ${{ runner.os }}-go-
+ - name: Cache Tools
+ uses: actions/cache@v3
+ id: cache-tool
+ with:
+ path: bin
+ key: ${{ runner.os }}-build-tool-${{ hashFiles('**version.mk') }}
+ restore-keys: |
+ ${{ runner.os }}-build-tool-
- uses: actions/setup-node@v3
with:
node-version: 16.15
+ cache: 'npm'
+ cache-dependency-path: ui/package-lock.json
- name: Install Go
- uses: actions/setup-go@v2
+ uses: actions/setup-go@v3
with:
- go-version: 1.19
- - name: Check out code into the Go module directory
- uses: actions/checkout@v2
+ go-version-file: 'go.mod'
- name: Update dependencies
+ if: steps.cache-go.outputs.cache-hit != 'true'
run: GOPROXY=https://proxy.golang.org go mod download
- name: Generate codes
run: make generate
diff --git a/Makefile b/Makefile
index 099a0b8..259e1da 100644
--- a/Makefile
+++ b/Makefile
@@ -25,6 +25,8 @@ include $(mk_dir).env
export
endif
+include scripts/build/version.mk
+
PROJECTS := ui banyand bydbctl
##@ Build targets
diff --git a/api/Makefile b/api/Makefile
index 876905f..bd8747e 100644
--- a/api/Makefile
+++ b/api/Makefile
@@ -18,6 +18,7 @@
NAME := api
+include ../scripts/build/version.mk
include ../scripts/build/base.mk
include ../scripts/build/lint-api.mk
include ../scripts/build/help.mk
@@ -35,13 +36,13 @@ $(BUF):
@rm -f $(tool_bin)/protoc-gen-openapiv2
@rm -f $(tool_bin)/protoc-gen-validate
@rm -f $(tool_bin)/buf
- @GOBIN=$(tool_bin) go install
google.golang.org/protobuf/cmd/[email protected]
- @GOBIN=$(tool_bin) go install
google.golang.org/grpc/cmd/[email protected]
- @GOBIN=$(tool_bin) go install
github.com/pseudomuto/protoc-gen-doc/cmd/[email protected]
- @GOBIN=$(tool_bin) go install
github.com/grpc-ecosystem/grpc-gateway/v2/[email protected]
- @GOBIN=$(tool_bin) go install
github.com/grpc-ecosystem/grpc-gateway/v2/[email protected]
- @GOBIN=$(tool_bin) go install
github.com/bufbuild/buf/cmd/buf@$(buf_version)
- @GOBIN=$(tool_bin) go install
github.com/envoyproxy/[email protected]
+ @GOBIN=$(tool_bin) go install
google.golang.org/protobuf/cmd/protoc-gen-go@$(PROTOC_GEN_GO_VERSION)
+ @GOBIN=$(tool_bin) go install
google.golang.org/grpc/cmd/protoc-gen-go-grpc@$(PROTOC_GEN_GO_GRPC_VERSION)
+ @GOBIN=$(tool_bin) go install
github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc@$(PROTOC_GEN_DOC_VERSION)
+ @GOBIN=$(tool_bin) go install
github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway@$(GRPC_GATEWAY_VERSION)
+ @GOBIN=$(tool_bin) go install
github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2@$(GRPC_GATEWAY_VERSION)
+ @GOBIN=$(tool_bin) go install
github.com/bufbuild/buf/cmd/buf@$(BUF_VERSION)
+ @GOBIN=$(tool_bin) go install
github.com/envoyproxy/protoc-gen-validate@$(PROTOC_GEN_VALIDATE_VERSION)
generate: $(BUF)
@PATH=$(tool_bin):$(proto_dir) $(BUF) generate && \
diff --git a/banyand/Makefile b/banyand/Makefile
index e14e560..6822eb5 100644
--- a/banyand/Makefile
+++ b/banyand/Makefile
@@ -23,6 +23,7 @@ DEBUG_BINARIES := $(SERVER)-debug
IMG_NAME := skywalking-banyandb
+include ../scripts/build/version.mk
include ../scripts/build/base.mk
include ../scripts/build/generate_go.mk
include ../scripts/build/build.mk
diff --git a/banyand/tsdb/shard_test.go b/banyand/tsdb/shard_test.go
index 6b66ea4..d723223 100644
--- a/banyand/tsdb/shard_test.go
+++ b/banyand/tsdb/shard_test.go
@@ -285,6 +285,8 @@ var _ = Describe("Shard", func() {
if clock.TriggerTimer() {
GinkgoWriter.Println("01/03 01:00 has
been triggered")
}
+ // Trigger block queue cleanup job to remove
evitable blocks
+ clock.Add(1 * time.Minute)
return shard.State().OpenBlocks
}, flags.EventuallyTimeout).Should(Equal([]tsdb.BlockID{
{
diff --git a/bydbctl/Makefile b/bydbctl/Makefile
index dbfeff1..093f658 100644
--- a/bydbctl/Makefile
+++ b/bydbctl/Makefile
@@ -21,6 +21,7 @@ IMG_NAME := skywalking-bydbctl
BUILD_DIR ?= build/bin
+include ../scripts/build/version.mk
include ../scripts/build/base.mk
include ../scripts/build/generate_go.mk
diff --git a/scripts/build/base.mk b/scripts/build/base.mk
index a0d3b2b..58bd16f 100644
--- a/scripts/build/base.mk
+++ b/scripts/build/base.mk
@@ -22,8 +22,6 @@ root_dir := $(mk_dir)../..
tool_bin := $(root_dir)/bin
tool_include := "$(root_dir)/include"
-buf_version ?= v1.5.0
-
# Retrieve git versioning details so we can add to our binary assets
VERSION_PATH := github.com/apache/skywalking-banyandb/pkg/version
ifdef RELEASE_VERSION
diff --git a/scripts/build/generate_go.mk b/scripts/build/generate_go.mk
index 4869540..786323f 100644
--- a/scripts/build/generate_go.mk
+++ b/scripts/build/generate_go.mk
@@ -20,7 +20,7 @@ MOCKGEN := $(tool_bin)/mockgen
$(MOCKGEN):
@echo "Install mock generate tool..."
@mkdir -p $(tool_bin)
- @GOBIN=$(tool_bin) go install github.com/golang/mock/[email protected]
+ @GOBIN=$(tool_bin) go install
github.com/golang/mock/mockgen@$(MOCKGEN_VERSION)
.PHONY: generate
generate: $(MOCKGEN)
diff --git a/scripts/build/ginkgo.mk b/scripts/build/ginkgo.mk
index b3152b6..f21ba2b 100644
--- a/scripts/build/ginkgo.mk
+++ b/scripts/build/ginkgo.mk
@@ -2,4 +2,4 @@ GINKGO := $(tool_bin)/ginkgo
$(GINKGO):
@echo "Install ginkgo..."
@mkdir -p $(tool_bin)
- @GOBIN=$(tool_bin) go install github.com/onsi/ginkgo/v2/[email protected]
+ @GOBIN=$(tool_bin) go install
github.com/onsi/ginkgo/v2/ginkgo@$(GINKGO_VERSION)
diff --git a/scripts/build/license.mk b/scripts/build/license.mk
index 19e978c..367d7ef 100644
--- a/scripts/build/license.mk
+++ b/scripts/build/license.mk
@@ -20,4 +20,4 @@ LICENSE_EYE := $(tool_bin)/license-eye
$(LICENSE_EYE):
@echo "Install license-eye..."
@mkdir -p $(tool_bin)
- @GOBIN=$(tool_bin) go install
github.com/apache/skywalking-eyes/cmd/license-eye@f461a46e74e5fa22e9f9599a355ab4f0ac265469
+ @GOBIN=$(tool_bin) go install
github.com/apache/skywalking-eyes/cmd/license-eye@$(LICENSE_EYE_VERSION)
diff --git a/scripts/build/lint-bin.mk b/scripts/build/lint-bin.mk
index 0668b74..427ffce 100644
--- a/scripts/build/lint-bin.mk
+++ b/scripts/build/lint-bin.mk
@@ -1,4 +1,4 @@
LINTER := $(tool_bin)/golangci-lint
$(LINTER):
- @GOBIN=$(tool_bin) go install
github.com/golangci/golangci-lint/cmd/[email protected]
+ @GOBIN=$(tool_bin) go install
github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
diff --git a/scripts/build/lint.mk b/scripts/build/lint.mk
index ccf33fd..f7c81b8 100644
--- a/scripts/build/lint.mk
+++ b/scripts/build/lint.mk
@@ -22,7 +22,7 @@ include $(mk_dir)lint-bin.mk
REVIVE := $(tool_bin)/revive
$(REVIVE):
- @GOBIN=$(tool_bin) go install github.com/mgechev/revive@latest
+ @GOBIN=$(tool_bin) go install
github.com/mgechev/revive@$(REVIVE_VERSION)
.PHONY: lint
lint: $(LINTER) $(REVIVE) ## Run all linters
diff --git a/scripts/build/generate_go.mk b/scripts/build/version.mk
similarity index 64%
copy from scripts/build/generate_go.mk
copy to scripts/build/version.mk
index 4869540..e61d6c0 100644
--- a/scripts/build/generate_go.mk
+++ b/scripts/build/version.mk
@@ -6,7 +6,7 @@
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE_2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
@@ -16,12 +16,17 @@
# under the License.
#
-MOCKGEN := $(tool_bin)/mockgen
-$(MOCKGEN):
- @echo "Install mock generate tool..."
- @mkdir -p $(tool_bin)
- @GOBIN=$(tool_bin) go install github.com/golang/mock/[email protected]
+BUF_VERSION := v1.5.0
+PROTOC_GEN_GO_VERSION := v1.28.0
+PROTOC_GEN_GO_GRPC_VERSION := v1.2.0
+PROTOC_GEN_DOC_VERSION := v1.5.1
+GRPC_GATEWAY_VERSION := v2.10.3
+PROTOC_GEN_VALIDATE_VERSION := v0.6.7
-.PHONY: generate
-generate: $(MOCKGEN)
- @PATH=$(tool_bin):$$PATH; go generate ./...
+GOLANGCI_LINT_VERSION := v1.50.0
+REVIVE_VERSION := v1.2.4
+LICENSE_EYE_VERSION := f461a46e74e5fa22e9f9599a355ab4f0ac265469
+
+MOCKGEN_VERSION := v1.6.0
+
+GINKGO_VERSION := v2.1.4
\ No newline at end of file
diff --git a/scripts/ci/check/Makefile b/scripts/ci/check/Makefile
index 58fbb6a..9d3d7bb 100644
--- a/scripts/ci/check/Makefile
+++ b/scripts/ci/check/Makefile
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+include ../../build/version.mk
include ../../build/base.mk
include ../../build/lint.mk
include ../../build/generate_go.mk
diff --git a/ui/Makefile b/ui/Makefile
index 298af5a..3bac6cd 100644
--- a/ui/Makefile
+++ b/ui/Makefile
@@ -19,6 +19,7 @@
NAME := ui
DIST_INDEX := dist/index.html
+include ../scripts/build/version.mk
include ../scripts/build/base.mk
include ../scripts/build/lint.mk
include ../scripts/build/help.mk
@@ -36,6 +37,9 @@ all: build
.PHONY: generate
generate: install
+generate: $(DIST_INDEX)
+
+$(DIST_INDEX):
@echo "Building $(NAME)"
npm run build
@echo "Done building $(NAME)"
