This is an automated email from the ASF dual-hosted git repository. hanahmily pushed a commit to branch ci in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 0f2f3f3b2ec465987c08b7f1eaab9342905c456f Author: Gao Hongtao <[email protected]> AuthorDate: Thu Oct 23 09:37:21 2025 +0800 Refactor CI Workflows to Use Centralized Preparation Step * Consolidate preparation steps across multiple CI workflows by utilizing a shared `prepare.yml` file. * Update jobs in `ci.yml`, `e2e.yml`, `flaky-test.yml`, `publish-docker.yml`, `slow-test.yml`, and `test.yml` to streamline setup processes and improve maintainability. * Remove redundant steps such as code checkout and tool installations, enhancing workflow efficiency. --- .github/actions/build-docker-image/action.yml | 61 +++++++++++++++++++++++ .github/actions/setup-build-env/action.yml | 59 +++++++++++++++++++++++ .github/workflows/ci.yml | 69 +++------------------------ .github/workflows/e2e.yml | 56 +++++++++++++--------- .github/workflows/flaky-test.yml | 3 ++ .github/workflows/prepare.yml | 67 ++++++++++++++++++++++++++ .github/workflows/publish-docker.yml | 19 ++------ .github/workflows/slow-test.yml | 9 +++- .github/workflows/test.yml | 21 ++------ 9 files changed, 244 insertions(+), 120 deletions(-) diff --git a/.github/actions/build-docker-image/action.yml b/.github/actions/build-docker-image/action.yml new file mode 100644 index 00000000..372a191f --- /dev/null +++ b/.github/actions/build-docker-image/action.yml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: 'Build Docker Image' +description: 'Build, package, and upload BanyanDB Docker image' + +inputs: + tag: + description: 'Docker image tag' + required: true + hub: + description: 'Docker hub namespace' + required: false + default: 'apache' + artifact-name: + description: 'Name for the uploaded docker image artifact' + required: false + default: 'banyandb-docker-image' + +runs: + using: composite + steps: + - name: Build binary + shell: bash + run: make release + + - name: Build docker image + shell: bash + run: | + make -C test/docker build + docker image ls + env: + TAG: ${{ inputs.tag }} + HUB: ${{ inputs.hub }} + + - name: Package docker image + shell: bash + run: | + IMAGE_NAME="${{ inputs.hub }}/skywalking-banyandb:${{ inputs.tag }}-testing" + docker save "$IMAGE_NAME" | gzip > banyandb-testing-image.tar.gz + ls -lh banyandb-testing-image.tar.gz + + - name: Upload docker image artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact-name }} + path: banyandb-testing-image.tar.gz + retention-days: 1 diff --git a/.github/actions/setup-build-env/action.yml b/.github/actions/setup-build-env/action.yml new file mode 100644 index 00000000..d54b4ec3 --- /dev/null +++ b/.github/actions/setup-build-env/action.yml @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: 'Setup Build Environment' +description: 'Common setup steps for BanyanDB builds' + +inputs: + download-artifacts: + description: 'Whether to download build artifacts' + required: false + default: 'true' + setup-docker: + description: 'Whether to setup Docker Buildx' + required: false + default: 'false' + +runs: + using: composite + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + with: + fetch-tags: true + + - name: Download build artifacts + if: inputs.download-artifacts == 'true' + uses: actions/download-artifact@v4 + with: + name: build-artifacts + + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: 20.12 + cache: 'npm' + cache-dependency-path: ui/package-lock.json + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + cache: true + + - name: Set up Docker Buildx + if: inputs.setup-docker == 'true' + uses: docker/setup-buildx-action@v1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index feb5a18a..4bcbc69c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,62 +28,17 @@ env: jobs: prepare: name: Prepare (Generate) - runs-on: ubuntu-latest - steps: - - name: Check out code into the Go module directory - uses: actions/checkout@v4 - with: - fetch-tags: true - - name: Cache Tools - uses: actions/cache@v4 - id: cache-tool - with: - path: bin - key: ${{ runner.os }}-tool-${{ hashFiles('**/version.mk') }} - restore-keys: | - ${{ runner.os }}-tool- - - uses: actions/setup-node@v4 - with: - node-version: 20.12 - cache: 'npm' - cache-dependency-path: ui/package-lock.json - - name: Install Go - uses: actions/setup-go@v5 - with: - go-version-file: 'go.mod' - cache: true - - name: Generate codes - run: make generate - - name: Build UI - run: | - cd ui - npm ci - npm run build - cd .. - - name: Upload generated files - uses: actions/upload-artifact@v4 - with: - name: build-artifacts - path: | - **/*.pb.go - **/*.pb.gw.go - **/*.pb.validate.go - **/*_mock.go - ui/dist/** - retention-days: 1 + uses: ./.github/workflows/prepare.yml check: name: Check runs-on: ubuntu-latest needs: [prepare] steps: - - name: Check out code into the Go module directory - uses: actions/checkout@v4 + - name: Setup build environment + uses: ./.github/actions/setup-build-env with: - fetch-tags: true - - name: Download build artifacts - uses: actions/download-artifact@v4 - with: - name: build-artifacts + download-artifacts: 'true' + setup-docker: 'false' - name: Cache Tools uses: actions/cache@v4 id: cache-tool @@ -92,11 +47,6 @@ jobs: key: ${{ runner.os }}-check-tool-${{ hashFiles('**/version.mk') }} restore-keys: | ${{ runner.os }}-check-tool- - - uses: actions/setup-node@v4 - with: - node-version: 20.12 - cache: 'npm' - cache-dependency-path: ui/package-lock.json - name: npm ci and format run: | cd ui @@ -110,11 +60,6 @@ jobs: git status -s; \ exit 1; \ fi - - name: Install Go - uses: actions/setup-go@v5 - with: - go-version-file: 'go.mod' - cache: true - name: Check License Header run: make license-check - name: Check requirements @@ -127,8 +72,6 @@ jobs: run: make license-dep - name: Check run: make check - - name: Build Release - run: make release test-banyand: name: Test Banyand needs: [check] @@ -153,7 +96,7 @@ jobs: name: E2E Tests needs: [check] uses: ./.github/workflows/e2e.yml - + result: name: Continuous Integration runs-on: ubuntu-24.04 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 39026c64..590aba68 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -30,11 +30,33 @@ concurrency: cancel-in-progress: true jobs: + prepare: + if: github.event_name == 'schedule' && github.repository == 'apache/skywalking-banyandb' + uses: ./.github/workflows/prepare.yml + + DockerImage: + if: github.event_name == 'schedule' && github.repository == 'apache/skywalking-banyandb' + name: Build Docker image + runs-on: ubuntu-latest + timeout-minutes: 45 + needs: [prepare] + steps: + - name: Setup build environment + uses: ./.github/actions/setup-build-env + with: + download-artifacts: 'true' + setup-docker: 'true' + - name: Build and upload docker image + uses: ./.github/actions/build-docker-image + with: + tag: ${{ github.sha }} + StoragePlugins: if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking-banyandb') || (github.event_name != 'schedule') name: Storage E2E runs-on: ubuntu-latest timeout-minutes: 90 + needs: [prepare, DockerImage] strategy: fail-fast: false matrix: @@ -53,36 +75,24 @@ jobs: env: TAG: ${{ github.sha }} steps: - - name: Check out code into the Go module directory - uses: actions/checkout@v4 + - name: Setup build environment + uses: ./.github/actions/setup-build-env with: - fetch-tags: true - - name: Download build artifacts + download-artifacts: 'true' + setup-docker: 'false' + - name: Download docker image artifact uses: actions/download-artifact@v4 with: - name: build-artifacts - - uses: actions/setup-node@v3 - with: - node-version: 20.12 - cache: 'npm' - cache-dependency-path: ui/package-lock.json - - name: Install Go - uses: actions/setup-go@v5 - with: - go-version-file: 'go.mod' - cache: true - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Build binary - run: make release - - name: Build docker image + name: banyandb-docker-image + path: /tmp + - name: Load docker image run: | - make -C test/docker build | make -C test/docker build + docker load --input /tmp/banyandb-testing-image.tar.gz docker image ls - name: Generate data for lifecycle if: ${{ matrix.test.name == 'Lifecycle' }} run: | - bash test/e2e-v2/cases/lifecycle/data-generator/gen.sh + bash test/e2e-v2/cases/lifecycle/data-generator/gen.sh - name: ${{ matrix.test.name }} uses: apache/skywalking-infra-e2e@7e4b5b68716fdb7b79b21fa8908f9db497e1b115 with: @@ -112,7 +122,7 @@ jobs: if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking-banyandb') || (github.event_name != 'schedule') runs-on: ubuntu-latest timeout-minutes: 90 - needs: [StoragePlugins] + needs: [prepare, StoragePlugins] steps: - name: To pass or not pass run: echo "Just to make the GitHub merge button green" diff --git a/.github/workflows/flaky-test.yml b/.github/workflows/flaky-test.yml index 3dab9173..f5e0b57c 100644 --- a/.github/workflows/flaky-test.yml +++ b/.github/workflows/flaky-test.yml @@ -24,7 +24,10 @@ env: SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 jobs: + prepare: + uses: ./.github/workflows/prepare.yml test: + needs: [prepare] if: github.repository == 'apache/skywalking-banyandb' uses: ./.github/workflows/test.yml with: diff --git a/.github/workflows/prepare.yml b/.github/workflows/prepare.yml new file mode 100644 index 00000000..7302d9b0 --- /dev/null +++ b/.github/workflows/prepare.yml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Reusable Prepare + +on: + workflow_call: + +jobs: + prepare: + name: Prepare (Generate) + runs-on: ubuntu-latest + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + with: + fetch-tags: true + - name: Cache Tools + uses: actions/cache@v4 + id: cache-tool + with: + path: bin + key: ${{ runner.os }}-tool-${{ hashFiles('**/version.mk') }} + restore-keys: | + ${{ runner.os }}-tool- + - uses: actions/setup-node@v4 + with: + node-version: 20.12 + cache: 'npm' + cache-dependency-path: ui/package-lock.json + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + cache: true + - name: Generate codes + run: make generate + - name: Build UI + run: | + cd ui + npm ci + npm run build + cd .. + - name: Upload generated files + uses: actions/upload-artifact@v4 + with: + name: build-artifacts + path: | + **/*.pb.go + **/*.pb.gw.go + **/*.pb.validate.go + **/*_mock.go + ui/dist/** + retention-days: 1 diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 8f1a76ae..acbc49ca 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -39,10 +39,11 @@ jobs: env: TAG: ${{ github.sha }} steps: - - name: Check out code into the Go module directory - uses: actions/checkout@v4 + - name: Setup build environment + uses: ./.github/actions/setup-build-env with: - fetch-tags: true + download-artifacts: 'false' + setup-docker: 'true' - name: Cache Tools uses: actions/cache@v4 id: cache-tool @@ -51,20 +52,8 @@ jobs: key: ${{ runner.os }}-build-tool-${{ hashFiles('**version.mk') }} restore-keys: | ${{ runner.os }}-build-tool- - - uses: actions/setup-node@v4 - with: - node-version: 20.12 - cache: 'npm' - cache-dependency-path: ui/package-lock.json - - name: Install Go - uses: actions/setup-go@v5 - with: - go-version-file: 'go.mod' - cache: true - name: Generate codes run: make generate - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - name: Build Linux binaries run: | TARGET_OS=linux PLATFORMS=linux/amd64,linux/arm64 make release diff --git a/.github/workflows/slow-test.yml b/.github/workflows/slow-test.yml index 498df535..8b0a2aab 100644 --- a/.github/workflows/slow-test.yml +++ b/.github/workflows/slow-test.yml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Slow Test +name: Slow Test on: schedule: @@ -24,9 +24,13 @@ env: SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 jobs: - + prepare: + if: github.repository == 'apache/skywalking-banyandb' + uses: ./.github/workflows/prepare.yml + test: if: github.repository == 'apache/skywalking-banyandb' + needs: [prepare] uses: ./.github/workflows/test.yml with: test-name: Slow Tests @@ -35,6 +39,7 @@ jobs: property-repair: if: github.repository == 'apache/skywalking-banyandb' + needs: [prepare] uses: ./.github/workflows/test.yml with: test-name: Property Repair Tests diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8c6c0124..169814ca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,14 +51,11 @@ jobs: steps: - name: Set timezone run: sudo timedatectl set-timezone ${{ inputs.timezone || 'UTC' }} - - name: Check out code into the Go module directory - uses: actions/checkout@v4 + - name: Setup build environment + uses: ./.github/actions/setup-build-env with: - fetch-tags: true - - name: Download build artifacts - uses: actions/download-artifact@v4 - with: - name: build-artifacts + download-artifacts: 'true' + setup-docker: 'false' - name: Cache tools uses: actions/cache@v4 id: cache-tool @@ -67,16 +64,6 @@ jobs: key: ${{ runner.os }}-test-tool-${{ hashFiles('**version.mk') }} restore-keys: | ${{ runner.os }}-test-tool- - - uses: actions/setup-node@v4 - with: - node-version: 20.12 - cache: 'npm' - cache-dependency-path: ui/package-lock.json - - name: Install Go - uses: actions/setup-go@v5 - with: - go-version-file: 'go.mod' - cache: true - name: Test integration and banyand run: TEST_CI_OPTS="--cover --covermode atomic --coverprofile=coverage.out ${{ inputs.options }}" make test-ci PKG=${{ inputs.pkg }} - if: ${{ failure() }}
