This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch tristan/remote-cache-testing in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 18015fafdc56121e6666f5dec0b26d69575434f4 Author: Tristan van Berkom <[email protected]> AuthorDate: Wed Jan 20 22:03:33 2021 +0900 .github: Revived remote cache testing This patch does the following things: * ci.remote-execution-cluster.yml -> ci.remote-execution.yml * .github/workflows/ci.yml: Refactored to add a second test matrix The 'test-services' matrix runs tests which require adjacent services run in parallel with the test defined in ci.docker-compose.yml * .github/compose/ci.buildstream-remote-cache.yml: Added composition file to run the buildstream artifact cache `bst-artifact-server` * .github/compose/ci.buildbarn-remote-cache.yml: Added composition file to run the buildbarn index and storage services. * .github/compose/ci.docker-compose.yml: Add the test declarations to run tests with `--remote-cache` to test against the artifact cache servers. --- .github/compose/buildbarn-config/asset.jsonnet | 32 ++++++++++++ .github/compose/buildbarn-config/storage.jsonnet | 26 ++++++++++ .github/compose/ci.buildbarn-remote-cache.yml | 59 ++++++++++++++++++++++ .github/compose/ci.buildstream-remote-cache.yml | 36 +++++++++++++ .github/compose/ci.docker-compose.yml | 29 +++++++++-- ...ecution-cluster.yml => ci.remote-execution.yml} | 4 +- .github/workflows/ci.yml | 25 ++++++--- 7 files changed, 199 insertions(+), 12 deletions(-) diff --git a/.github/compose/buildbarn-config/asset.jsonnet b/.github/compose/buildbarn-config/asset.jsonnet new file mode 100644 index 0000000..aad06a2 --- /dev/null +++ b/.github/compose/buildbarn-config/asset.jsonnet @@ -0,0 +1,32 @@ +{ + fetcher: { + caching: { + fetcher: { + // We should never be fetching anything which is not already returned by the caching fetcher. + 'error': { + code: 5, + message: "Asset Not Found", + } + } + } + }, + + assetStore: { + circular: { + directory: '/storage', + offsetFileSizeBytes: 1024 * 1024, + offsetCacheSize: 1000, + dataFileSizeBytes: 100 * 1024 * 1024, + dataAllocationChunkSizeBytes: 1048576, + instances: [''], + }, + }, + httpListenAddress: ':1111', + grpcServers: [{ + listenAddresses: [':7981'], + authenticationPolicy: { allow: {} }, + }], + allowUpdatesForInstances: [''], + maximumMessageSizeBytes: 16 * 1024 * 1024, +} + diff --git a/.github/compose/buildbarn-config/storage.jsonnet b/.github/compose/buildbarn-config/storage.jsonnet new file mode 100644 index 0000000..11fbdbe --- /dev/null +++ b/.github/compose/buildbarn-config/storage.jsonnet @@ -0,0 +1,26 @@ +{ + blobstore: { + contentAddressableStorage: { + circular: { + directory: '/cas', + offsetFileSizeBytes: 16 * 1024 * 1024, + offsetCacheSize: 10000, + dataFileSizeBytes: 10 * 1024 * 1024 * 1024, + dataAllocationChunkSizeBytes: 16 * 1024 * 1024, + }, + }, + actionCache: { + 'error': { + code: 12, # UNIMPLEMENTED + message: "AC requests are not supported for this endpoint.", + } + }, + }, + httpListenAddress: ':6981', + grpcServers: [{ + listenAddresses: [':7982'], + authenticationPolicy: { allow: {} }, + }], + allowAcUpdatesForInstanceNamePrefixes: [''], + maximumMessageSizeBytes: 16 * 1024 * 1024, +} diff --git a/.github/compose/ci.buildbarn-remote-cache.yml b/.github/compose/ci.buildbarn-remote-cache.yml new file mode 100644 index 0000000..a7a8892 --- /dev/null +++ b/.github/compose/ci.buildbarn-remote-cache.yml @@ -0,0 +1,59 @@ +## +# Buildbarn Compose manifest for BuildStream. +# +# Spins-up a unnamed and unauthenticated cache server: +# - STORAGE at http://localhost:7982 +# - INDEX at: http://localhost:7981 +# +# BuildStream configuration snippet: +# +# artifacts: +# - url: https://localhost:7981 +# type: index +# push: true +# - url: https://localhost:7982 +# type: storage +# push: true +# +# Basic usage: +# - docker-compose -f ci.buildbarn-remote-cache.yml up +# - docker-compose -f ci.buildbarn-remote-cache.yml down + +version: '3.4' + +services: + bb-asset: + image: buildbarn/bb-remote-asset:20200903T103837Z-90136c4 + command: /config/asset.jsonnet + restart: unless-stopped + expose: + - "7981" + ports: + - "7981:7981" + volumes: + - type: volume + source: assets + target: /storage + - type: bind + source: ./buildbarn-config/ + target: /config + + bb-storage: + image: buildbarn/bb-storage:20200816T115912Z-d83e1f0 + command: /config/storage.jsonnet + restart: unless-stopped + expose: + - "7982" + ports: + - "7982:7982" + volumes: + - type: volume + source: cas + target: /cas + - type: bind + source: ./buildbarn-config/ + target: /config + +volumes: + assets: + cas: diff --git a/.github/compose/ci.buildstream-remote-cache.yml b/.github/compose/ci.buildstream-remote-cache.yml new file mode 100644 index 0000000..9f853fa --- /dev/null +++ b/.github/compose/ci.buildstream-remote-cache.yml @@ -0,0 +1,36 @@ +## +# A Builstream Artifact Server +# +# Spins-up a unnamed and unauthenticated grid: +# - artifacts server at: http://localhost:50052 +# +# BuildStream configuration snippet: +# +# artifacts: +# url: http://localhost:50052 +# push: true +# +# Basic usage: +# - docker-compose -f ci.buildstream-remote-cache.yml up +# - docker-compose -f ci.buildstream-remote-cache.yml down +# +version: "3.2" + +services: + controller: + image: buildstream/buildstream:dev + command: ["bst-artifact-server","--port", + "50052", + "--enable-push", + "/artifacts" + ] + ports: + - 50052:50052 + networks: + - grid +networks: + grid: + driver: bridge + +volumes: + cache: diff --git a/.github/compose/ci.docker-compose.yml b/.github/compose/ci.docker-compose.yml index 88ab894..614a79c 100644 --- a/.github/compose/ci.docker-compose.yml +++ b/.github/compose/ci.docker-compose.yml @@ -67,12 +67,33 @@ services: REMOTE_EXECUTION_SERVICE: http://localhost:50051 SOURCE_CACHE_SERVICE: http://localhost:50052 - # The ci.remote-execution-cluster.yml takes care of bringing up - # a remote execution cluster, exposing the ports 50052 and 50051 - # on the localhost network. + # We need to use host networking mode in order to be able to + # properly resolve services exposed by adjacent containers. + # + network_mode: host + + buildstream-remote-cache: + <<: *tests-template + command: tox -vvvvv -- --color=yes --remote-cache + environment: + TOXENV: ${CI_TOXENV_MAIN} + ARTIFACT_CACHE_SERVICE: http://localhost:50052 + + # We need to use host networking mode in order to be able to + # properly resolve services exposed by adjacent containers. # + network_mode: host + + buildbarn-remote-cache: + <<: *tests-template + command: tox -vvvvv -- --color=yes --remote-cache + environment: + TOXENV: ${CI_TOXENV_MAIN} + ARTIFACT_INDEX_SERVICE: http://localhost:7981 + ARTIFACT_STORAGE_SERVICE: http://localhost:7982 + # We need to use host networking mode in order to be able to - # properly resolve these services. + # properly resolve services exposed by adjacent containers. # network_mode: host diff --git a/.github/compose/ci.remote-execution-cluster.yml b/.github/compose/ci.remote-execution.yml similarity index 94% rename from .github/compose/ci.remote-execution-cluster.yml rename to .github/compose/ci.remote-execution.yml index 75d0ec7..a663362 100644 --- a/.github/compose/ci.remote-execution-cluster.yml +++ b/.github/compose/ci.remote-execution.yml @@ -19,8 +19,8 @@ # url: http://localhost:50051 # # Basic usage: -# - docker-compose -f buildgrid-compose.yml up -# - docker-compose -f buildgrid-compose.yml down +# - docker-compose -f ci.remote-execution.yml up +# - docker-compose -f ci.remote-execution.yml down # version: "3.2" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 292e497..5dc2bba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,8 +76,21 @@ jobs: run \ ${{ matrix.test-name }} - remote-execution: + # Matrix of tests which run against remote services which we bring up adjacently + service-tests: runs-on: ubuntu-20.04 + + strategy: + matrix: + + # The names here map to valid services defined in "../compose/ci.docker-compose.yml", + # and they also map to corresponding filenames of services which are expected + # to be run in the background + test-name: + - remote-execution + - buildstream-remote-cache + - buildbarn-remote-cache + steps: - name: Check out repository uses: actions/checkout@v2 @@ -92,26 +105,26 @@ jobs: run: | docker-compose \ --env-file ${GITHUB_WORKSPACE}/.github/common.env \ - --file ${GITHUB_WORKSPACE}/.github/compose/ci.remote-execution-cluster.yml \ + --file ${GITHUB_WORKSPACE}/.github/compose/ci.${{ matrix.test-name }}.yml \ up --detach --renew-anon-volumes --remove-orphans - name: Run the remote execution tests run: | docker-compose \ --env-file ${GITHUB_WORKSPACE}/.github/common.env \ - --file ${GITHUB_WORKSPACE}/.github/compose/ci.docker-compose.yml run remote-execution + --file ${GITHUB_WORKSPACE}/.github/compose/ci.docker-compose.yml run ${{ matrix.test-name }} - name: Bring down the RE cluster run: | docker-compose \ --env-file ${GITHUB_WORKSPACE}/.github/common.env \ - --file ${GITHUB_WORKSPACE}/.github/compose/ci.remote-execution-cluster.yml stop + --file ${GITHUB_WORKSPACE}/.github/compose/ci.${{ matrix.test-name }}.yml stop docker-compose \ --env-file ${GITHUB_WORKSPACE}/.github/common.env \ - --file ${GITHUB_WORKSPACE}/.github/compose/ci.remote-execution-cluster.yml logs + --file ${GITHUB_WORKSPACE}/.github/compose/ci.${{ matrix.test-name }}.yml logs docker-compose \ --env-file ${GITHUB_WORKSPACE}/.github/common.env \ - --file ${GITHUB_WORKSPACE}/.github/compose/ci.remote-execution-cluster.yml down + --file ${GITHUB_WORKSPACE}/.github/compose/ci.${{ matrix.test-name }}.yml down docs: runs-on: ubuntu-20.04
