This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch qinusty/add-bb-remote-exec in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 459f0a559d6eb62dd2b02f6646bc81b4ff517cac Author: Josh Smith <[email protected]> AuthorDate: Wed Sep 2 17:51:23 2020 +0100 Add buildbarn remote-execution test case This introduces remote execution with bb-remote-asset as the storage index service. Ports have been standardised across the bb-remote-cache test also. --- .gitlab-ci.yml | 16 +++++- .gitlab-ci/buildbarn-remote-cache.yml | 10 ++-- .gitlab-ci/buildbarn-remote-execution.yml | 83 +++++++++++++++++++++++++++++++ .gitlab-ci/config/asset.jsonnet | 2 +- .gitlab-ci/config/common.libsonnet | 12 +++++ .gitlab-ci/config/frontend.jsonnet | 35 +++++++++++++ .gitlab-ci/config/runner.jsonnet | 9 ++++ .gitlab-ci/config/scheduler.jsonnet | 15 ++++++ .gitlab-ci/config/storage.jsonnet | 2 +- .gitlab-ci/config/worker.jsonnet | 37 ++++++++++++++ 10 files changed, 210 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5f8949d..f6c1217 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -205,10 +205,22 @@ tests-bb-remote-cache: variables: <<: *docker-variables COMPOSE_MANIFEST: .gitlab-ci/buildbarn-remote-cache.yml # < *remote-test - ARTIFACT_INDEX_SERVICE: http://docker:7981 - ARTIFACT_STORAGE_SERVICE: http://docker:7982 + ARTIFACT_INDEX_SERVICE: http://docker:8979 + ARTIFACT_STORAGE_SERVICE: http://docker:8980 PYTEST_ARGS: "--color=yes --remote-cache" +tests-bb-remote-execution: + <<: *tests + <<: *remote-test # Spin up server stack + variables: + <<: *docker-variables + COMPOSE_MANIFEST: .gitlab-ci/buildbarn-remote-execution.yml # < *remote-test + ARTIFACT_INDEX_SERVICE: http://docker:8979 + ARTIFACT_STORAGE_SERVICE: http://docker:8980 + REMOTE_EXECUTION_SERVICE: http://docker:8980 + SOURCE_CACHE_SERVICE: http://docker:8980 + PYTEST_ARGS: "--color=yes --remote-execution" + tests-no-usedevelop: # Ensure that tests also pass without `--develop` flag. image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION} diff --git a/.gitlab-ci/buildbarn-remote-cache.yml b/.gitlab-ci/buildbarn-remote-cache.yml index 0122fa8..4d22244 100644 --- a/.gitlab-ci/buildbarn-remote-cache.yml +++ b/.gitlab-ci/buildbarn-remote-cache.yml @@ -23,13 +23,11 @@ version: '3.4' services: bb-asset: - image: qinusty/bb-asset-hub:20200814T141139Z-5e72dd1 + image: buildbarn/bb-remote-asset:20200828T155422Z-9226c9e command: /config/asset.jsonnet restart: unless-stopped - expose: - - "7981" ports: - - "7981:7981" + - "8979:8979" volumes: - type: volume source: assets @@ -42,10 +40,8 @@ services: image: buildbarn/bb-storage:20200810T194216Z-94a85b4 command: /config/storage.jsonnet restart: unless-stopped - expose: - - "7982" ports: - - "7982:7982" + - 8980:8980 volumes: - type: volume source: cas diff --git a/.gitlab-ci/buildbarn-remote-execution.yml b/.gitlab-ci/buildbarn-remote-execution.yml new file mode 100644 index 0000000..75d0d36 --- /dev/null +++ b/.gitlab-ci/buildbarn-remote-execution.yml @@ -0,0 +1,83 @@ +version: '3' +services: + frontend: + image: buildbarn/bb-storage:20200810T194216Z-94a85b4 + command: + - /config/frontend.jsonnet + ports: + - 7980:7980 + - 8980:8980 + volumes: + - ./config:/config + - ac:/ac + - cas:/cas + + scheduler: + image: buildbarn/bb-scheduler:20200816T115954Z-9fae5ec + command: + - /config/scheduler.jsonnet + expose: + - 8982 + - 8983 + ports: + - 7982:7982 + volumes: + - ./config:/config + + worker-ubuntu16-04: + image: buildbarn/bb-worker:20200816T115954Z-9fae5ec + command: + - /config/worker.jsonnet + ports: + - 7986:7986 + volumes: + - ./config:/config + - ./worker:/worker + depends_on: + - worker-setup + + runner-ubuntu16-04: + image: l.gcr.io/google/rbe-ubuntu16-04@sha256:b516a2d69537cb40a7c6a7d92d0008abb29fba8725243772bdaf2c83f1be2272 + command: + - sh + - -c + - while ! test -f /bb/installed; do sleep 1; done; exec /bb/tini -v -g -- /bb/bb_runner /config/runner.jsonnet + network_mode: none + volumes: + - ./config:/config + - ./bb:/bb + - ./worker:/worker + depends_on: + - runner-installer + - worker-setup + - worker-ubuntu16-04 + + runner-installer: + image: buildbarn/bb-runner-installer:20200801T202247Z-4ea244f + volumes: + - ./bb:/bb + + worker-setup: + image: busybox + volumes: + - ./worker:/worker + command: + - sh + - -c + - mkdir -m 0777 /worker/build && mkdir -m 0700 /worker/cache + + bb-asset: + image: buildbarn/bb-remote-asset:20200828T155422Z-9226c9e + command: /config/asset.jsonnet + restart: unless-stopped + ports: + - 8979:8979 + - 7981:7981 + volumes: + - assets:/storage + - ./config/:/config + +volumes: + assets: + ac: + cas: diff --git a/.gitlab-ci/config/asset.jsonnet b/.gitlab-ci/config/asset.jsonnet index aad06a2..3bd5ab8 100644 --- a/.gitlab-ci/config/asset.jsonnet +++ b/.gitlab-ci/config/asset.jsonnet @@ -23,7 +23,7 @@ }, httpListenAddress: ':1111', grpcServers: [{ - listenAddresses: [':7981'], + listenAddresses: [':8979'], authenticationPolicy: { allow: {} }, }], allowUpdatesForInstances: [''], diff --git a/.gitlab-ci/config/common.libsonnet b/.gitlab-ci/config/common.libsonnet new file mode 100644 index 0000000..75857fe --- /dev/null +++ b/.gitlab-ci/config/common.libsonnet @@ -0,0 +1,12 @@ +{ + blobstore: { + contentAddressableStorage: { + grpc: { address: 'frontend:8980' }, + }, + actionCache: { + grpc: { address: 'frontend:8980' }, + }, + }, + httpListenAddress: ':80', + maximumMessageSizeBytes: 16 * 1024 * 1024, +} \ No newline at end of file diff --git a/.gitlab-ci/config/frontend.jsonnet b/.gitlab-ci/config/frontend.jsonnet new file mode 100644 index 0000000..486ca6d --- /dev/null +++ b/.gitlab-ci/config/frontend.jsonnet @@ -0,0 +1,35 @@ +local common = import 'common.libsonnet'; + +{ + blobstore: { + contentAddressableStorage: { + circular: { + directory: '/cas', + offsetFileSizeBytes: 16 * 1024 * 1024, + offsetCacheSize: 10000, + dataFileSizeBytes: 10 * 1024 * 1024 * 1024, + dataAllocationChunkSizeBytes: 16 * 1024 * 1024, + }, + }, + actionCache: { + circular: { + directory: '/ac', + offsetFileSizeBytes: 1024 * 1024, + offsetCacheSize: 1000, + dataFileSizeBytes: 100 * 1024 * 1024, + dataAllocationChunkSizeBytes: 1048576, + instances: [''], + }, + }, + }, + httpListenAddress: ':7980', + grpcServers: [{ + listenAddresses: [':8980'], + authenticationPolicy: { allow: {} }, + }], + schedulers: { + '': { endpoint: { address: 'scheduler:8981'} }, + }, + allowAcUpdatesForInstanceNamePrefixes: [''], + maximumMessageSizeBytes: common.maximumMessageSizeBytes, +} diff --git a/.gitlab-ci/config/runner.jsonnet b/.gitlab-ci/config/runner.jsonnet new file mode 100644 index 0000000..bb09d45 --- /dev/null +++ b/.gitlab-ci/config/runner.jsonnet @@ -0,0 +1,9 @@ + +{ + buildDirectoryPath: '/worker/build', + grpcServers: [{ + listenPaths: ['/worker/runner'], + authenticationPolicy: { allow: {} }, + }], + chrootIntoInputRoot: true, +} \ No newline at end of file diff --git a/.gitlab-ci/config/scheduler.jsonnet b/.gitlab-ci/config/scheduler.jsonnet new file mode 100644 index 0000000..f3a558c --- /dev/null +++ b/.gitlab-ci/config/scheduler.jsonnet @@ -0,0 +1,15 @@ +local common = import 'common.libsonnet'; + +{ + httpListenAddress: ':7982', + clientGrpcServers: [{ + listenAddresses: [':8981'], + authenticationPolicy: { allow: {} }, + }], + workerGrpcServers: [{ + listenAddresses: [':8982'], + authenticationPolicy: { allow: {} }, + }], + contentAddressableStorage: common.blobstore.contentAddressableStorage, + maximumMessageSizeBytes: common.maximumMessageSizeBytes, +} \ No newline at end of file diff --git a/.gitlab-ci/config/storage.jsonnet b/.gitlab-ci/config/storage.jsonnet index 11fbdbe..40a1056 100644 --- a/.gitlab-ci/config/storage.jsonnet +++ b/.gitlab-ci/config/storage.jsonnet @@ -18,7 +18,7 @@ }, httpListenAddress: ':6981', grpcServers: [{ - listenAddresses: [':7982'], + listenAddresses: [':8980'], authenticationPolicy: { allow: {} }, }], allowAcUpdatesForInstanceNamePrefixes: [''], diff --git a/.gitlab-ci/config/worker.jsonnet b/.gitlab-ci/config/worker.jsonnet new file mode 100644 index 0000000..75ee4e3 --- /dev/null +++ b/.gitlab-ci/config/worker.jsonnet @@ -0,0 +1,37 @@ +local common = import 'common.libsonnet'; + +{ + blobstore: { + contentAddressableStorage: { + grpc: { address: 'frontend:8980' }, + }, + actionCache: { + grpc: { address: 'frontend:8980' }, + } + }, + maximumMessageSizeBytes: common.maximumMessageSizeBytes, + scheduler: { address: 'scheduler:8982' }, + httpListenAddress: ':7986', + maximumMemoryCachedDirectories: 1000, + instanceName: '', + buildDirectories: [{ + native: { + buildDirectoryPath: '/worker/build', + cacheDirectoryPath: '/worker/cache', + maximumCacheFileCount: 10000, + maximumCacheSizeBytes: 5 * 1024 * 1024 * 1024, + cacheReplacementPolicy: 'LEAST_RECENTLY_USED', + }, + runners: [{ + endpoint: { address: 'unix:///worker/runner' }, + concurrency: 8, + platform: { + properties: [{ name: "ISA" , value:"x86-64"}, { name:"OSFamily", value:"linux"}], + }, + defaultExecutionTimeout: '1800s', + maximumExecutionTimeout: '3600s', + workerId: { + }, + }], + }], +} \ No newline at end of file
