This is an automated email from the ASF dual-hosted git repository. chandan pushed a commit to branch chandan/compose in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit cca3373a28d775a99266765c11d373bf38a5733c Author: Chandan Singh <[email protected]> AuthorDate: Tue Jan 12 21:27:57 2021 +0000 ci: Replace docker CLI monstrosity with Compose Replace our bespoke and rather long to type `docker` CLI invocations with more manageable docker-compose commands. This should also be useful for better replicating what the CI does on laptops, and also to help users to run tests inside a container (especially when that's not possible natively like on MacOS). On a technical note, Compose does support YAML anchors, unlike GitHub Actions. That allows us to reuse the common configuration options without having to duplicate them everywhere. This also removes most of the duplication in the image names, save for the test about missing dependencies. That should also be fixable easily. --- .github/ci.env | 1 + .github/compose/ci.docker-compose.yml | 60 ++++++++++++++ .github/workflows/ci.yml | 148 ++++++++++------------------------ 3 files changed, 103 insertions(+), 106 deletions(-) diff --git a/.github/ci.env b/.github/ci.env new file mode 100644 index 0000000..a1dbb49 --- /dev/null +++ b/.github/ci.env @@ -0,0 +1 @@ +CI_IMAGE_VERSION=master-240510553 diff --git a/.github/compose/ci.docker-compose.yml b/.github/compose/ci.docker-compose.yml new file mode 100644 index 0000000..cc7a544 --- /dev/null +++ b/.github/compose/ci.docker-compose.yml @@ -0,0 +1,60 @@ +version: '3.4' + +x-tests-template: &tests-template + command: tox -vvvvv -- --color=yes --integration + devices: + - /dev/fuse:/dev/fuse + environment: + BST_PLUGINS_EXPERIMENTAL_VERSION: 1.93.4 + TOXENV: py36,py37,py38-nocover,py36-plugins,py37-plugins,py38-plugins-nocover + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${CI_IMAGE_VERSION:-latest} + privileged: true + volumes: + - ../..:/home/testuser/buildstream + working_dir: /home/testuser/buildstream + + +services: + + fedora-31: + <<: *tests-template + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:31-${CI_IMAGE_VERSION:-latest} + + fedora-32: + <<: *tests-template + + debian-10: + <<: *tests-template + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-debian:10-${CI_IMAGE_VERSION:-latest} + + ubuntu-18.04: + <<: *tests-template + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-ubuntu:18.04-${CI_IMAGE_VERSION:-latest} + + tests-centos-7.7.1908: + <<: *tests-template + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-centos:7.7.1908-${CI_IMAGE_VERSION:-latest} + + no-usedevelop: + <<: *tests-template + environment: + TOXENV: py36-nocover,py37-nocover,py38-nocover + + plugins-master: + <<: *tests-template + environment: + BST_PLUGINS_EXPERIMENTAL_VERSION: master + + docs: + <<: *tests-template + command: tox -e docs + environment: + BST_FORCE_SESSION_REBUILD: 1 + + lint: + <<: *tests-template + command: tox -e lint + + mypy: + <<: *tests-template + command: tox -e docs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9f35c9..01252ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,6 @@ on: # - publishing docs to gh-pages # - persistent artifact cache # - overnight jobs -# - other one-off jobs like missing-deps, plugin jobs etc # - wsl tasks (TODO: Check if GitHub's Windows runners allow WSL) # # New opportunities: @@ -53,25 +52,21 @@ jobs: fail-fast: false matrix: - # Main test targets, the name defines the image that will be used as - # the base for running tests. + # The names here should map to a valid service defined in + # "../compose/ci.docker-compose.yml" test-name: - - debian:10 - - fedora:31 - - fedora:32 - - ubuntu:18.04 - - centos:7.7.1908 + - debian-10 + - fedora-31 + - fedora-32 + - ubuntu-18.04 + - centos-7.7.1908 + - no-usedevelop + - lint + - mypy include: - # Ensure that tests also pass without `--develop` flag. - - test-name: no-usedevelop - image-name: fedora:32 - toxenv: py36-nocover,py37-nocover,py38-nocover - # Test the master version of some external plugins - test-name: plugins-master - image-name: fedora:32 - bst-plugins-experimental-version: master allow-failure: true steps: @@ -80,50 +75,43 @@ jobs: # BuildStream requires tags to be able to find its version. with: fetch-depth: 0 - # XXX: Our run command looks like a monstrosity because we need to - # specify `--device /dev/fuse` and there's no way to do that using the - # `container` directive directly. - # This is also why we have forward environment variables by hand. - # TODO: In future, we should find a way to simplify this. See some - # relevant discussion at: - # https://github.community/t/how-to-run-privileged-docker-container/16431. - # XXX: Value of `volume` and `workdir` must match how GitHub - # Actions sets up paths. - # TODO: Have test user pre-created in the test image. - - name: Run tox inside a container - run: | - - cat << EOF > runtox.sh - #!/bin/bash - - # Create user - useradd -Um buildstream - chown -R buildstream:buildstream . - # Diagnostics - echo "Running diagnostics checks" - mount - df -h - tox --version + - name: Run tests with Docker Compose + run: | + docker-compose \ + --env-file ${GITHUB_WORKSPACE}/.github/ci.env \ + --file ${GITHUB_WORKSPACE}/.github/compose/ci.docker-compose.yml \ + run \ + ${{ matrix.test-name }} - # Run tox as user, ensure we have a login shell - echo "Running tests" - su buildstream -c '/bin/bash --login -c "tox -vvvvv -- $PYTEST_ARGS"' - EOF + docs: + runs-on: ubuntu-20.04 + env: + BST_FORCE_SESSION_REBUILD: 1 + steps: + - name: Check out repository + uses: actions/checkout@v2 + # BuildStream requires tags to be able to find its version. + with: + fetch-depth: 0 - chmod +x runtox.sh + - name: Build documentation using Docker Compose + run: | + docker-compose \ + --env-file ${GITHUB_WORKSPACE}/.github/ci.env \ + --file ${GITHUB_WORKSPACE}/.github/compose/ci.docker-compose.yml \ + run \ + docs - docker run \ - --privileged \ - --device /dev/fuse \ - --env PYTEST_ARGS \ - --env TOXENV=${{ matrix.toxenv || env.TOXENV }} \ - --env BST_PLUGINS_EXPERIMENTAL_VERSION=${{ matrix.bst-plugins-experimental-version || env.BST_PLUGINS_EXPERIMENTAL_VERSION }} \ - --volume /home/runner/work:/__w \ - --workdir /__w/buildstream/buildstream \ - "$CI_IMAGE_PREFIX"-${{ matrix.image-name || matrix.test-name }}-"$CI_IMAGE_SUFFIX" \ - ./runtox.sh + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: docs + path: doc/build/html + # TODO: Consolidate with rest of the Docker Compose setup. + # Providing a base image with the dependencies not added in the first place + # should make that simpler. tests-fedora-missing-deps: runs-on: ubuntu-20.04 container: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-master-177137613 @@ -146,55 +134,3 @@ jobs: chown -R buildstream:buildstream . - name: Run tests run: su buildstream -c "tox -- $PYTEST_ARGS" - - mypy: - runs-on: ubuntu-20.04 - container: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-master-177137613 - steps: - - name: Check out repository - uses: actions/checkout@v2 - # BuildStream requires tags to be able to find its version. - with: - fetch-depth: 0 - - name: Run tox inside a container - run: tox -e mypy - - lint: - runs-on: ubuntu-20.04 - container: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-master-177137613 - steps: - - name: Check out repository - uses: actions/checkout@v2 - # BuildStream requires tags to be able to find its version. - with: - fetch-depth: 0 - - name: Run tox inside a container - run: tox -e format-check,lint - - docs: - runs-on: ubuntu-20.04 - env: - BST_FORCE_SESSION_REBUILD: 1 - steps: - - name: Check out repository - uses: actions/checkout@v2 - # BuildStream requires tags to be able to find its version. - with: - fetch-depth: 0 - - name: Run tox inside a container - run: | - docker run \ - --privileged \ - --device /dev/fuse \ - --env BST_FORCE_SESSION_REBUILD \ - --volume /home/runner/work:/__w \ - --workdir /__w/buildstream/buildstream \ - $CI_IMAGE \ - tox -e docs - - - name: Upload artifacts - uses: actions/upload-artifact@v2 - with: - name: docs - path: doc/build/html -
