On 6/20/24 14:57, Ales Musil wrote: > The DPDK was built as extra step in the CI, however it is useful to > have it inside the container prepared. This also helps with > reproduction of failures with DPDK by having the exact version inside > the container already. > > Also bump the Ubuntu version for container builds to 24.04 to get > Podman 4. This is required to avoid the tar permissions error: > > tar: dpdk-23.11/.ci: Cannot change mode to rwxrwxr-x: Operation not permitted > > Signed-off-by: Ales Musil <[email protected]> > --- > .ci/ci.sh | 6 -- > .ci/dpdk-build.sh | 62 ------------------ > .ci/dpdk-prepare.sh | 11 ---- > .ci/linux-build.sh | 7 +- > .github/workflows/containers.yml | 2 +- > .github/workflows/test.yml | 81 +----------------------- > Makefile.am | 2 - > utilities/containers/fedora/Dockerfile | 1 + > utilities/containers/prepare.sh | 56 ++++++++++++++++ > utilities/containers/py-requirements.txt | 1 + > utilities/containers/ubuntu/Dockerfile | 1 + > 11 files changed, 66 insertions(+), 164 deletions(-) > delete mode 100755 .ci/dpdk-build.sh > delete mode 100755 .ci/dpdk-prepare.sh > > diff --git a/.ci/ci.sh b/.ci/ci.sh > index 6beeace84..f543967dc 100755 > --- a/.ci/ci.sh > +++ b/.ci/ci.sh > @@ -16,7 +16,6 @@ > > OVN_PATH=${OVN_PATH:-$PWD} > OVS_PATH=${OVS_PATH:-$OVN_PATH/ovs} > -DPDK_PATH=${DPDK_PATH:-$OVN_PATH/dpdk-dir} > CONTAINER_CMD=${CONTAINER_CMD:-podman} > CONTAINER_WORKSPACE="/workspace" > CONTAINER_WORKDIR="/workspace/ovn-tmp" > @@ -163,17 +162,12 @@ if [ "$ARCH" = "aarch64" ] && ! check_clang_version_ge > "16.0.0"; then > ASAN_OPTIONS="detect_leaks=0" > fi > > -if [ -z "$DPDK" ]; then > - mkdir -p "$DPDK_PATH" > -fi > - > CONTAINER_ID="$($CONTAINER_CMD run --privileged -d \ > --pids-limit=-1 \ > --env ASAN_OPTIONS=$ASAN_OPTIONS \ > -v /lib/modules/$(uname -r):/lib/modules/$(uname -r):ro \ > -v $OVN_PATH:$CONTAINER_WORKSPACE/ovn:Z \ > -v $OVS_PATH:$CONTAINER_WORKSPACE/ovs:Z \ > - -v $DPDK_PATH:$CONTAINER_WORKSPACE/dpdk-dir:Z \ > $IMAGE_NAME)" > trap remove_container EXIT > > diff --git a/.ci/dpdk-build.sh b/.ci/dpdk-build.sh > deleted file mode 100755 > index 0c13c98c9..000000000 > --- a/.ci/dpdk-build.sh > +++ /dev/null > @@ -1,62 +0,0 @@ > -#!/bin/bash > - > -set -o errexit > -set -x > - > -function build_dpdk() > -{ > - local DPDK_VER=$1 > - local DPDK_OPTS="" > - local DPDK_INSTALL_DIR="$(pwd)/dpdk-dir" > - local VERSION_FILE="$DPDK_INSTALL_DIR/cached-version" > - > - rm -rf dpdk-src > - rm -rf $DPDK_INSTALL_DIR > - > - if [ "${DPDK_VER##refs/*/}" != "${DPDK_VER}" ]; then > - git clone --single-branch $DPDK_GIT dpdk-src -b > "${DPDK_VER##refs/*/}" > - pushd dpdk-src > - git log -1 --oneline > - else > - wget https://fast.dpdk.org/rel/dpdk-$1.tar.xz > - tar xvf dpdk-$1.tar.xz > /dev/null > - DIR_NAME=$(tar -tf dpdk-$1.tar.xz | head -1 | cut -f1 -d"/") > - mv ${DIR_NAME} dpdk-src > - pushd dpdk-src > - fi > - > - # Switching to 'default' machine to make the dpdk cache usable on > - # different CPUs. We can't be sure that all CI machines are exactly same. > - DPDK_OPTS="$DPDK_OPTS -Dmachine=default" > - > - # Disable building DPDK unit tests. Not needed for OVS build or tests. > - DPDK_OPTS="$DPDK_OPTS -Dtests=false" > - > - # Disable DPDK developer mode, this results in less build checks and less > - # meson verbose outputs. > - DPDK_OPTS="$DPDK_OPTS -Ddeveloper_mode=disabled" > - > - # OVS compilation and the "ovn-system-dpdk" unit tests (run in the CI) > - # only depend on virtio/tap drivers. > - # We can disable all remaining drivers to save compilation time. > - DPDK_OPTS="$DPDK_OPTS -Denable_drivers=net/null,net/tap,net/virtio" > - # OVS depends on the vhost library (and its dependencies). > - # net/tap depends on the gso library. > - DPDK_OPTS="$DPDK_OPTS -Denable_libs=cryptodev,dmadev,gso,vhost" > - > - # Install DPDK using prefix. > - DPDK_OPTS="$DPDK_OPTS --prefix=$DPDK_INSTALL_DIR" > - > - meson $DPDK_OPTS build > - ninja -C build > - ninja -C build install > - popd > - > - # Remove examples sources. > - rm -rf $DPDK_INSTALL_DIR/share/dpdk/examples > - > - echo "Installed DPDK in $DPDK_INSTALL_DIR" > - echo "${DPDK_VER}" > ${VERSION_FILE} > -} > - > -build_dpdk $DPDK_VER > diff --git a/.ci/dpdk-prepare.sh b/.ci/dpdk-prepare.sh > deleted file mode 100755 > index 5543da90a..000000000 > --- a/.ci/dpdk-prepare.sh > +++ /dev/null > @@ -1,11 +0,0 @@ > -#!/bin/bash > - > -set -ev > - > -# Installing wheel separately because it may be needed to build some > -# of the packages during dependency backtracking and pip >= 22.0 will > -# stop backtracking on build failures: > -# https://github.com/pypa/pip/issues/10655 > -pip3 install --disable-pip-version-check --user wheel > -pip3 install --disable-pip-version-check --user pyelftools > -pip3 install --user 'meson==0.53.2' > diff --git a/.ci/linux-build.sh b/.ci/linux-build.sh > index d4c57d32c..75a9480f9 100755 > --- a/.ci/linux-build.sh > +++ b/.ci/linux-build.sh > @@ -15,10 +15,10 @@ function install_dpdk() > { > local DPDK_INSTALL_DIR="$(pwd)/dpdk-dir" > local VERSION_FILE="${DPDK_INSTALL_DIR}/cached-version" > - local DPDK_LIB=${DPDK_INSTALL_DIR}/lib/x86_64-linux-gnu > + local DPDK_PC=$(find $DPDK_INSTALL_DIR -type f -name libdpdk-libs.pc) > > # Export the following path for pkg-config to find the .pc file. > - export PKG_CONFIG_PATH=$DPDK_LIB/pkgconfig/:$PKG_CONFIG_PATH > + export PKG_CONFIG_PATH="$(dirname $DPDK_PC):$PKG_CONFIG_PATH" > > if [ ! -f "${VERSION_FILE}" ]; then > echo "Could not find DPDK in $DPDK_INSTALL_DIR" > @@ -26,8 +26,7 @@ function install_dpdk() > fi > > # As we build inside a container we need to update the prefix. > - sed -i -E "s|^prefix=.*|prefix=${DPDK_INSTALL_DIR}|" \ > - "$DPDK_LIB/pkgconfig/libdpdk-libs.pc" > + sed -i -E "s|^prefix=.*|prefix=${DPDK_INSTALL_DIR}|" $DPDK_PC > > # Update the library paths. > sudo ldconfig > diff --git a/.github/workflows/containers.yml > b/.github/workflows/containers.yml > index 87e28d645..4cce255a8 100644 > --- a/.github/workflows/containers.yml > +++ b/.github/workflows/containers.yml > @@ -15,7 +15,7 @@ env: > > jobs: > container: > - runs-on: ubuntu-22.04 > + runs-on: ubuntu-24.04 > strategy: > matrix: > distro: [ fedora, ubuntu ] > diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml > index efe2dac25..befa0bfac 100644 > --- a/.github/workflows/test.yml > +++ b/.github/workflows/test.yml > @@ -12,74 +12,6 @@ concurrency: > cancel-in-progress: true > > jobs: > - build-dpdk: > - env: > - dependencies: gcc libnuma-dev ninja-build > - CC: gcc > - DPDK_GIT: https://dpdk.org/git/dpdk > - DPDK_VER: 23.11 > - name: dpdk gcc > - outputs: > - dpdk_key: ${{ steps.gen_dpdk_key.outputs.key }} > - runs-on: ubuntu-20.04 > - timeout-minutes: 30 > - > - steps: > - - name: checkout > - uses: actions/checkout@v4 > - > - - name: update PATH > - run: | > - echo "$HOME/bin" >> $GITHUB_PATH > - echo "$HOME/.local/bin" >> $GITHUB_PATH > - > - - name: create ci signature file for the dpdk cache key > - # This will collect most of DPDK related lines, so hash will be > different > - # if something changed in a way we're building DPDK including DPDK_VER. > - # This also allows us to use cache from any branch as long as version > - # and a way we're building DPDK stays the same. > - run: | > - grep -irE 'RTE_|DPDK|meson|ninja' .ci/dpdk-* > dpdk-ci-signature > - grep -rwE 'DPDK_GIT|DPDK_VER' .github/ >> dpdk-ci-signature > - if [ "${DPDK_VER##refs/*/}" != "${DPDK_VER}" ]; then > - git ls-remote --heads $DPDK_GIT $DPDK_VER >> dpdk-ci-signature > - fi > - cat dpdk-ci-signature > - > - - name: generate ci DPDK key > - id: gen_dpdk_key > - env: > - ci_key: ${{ hashFiles('dpdk-ci-signature') }} > - run: echo 'key=dpdk-${{ env.ci_key }}' >> $GITHUB_OUTPUT > - > - - name: cache > - id: dpdk_cache > - uses: actions/cache@v4 > - with: > - path: dpdk-dir > - key: ${{ steps.gen_dpdk_key.outputs.key }} > - > - - name: set up python > - if: steps.dpdk_cache.outputs.cache-hit != 'true' > - uses: actions/setup-python@v5 > - with: > - python-version: '3.9' > - > - - name: update APT cache > - if: steps.dpdk_cache.outputs.cache-hit != 'true' > - run: sudo apt update || true > - - name: install common dependencies > - if: steps.dpdk_cache.outputs.cache-hit != 'true' > - run: sudo apt install -y ${{ env.dependencies }} > - > - - name: prepare > - if: steps.dpdk_cache.outputs.cache-hit != 'true' > - run: ./.ci/dpdk-prepare.sh > - > - - name: build > - if: steps.dpdk_cache.outputs.cache-hit != 'true' > - run: ./.ci/dpdk-build.sh > - > prepare-container: > # This job has the following matrix, x: Job trigger, y: Branch > # (scheduled jobs run only on main): > @@ -93,7 +25,7 @@ jobs: > env: > DEPENDENCIES: podman > name: Prepare container > - runs-on: ubuntu-22.04 > + runs-on: ubuntu-24.04 > > steps: > - uses: actions/checkout@v4 > @@ -124,7 +56,7 @@ jobs: > run: podman pull ghcr.io/ovn-org/ovn-tests:${{ env.IMAGE_DISTRO }} > > - name: Export image > - run: podman save -o /tmp/image.tar ovn-org/ovn-tests > + run: podman save -o /tmp/image.tar ovn-org/ovn-tests:${{ > env.IMAGE_DISTRO }} > > - name: Cache image > id: image_cache > @@ -134,7 +66,7 @@ jobs: > key: ${{ github.sha }}/${{ github.event_name }} > > build-linux: > - needs: [build-dpdk, prepare-container] > + needs: [prepare-container] > env: > ARCH: ${{ matrix.cfg.arch }} > CC: ${{ matrix.cfg.compiler }} > @@ -204,13 +136,6 @@ jobs: > sort -V | tail -1) > working-directory: ovs > > - - name: cache dpdk > - if: matrix.cfg.dpdk != '' > - uses: actions/cache@v4 > - with: > - path: dpdk-dir > - key: ${{ needs.build-dpdk.outputs.dpdk_key }} > - > - name: image cache > id: image_cache > uses: actions/cache@v4 > diff --git a/Makefile.am b/Makefile.am > index bfc9565e8..6b0f1913a 100644 > --- a/Makefile.am > +++ b/Makefile.am > @@ -87,8 +87,6 @@ EXTRA_DIST = \ > NOTICE \ > .cirrus.yml \ > .ci/ci.sh \ > - .ci/dpdk-build.sh \ > - .ci/dpdk-prepare.sh \ > .ci/linux-build.sh \ > .ci/linux-util.sh \ > .ci/osx-build.sh \ > diff --git a/utilities/containers/fedora/Dockerfile > b/utilities/containers/fedora/Dockerfile > index 9e17a6b20..078180cff 100755 > --- a/utilities/containers/fedora/Dockerfile > +++ b/utilities/containers/fedora/Dockerfile > @@ -27,6 +27,7 @@ RUN dnf -y update \ > libcap-ng-devel \ > libtool \ > net-tools \ > + ninja-build \ > nmap-ncat \ > numactl-devel \ > openssl \ > diff --git a/utilities/containers/prepare.sh b/utilities/containers/prepare.sh > index 3aac94590..8e35b4354 100755 > --- a/utilities/containers/prepare.sh > +++ b/utilities/containers/prepare.sh > @@ -1,5 +1,8 @@ > #!/bin/bash -xe > > +DPDK_GIT=https://dpdk.org/git/dpdk > +DPDK_VER=23.11 > + > function compile_sparse() > { > git clone git://git.kernel.org/pub/scm/devel/sparse/sparse.git \ > @@ -31,6 +34,59 @@ function install_python_dep() > python3 -m pip install -r /tmp/py-requirements.txt > } > > +function build_dpdk() > +{ > + local DPDK_OPTS="" > + local DPDK_INSTALL_DIR="$(pwd)/dpdk-dir" > + local VERSION_FILE="$DPDK_INSTALL_DIR/cached-version" > + > + if [ "${DPDK_VER##refs/*/}" != "${DPDK_VER}" ]; then > + git clone --single-branch $DPDK_GIT dpdk-src \ > + -b "${DPDK_VER##refs/*/}" > + pushd dpdk-src > + git log -1 --oneline > + else > + curl -O https://fast.dpdk.org/rel/dpdk-$DPDK_VER.tar.xz > + tar --no-same-owner -xvf dpdk-$DPDK_VER.tar.xz > /dev/null > + DIR_NAME=$(tar -tf dpdk-$DPDK_VER.tar.xz | head -1 | cut -f1 -d"/") > + mv ${DIR_NAME} dpdk-src > + pushd dpdk-src > + fi > + > + # Switching to 'default' machine to make the dpdk cache usable on > + # different CPUs. We can't be sure that all CI machines are exactly > same. > + DPDK_OPTS="$DPDK_OPTS -Dmachine=default" > + > + # Disable building DPDK unit tests. Not needed for OVS build or tests. > + DPDK_OPTS="$DPDK_OPTS -Dtests=false" > + > + # Disable DPDK developer mode, this results in less build checks and > less > + # meson verbose outputs. > + DPDK_OPTS="$DPDK_OPTS -Ddeveloper_mode=disabled" > + > + # OVS compilation and the "ovn-system-dpdk" unit tests (run in the CI) > + # only depend on virtio/tap drivers. > + # We can disable all remaining drivers to save compilation time. > + DPDK_OPTS="$DPDK_OPTS -Denable_drivers=net/null,net/tap,net/virtio" > + # OVS depends on the vhost library (and its dependencies). > + # net/tap depends on the gso library. > + DPDK_OPTS="$DPDK_OPTS -Denable_libs=cryptodev,dmadev,gso,vhost" > + > + # Install DPDK using prefix. > + DPDK_OPTS="$DPDK_OPTS --prefix=$DPDK_INSTALL_DIR" > + > + meson $DPDK_OPTS build > + ninja -C build > + ninja -C build install > + popd > + > + # Remove examples sources. > + rm -rf $DPDK_INSTALL_DIR/share/dpdk/examples > + > + echo "${DPDK_VER}" > ${VERSION_FILE} > +} > + > compile_sparse > compile_openbfdd > install_python_dep > +build_dpdk > diff --git a/utilities/containers/py-requirements.txt > b/utilities/containers/py-requirements.txt > index a8e8f17da..7b12b4ab5 100644 > --- a/utilities/containers/py-requirements.txt > +++ b/utilities/containers/py-requirements.txt > @@ -1,4 +1,5 @@ > flake8>=6.1.0 > +meson==0.53.2
Not a full review, but you're chanign the base image and the python version as a result for the DPDK build. This version of meson is not compatible with Python 3.12. Here is an exmaple of what I did for OVS: https://github.com/openvswitch/ovs/commit/16b7475414fa1eaf0ab1d723fdb6978060511c44 Best regards, Ilya Maximets. _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
