This is an automated email from the ASF dual-hosted git repository. mgrigorov pushed a commit to branch avro-3653-arm64-on-github-actions in repository https://gitbox.apache.org/repos/asf/avro.git
commit ae3b02272ed0ebb3ac10606965d4271bce878ca6 Author: Martin Tzvetanov Grigorov <[email protected]> AuthorDate: Mon Oct 2 16:37:26 2023 +0300 AVRO-3653: Use the new self-hosted ARM64 nodes provided by ASF Infra Signed-off-by: Martin Tzvetanov Grigorov <[email protected]> --- .github/workflows/test-arm64.yml | 370 ++++++++++++++++----------------------- lang/csharp/common.props | 2 +- lang/py/avro/test/test_bench.py | 4 +- lang/py/tox.ini | 5 +- 4 files changed, 159 insertions(+), 222 deletions(-) diff --git a/.github/workflows/test-arm64.yml b/.github/workflows/test-arm64.yml index 85f048e74..397b80af1 100644 --- a/.github/workflows/test-arm64.yml +++ b/.github/workflows/test-arm64.yml @@ -17,7 +17,8 @@ name: CI on Linux ARM64 on: push: - branches: [ master ] + branches: + - master pull_request: branches: - master @@ -25,11 +26,17 @@ on: jobs: java: name: Java on Linux ARM64 - runs-on: ubuntu-latest + runs-on: ["self-hosted", "asf-arm"] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 + + - name: Setup Temurin JDK + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 11 - name: Cache Local Maven Repository uses: actions/cache@v3 @@ -40,89 +47,64 @@ jobs: ${{ runner.os }}-maven- - name: Build - uses: uraimo/run-on-arch-action@v2 - with: - env: | - GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }} - arch: aarch64 - distro: ubuntu22.04 - githubToken: ${{ github.token }} - dockerRunArgs: | - --volume "${PWD}:/avro" --volume "${HOME}/.m2:/root/.m2" --workdir /avro/lang/java - install: | - apt-get update -q - apt-get install -q -y openjdk-8-jdk wget tar - run: | - set -x - export MAVEN_VERSION="3.9.1" - wget https://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz - tar zxvf apache-maven-$MAVEN_VERSION-bin.tar.gz - export M2_HOME=$PWD/apache-maven-$MAVEN_VERSION - export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-arm64" - export PATH="$M2_HOME/bin:$JAVA_HOME/bin:$PATH" - java -version - mvn -version - MAVEN_OPTS="-Dsurefire.excludes=*TestCustomCodec*,*TestAllCodecs*,*TestNettyServer*" ./build.sh clean test + run: | + set -x + cd lang/java + export MAVEN_VERSION="3.9.4" + wget https://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz + tar zxvf apache-maven-$MAVEN_VERSION-bin.tar.gz + export M2_HOME=$PWD/apache-maven-$MAVEN_VERSION + export PATH="$M2_HOME/bin:$PATH" + java -version + mvn -version + #MAVEN_OPTS="-Dsurefire.excludes=*TestCustomCodec*,*TestAllCodecs*,*TestNettyServer*" ./build.sh clean test + ./build.sh clean test c: name: C on Linux ARM64 - runs-on: ubuntu-latest + runs-on: ["self-hosted", "asf-arm"] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -q -y cmake liblzma-dev libsnappy-dev libjansson-dev zlib1g-dev pkg-config - name: Build - uses: uraimo/run-on-arch-action@v2 - with: - env: | - GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }} - arch: aarch64 - distro: ubuntu22.04 - githubToken: ${{ github.token }} - dockerRunArgs: | - --volume "${PWD}:/avro" --workdir /avro/lang/c - install: | - apt-get update -q - apt-get install -q -y cmake liblzma-dev libsnappy-dev libjansson-dev zlib1g-dev pkg-config - run: | - set -x - ./build.sh clean test + run: | + set -x + cd lang/c + ./build.sh clean test cplusplus: name: C++ on Linux ARM64 - runs-on: ubuntu-latest + runs-on: ["self-hosted", "asf-arm"] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -q -y gcc g++ libboost-all-dev cmake - name: Build - uses: uraimo/run-on-arch-action@v2 - with: - env: | - GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }} - arch: aarch64 - distro: ubuntu22.04 - githubToken: ${{ github.token }} - dockerRunArgs: | - --volume "${PWD}:/avro" --workdir /avro/lang/c++ - install: | - apt-get update -q - apt-get install -q -y gcc g++ libboost-all-dev cmake - run: | - set -x - ./build.sh clean test + run: | + set -x + cd lang/c++ + ./build.sh clean test csharp: name: C# on Linux ARM64 - runs-on: ubuntu-latest - env: - GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }} + runs-on: ["self-hosted", "asf-arm"] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Cache Nuget uses: actions/cache@v3 @@ -132,69 +114,56 @@ jobs: restore-keys: | ${{ runner.os }}-nuget- + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -q -y wget libzstd-dev libicu-dev + wget https://dot.net/v1/dotnet-install.sh + bash ./dotnet-install.sh --channel "3.1" --install-dir "$HOME/.dotnet" # 3.1 + bash ./dotnet-install.sh --channel "5.0" --install-dir "$HOME/.dotnet" # 5.0 + bash ./dotnet-install.sh --channel "6.0" --install-dir "$HOME/.dotnet" # 6.0 + bash ./dotnet-install.sh --channel "7.0" --install-dir "$HOME/.dotnet" # 7.0 + - name: Build - uses: uraimo/run-on-arch-action@v2 - with: - env: | - GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }} - arch: aarch64 - distro: ubuntu22.04 - githubToken: ${{ github.token }} - dockerRunArgs: | - --volume "${PWD}:/avro" --volume "$HOME/.nuget/packages:/root/.nuget/packages" --workdir /avro/lang/csharp - install: | - apt-get update -q - apt-get install -q -y wget libzstd-dev libicu-dev - wget https://dot.net/v1/dotnet-install.sh - bash ./dotnet-install.sh --channel "3.1" --install-dir "$HOME/.dotnet" # 3.1 - bash ./dotnet-install.sh --channel "5.0" --install-dir "$HOME/.dotnet" # 5.0 - bash ./dotnet-install.sh --channel "6.0" --install-dir "$HOME/.dotnet" # 6.0 - bash ./dotnet-install.sh --channel "7.0" --install-dir "$HOME/.dotnet" # 7.0 - run: | - set -x - export PATH=$HOME/.dotnet:$PATH - dotnet --list-sdks - ./build.sh clean test + run: | + set -x + export PATH=$HOME/.dotnet:$PATH + dotnet --list-sdks + cd lang/csharp + ./build.sh clean test python: name: Python on Linux ARM64 - runs-on: ubuntu-latest + runs-on: ["self-hosted", "asf-arm"] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 + # BENCH_MAX_WRITE_SECONDS: 300 + # BENCH_MAX_READ_SECONDS: 300 + + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -q -y software-properties-common + sudo add-apt-repository -y ppa:deadsnakes/ppa + sudo apt-get update -q + sudo apt-get install -q -y python3.8 python3.9 python3.10 python3.11 python3-pip git libbz2-dev libjansson-dev liblzma-dev libsnappy-dev libzstd-dev + python3 -m pip install --upgrade pip setuptools tox - name: Build - uses: uraimo/run-on-arch-action@v2 - with: - env: | - GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }} - BENCH_MAX_WRITE_SECONDS: 300 - BENCH_MAX_READ_SECONDS: 300 - arch: aarch64 - distro: ubuntu22.04 - githubToken: ${{ github.token }} - dockerRunArgs: | - --volume "${PWD}:/avro" --workdir /avro/lang/py - install: | - apt-get update -q - apt-get install -q -y software-properties-common - add-apt-repository -y ppa:deadsnakes/ppa - apt-get update -q - apt-get install -q -y python3.8 python3.9 python3.10 python3.11 python3-pip git libbz2-dev libjansson-dev liblzma-dev libsnappy-dev libzstd-dev - python3 -m pip install --upgrade pip setuptools tox - run: | - set -x - git config --global --add safe.directory /avro - ./build.sh clean test + run: | + set -x + cd lang/py + ./build.sh clean test ruby: name: Ruby on Linux ARM64 - runs-on: ubuntu-latest + runs-on: ["self-hosted", "asf-arm"] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Cache gems uses: actions/cache@v3 @@ -204,31 +173,24 @@ jobs: restore-keys: | ${{ runner.os }}-gems- + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -q -y ruby-dev bundler libsnappy-dev libyaml-dev - name: Build - uses: uraimo/run-on-arch-action@v2 - with: - env: | - GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }} - arch: aarch64 - distro: ubuntu22.04 - githubToken: ${{ github.token }} - dockerRunArgs: | - --volume "${PWD}:/avro" --workdir /avro/lang/ruby - install: | - apt-get update -q - apt-get install -q -y ruby-dev bundler libsnappy-dev libyaml-dev - run: | + run: | set -x + cd lang/ruby ./build.sh clean test rust: name: Rust on Linux ARM64 - runs-on: ubuntu-latest + runs-on: ["self-hosted", "asf-arm"] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Cache Cargo uses: actions/cache@v3 @@ -237,105 +199,85 @@ jobs: # and thus do not depend on the OS, arch nor rust version. path: ~/.cargo key: cargo-cache1- + - name: Cache Rust dependencies uses: actions/cache@v3 with: # these represent compiled steps of both dependencies and avro # and thus are specific for a particular OS, arch and rust version. - path: ~/target - key: ${{ runner.os }}-target-cache1-${{ matrix.rust }}- + path: lang/rust/target + key: ${{ runner.os }}-target-cache1-stable- - - name: Build - uses: uraimo/run-on-arch-action@v2 + - name: Rust Toolchain + uses: dtolnay/rust-toolchain@nightly with: - env: | - GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }} - arch: aarch64 - distro: ubuntu22.04 - githubToken: ${{ github.token }} - dockerRunArgs: | - --volume "${PWD}:/avro" --volume "$HOME/.cargo:/root/.cargo" --volume "$HOME/target:/root/avro/target" --workdir /avro/lang/rust - install: | - set -x - apt-get update -q - apt-get install -q -y cargo rustc - run: | - set -x - ./build.sh test + toolchain: stable + components: rustfmt + targets: x86_64-unknown-linux-gnu + + - name: Build + run: | + set -x + cd lang/rust + ./build.sh test perl: name: Perl on Linux ARM64 - runs-on: ubuntu-latest + runs-on: ["self-hosted", "asf-arm"] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -q -y libjansson-dev libcompress-raw-zlib-perl libcpan-uploader-perl libencode-perl libio-string-perl libjson-xs-perl libmodule-install-perl libmodule-install-readmefrompod-perl libobject-tiny-perl libperl-critic-perl libsnappy-dev libtest-exception-perl libtest-pod-perl cpanminus make gcc wget + cpanm Error::Simple + cpanm Regexp::Common + cpanm Try::Tiny + cpanm Compress::Zstd + cpanm Module::Install::Repository + cpanm inc::Module::Install - name: Build - uses: uraimo/run-on-arch-action@v2 - with: - env: | - GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }} - arch: aarch64 - distro: ubuntu22.04 - githubToken: ${{ github.token }} - dockerRunArgs: | - --volume "${PWD}:/avro" --workdir /avro/lang/perl - install: | - apt-get update -q - apt-get install -q -y libjansson-dev libcompress-raw-zlib-perl libcpan-uploader-perl libencode-perl libio-string-perl libjson-xs-perl libmodule-install-perl libmodule-install-readmefrompod-perl libobject-tiny-perl libperl-critic-perl libsnappy-dev libtest-exception-perl libtest-pod-perl cpanminus make gcc wget - cpanm Error::Simple - cpanm Regexp::Common - cpanm Try::Tiny - cpanm Compress::Zstd - cpanm Module::Install::Repository - cpanm inc::Module::Install - run: | - set -x - ./build.sh clean test + run: | + set -x + cd lang/perl + ./build.sh clean test php: name: PHP on Linux ARM64 - runs-on: ubuntu-latest + runs-on: ["self-hosted", "asf-arm"] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -q -y wget php php-xml php-mbstring php-curl php-gmp php-bz2 unzip libtidy-dev libpq5 + php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" + php -r "if (hash_file('sha384', 'composer-setup.php') === file_get_contents('https://composer.github.io/installer.sig')) { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" + php composer-setup.php --version=2.2.5 + php -r "unlink('composer-setup.php');" + sudo mv composer.phar /usr/local/bin/composer - name: Build - uses: uraimo/run-on-arch-action@v2 - with: - env: | - GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }} - arch: aarch64 - distro: ubuntu22.04 - githubToken: ${{ github.token }} - dockerRunArgs: | - --volume "${PWD}:/avro" --workdir /avro/lang/php - install: | - apt-get update -q - apt-get install -q -y wget php php-xml php-mbstring php-curl php-gmp php-bz2 unzip libtidy-dev libpq5 - php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" - php -r "if (hash_file('sha384', 'composer-setup.php') === file_get_contents('https://composer.github.io/installer.sig')) { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" - php composer-setup.php --version=2.2.5 - php -r "unlink('composer-setup.php');" - mv composer.phar /usr/local/bin/composer - run: | - set -x - echo $PATH - which composer - composer --version - ./build.sh clean test + run: | + set -x + cd lang/php + composer --version + ./build.sh clean test javascript: name: JavaScript on Linux ARM64 - runs-on: ubuntu-latest - env: - NODE_VERSION: "16.18.0" + runs-on: ["self-hosted", "asf-arm"] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Cache Npm uses: actions/cache@v3 @@ -345,23 +287,19 @@ jobs: restore-keys: | ${{ runner.os }}-node- - - name: Build - uses: uraimo/run-on-arch-action@v2 + - name: Setup Node + uses: actions/setup-node@v3 with: - env: | - GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }} - arch: aarch64 - distro: ubuntu22.04 - githubToken: ${{ github.token }} - dockerRunArgs: | - --volume "${PWD}:/avro" --volume "$HOME/.npm:/root/.npm" --workdir /avro/lang/js - install: | - set -x - apt-get update -q - apt-get install -q -y wget tar xz-utils - wget https://nodejs.org/dist/v${{ env.NODE_VERSION }}/node-v${{ env.NODE_VERSION }}-linux-arm64.tar.xz - tar xf node-v${{ env.NODE_VERSION }}-linux-arm64.tar.xz - run: | - set -x - export PATH=/node-v${{ env.NODE_VERSION }}-linux-arm64/bin:$PATH - ./build.sh clean test + node-version: 18 + + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -q -y wget tar xz-utils + + - name: Build + run: | + set -x + cd lang/js + #export PATH=/node-v${{ env.NODE_VERSION }}-linux-arm64/bin:$PATH + ./build.sh clean test diff --git a/lang/csharp/common.props b/lang/csharp/common.props index 6dd567c3c..1874ea2a1 100644 --- a/lang/csharp/common.props +++ b/lang/csharp/common.props @@ -19,7 +19,7 @@ <PropertyGroup Label="Common Versions"> <!-- Read version from shared VERSION.txt file --> - <AvroVersionFilePath>$(MSBuildThisFileDirectory)../../share/VERSION.txt</AvroVersionFilePath> + <AvroVersionFilePath>$(MSBuildThisFileDirectory)/../../share/VERSION.txt</AvroVersionFilePath> <AvroVersion>$([System.IO.File]::ReadAllText($(AvroVersionFilePath)).Trim())</AvroVersion> <!-- Parse out parts of the version --> diff --git a/lang/py/avro/test/test_bench.py b/lang/py/avro/test/test_bench.py index 05a4e64b8..460c5877e 100644 --- a/lang/py/avro/test/test_bench.py +++ b/lang/py/avro/test/test_bench.py @@ -55,8 +55,8 @@ SCHEMA = cast( READER = avro.io.DatumReader(SCHEMA) WRITER = avro.io.DatumWriter(SCHEMA) NUMBER_OF_TESTS = 10000 -MAX_WRITE_SECONDS = 10 if platform.python_implementation() == "PyPy" else int(os.environ.get("BENCH_MAX_WRITE_SECONDS", 3)) -MAX_READ_SECONDS = 10 if platform.python_implementation() == "PyPy" else int(os.environ.get("BENCH_MAX_READ_SECONDS", 3)) +MAX_WRITE_SECONDS = 10 if platform.python_implementation() == "PyPy" else 3 +MAX_READ_SECONDS = 10 if platform.python_implementation() == "PyPy" else 3 class TestBench(unittest.TestCase): diff --git a/lang/py/tox.ini b/lang/py/tox.ini index a7d02f857..67395abbf 100644 --- a/lang/py/tox.ini +++ b/lang/py/tox.ini @@ -25,6 +25,7 @@ envlist = py38 py39 py310 + py311 pypy3.7 pypy3.8 pypy3.9 @@ -58,9 +59,7 @@ commands = commands_post = coverage combine --append coverage report -passenv = - BENCH_MAX_READ_SECONDS - BENCH_MAX_WRITE_SECONDS + [testenv:build] ## # A virtual environment only used to build the wheel
