This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 7b6b5deb670debaf780eecf7c8266d1755b54234 Author: Gabor Gyimesi <[email protected]> AuthorDate: Tue Feb 18 13:31:06 2025 +0100 MINIFICPP-2422 Remove Ubuntu 20.04 support - Update CI scripts to use Ubuntu 24.04 workers instead of Ubuntu 20.04 - Update python dependencies for docker tests Closes #1928 Signed-off-by: Marton Szasz <[email protected]> --- .github/workflows/ci.yml | 4 ++-- .github/workflows/clear-actions-cache.yml | 6 ++--- aptitude.sh | 8 ------- bootstrap/package_manager.py | 6 ----- docker/DockerBuild.sh | 2 +- docker/requirements.txt | 21 +++++++++-------- .../cluster/checkers/ElasticSearchChecker.py | 2 ++ .../containers/AzureStorageServerContainer.py | 2 +- .../cluster/containers/MinifiC2ServerContainer.py | 17 ++++++++++---- .../cluster/containers/SyslogTcpClientContainer.py | 2 +- .../cluster/containers/SyslogUdpClientContainer.py | 2 +- docker/test/integration/features/steps/steps.py | 7 +++--- .../github_actions_cache_cleanup_tests.py | 26 +++++++++++----------- 13 files changed, 50 insertions(+), 55 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9e88a418..c6b3d97c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -475,7 +475,7 @@ jobs: path: build/bin rocky: name: "rocky" - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 180 steps: - id: checkout @@ -589,7 +589,7 @@ jobs: docker_tests: name: "Docker integration tests" needs: docker_build - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 180 steps: - id: checkout diff --git a/.github/workflows/clear-actions-cache.yml b/.github/workflows/clear-actions-cache.yml index 6c76da844..208382ef7 100644 --- a/.github/workflows/clear-actions-cache.yml +++ b/.github/workflows/clear-actions-cache.yml @@ -6,9 +6,9 @@ on: - completed workflow_dispatch: jobs: - ubuntu_20_04: - name: "ubuntu-20.04" - runs-on: ubuntu-20.04 + ubuntu_24_04: + name: "ubuntu-24.04" + runs-on: ubuntu-24.04 steps: - id: checkout uses: actions/checkout@v4 diff --git a/aptitude.sh b/aptitude.sh index f1f642eb5..9528fed32 100644 --- a/aptitude.sh +++ b/aptitude.sh @@ -23,10 +23,6 @@ verify_enable_platform(){ add_os_flags() { CC=${CC:-gcc} CXX=${CXX:-g++} - if [[ "$OS" = Ubuntu* && "$OS_MAJOR" -lt 22 ]]; then - CC=${CC:-gcc-11} - CXX=${CXX:-g++-11} - fi export CC export CXX CMAKE_BUILD_COMMAND="${CMAKE_BUILD_COMMAND} -DCMAKE_C_COMPILER=$CC -DCMAKE_CXX_COMPILER=$CXX" @@ -41,10 +37,6 @@ bootstrap_cmake(){ } bootstrap_compiler() { compiler_pkgs="gcc g++" - if [[ "$OS" = Ubuntu* && "$OS_MAJOR" -lt 22 ]]; then - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - compiler_pkgs="gcc-11 g++-11" - fi # shellcheck disable=SC2086 sudo apt-get -y install $compiler_pkgs } diff --git a/bootstrap/package_manager.py b/bootstrap/package_manager.py index 5f061b3a8..049697c71 100644 --- a/bootstrap/package_manager.py +++ b/bootstrap/package_manager.py @@ -127,12 +127,6 @@ class AptPackageManager(PackageManager): return set(lines) def install_compiler(self) -> str: - if distro.id() == "ubuntu" and int(distro.major_version()) < 22: - self.install({"compiler_prereq": {"apt-transport-https", "ca-certificates", "software-properties-common"}}) - _run_command_with_confirm("sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test", - no_confirm=self.no_confirm) - self.install({"compiler": {"build-essential", "g++-11"}}) - return "-DCMAKE_C_COMPILER=gcc-11 -DCMAKE_CXX_COMPILER=g++-11" self.install({"compiler": {"g++"}}) return "" diff --git a/docker/DockerBuild.sh b/docker/DockerBuild.sh index 50127fe2c..12a9a2982 100755 --- a/docker/DockerBuild.sh +++ b/docker/DockerBuild.sh @@ -43,7 +43,7 @@ function usage { echo "-p, --prefix Additional prefix added to the image tag" echo "-u, --uid User id to be used in the Docker image (default: 1000)" echo "-g, --gid Group id to be used in the Docker image (default: 1000)" - echo "-d, --distro-name Linux distribution build to be used for alternative builds (bionic|focal|fedora|centos)" + echo "-d, --distro-name Linux distribution build to be used for alternative builds (centos|rockylinux)" echo "-l --dump-location Path where to the output dump to be put" echo "-c --cmake-param CMake parameter passed in PARAM=value format" echo "-o --options Minifi options string" diff --git a/docker/requirements.txt b/docker/requirements.txt index 5c01145e5..a5b633219 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,15 +1,14 @@ pyjks==20.0.0 -shortuuid==1.0.11 -behavex==2.0.1 -docker==5.0.0 -kafka-python-ng==2.2.3 +shortuuid==1.0.13 +behavex==4.0.10 +docker==7.1.0 confluent-kafka==2.8.0 -PyYAML==6.0.1 -m2crypto==0.43.0 -watchdog==2.1.2 -pyopenssl==23.0.0 -azure-storage-blob==12.13.0 -prometheus-api-client==0.5.0 +PyYAML==6.0.2 +m2crypto==0.41.0 +watchdog==6.0.0 +pyopenssl==25.0.0 +azure-storage-blob==12.24.1 +prometheus-api-client==0.5.5 humanfriendly==10.0 requests<2.29 # https://github.com/docker/docker-py/issues/3113 -couchbase==4.3.2 +couchbase==4.3.5 diff --git a/docker/test/integration/cluster/checkers/ElasticSearchChecker.py b/docker/test/integration/cluster/checkers/ElasticSearchChecker.py index e35baa25c..b5c4e5983 100644 --- a/docker/test/integration/cluster/checkers/ElasticSearchChecker.py +++ b/docker/test/integration/cluster/checkers/ElasticSearchChecker.py @@ -13,12 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. import json +from utils import retry_check class ElasticSearchChecker: def __init__(self, container_communicator): self.container_communicator = container_communicator + @retry_check() def is_elasticsearch_empty(self, container_name): (code, output) = self.container_communicator.execute_command(container_name, ["curl", "-s", "-u", "elastic:password", "-k", "-XGET", "https://localhost:9200/_search";]) return code == 0 and '"hits":[]' in output diff --git a/docker/test/integration/cluster/containers/AzureStorageServerContainer.py b/docker/test/integration/cluster/containers/AzureStorageServerContainer.py index 716b0d339..f9dfbb273 100644 --- a/docker/test/integration/cluster/containers/AzureStorageServerContainer.py +++ b/docker/test/integration/cluster/containers/AzureStorageServerContainer.py @@ -31,7 +31,7 @@ class AzureStorageServerContainer(Container): logging.info('Creating and running azure storage server docker container...') self.client.containers.run( - "mcr.microsoft.com/azure-storage/azurite:3.23.0", + "mcr.microsoft.com/azure-storage/azurite:3.33.0", detach=True, name=self.name, network=self.network.name, diff --git a/docker/test/integration/cluster/containers/MinifiC2ServerContainer.py b/docker/test/integration/cluster/containers/MinifiC2ServerContainer.py index 1823166d3..9be1cdeb1 100644 --- a/docker/test/integration/cluster/containers/MinifiC2ServerContainer.py +++ b/docker/test/integration/cluster/containers/MinifiC2ServerContainer.py @@ -20,6 +20,8 @@ import jks from .Container import Container from OpenSSL import crypto +from cryptography.hazmat.primitives.serialization import pkcs12, BestAvailableEncryption, load_pem_private_key +from cryptography import x509 from ssl_utils.SSL_cert_utils import make_server_cert @@ -37,10 +39,17 @@ class MinifiC2ServerContainer(Container): os.chmod(self.server_keystore_file.name, 0o644) self.server_truststore_file = tempfile.NamedTemporaryFile(delete=False) - pkcs12 = crypto.PKCS12() - pkcs12.set_privatekey(feature_context.root_ca_key) - pkcs12.set_certificate(feature_context.root_ca_cert) - pkcs12_data = pkcs12.export(passphrase="abcdefgh") + private_key_pem = crypto.dump_privatekey(crypto.FILETYPE_PEM, feature_context.root_ca_key) + private_key = load_pem_private_key(private_key_pem, password=None) + certificate_pem = crypto.dump_certificate(crypto.FILETYPE_PEM, feature_context.root_ca_cert) + certificate = x509.load_pem_x509_certificate(certificate_pem) + pkcs12_data = pkcs12.serialize_key_and_certificates( + name=None, + key=private_key, + cert=certificate, + cas=None, + encryption_algorithm=BestAvailableEncryption(b'abcdefgh') + ) self.server_truststore_file.write(pkcs12_data) self.server_truststore_file.close() os.chmod(self.server_truststore_file.name, 0o644) diff --git a/docker/test/integration/cluster/containers/SyslogTcpClientContainer.py b/docker/test/integration/cluster/containers/SyslogTcpClientContainer.py index d34757124..b8af4d37c 100644 --- a/docker/test/integration/cluster/containers/SyslogTcpClientContainer.py +++ b/docker/test/integration/cluster/containers/SyslogTcpClientContainer.py @@ -29,7 +29,7 @@ class SyslogTcpClientContainer(Container): logging.info('Creating and running a Syslog tcp client docker container...') self.client.containers.run( - "ubuntu:20.04", + "ubuntu:24.04", detach=True, name=self.name, network=self.network.name, diff --git a/docker/test/integration/cluster/containers/SyslogUdpClientContainer.py b/docker/test/integration/cluster/containers/SyslogUdpClientContainer.py index 409c33fda..5019714f9 100644 --- a/docker/test/integration/cluster/containers/SyslogUdpClientContainer.py +++ b/docker/test/integration/cluster/containers/SyslogUdpClientContainer.py @@ -29,7 +29,7 @@ class SyslogUdpClientContainer(Container): logging.info('Creating and running a Syslog udp client docker container...') self.client.containers.run( - "ubuntu:20.04", + "ubuntu:24.04", detach=True, name=self.name, network=self.network.name, diff --git a/docker/test/integration/features/steps/steps.py b/docker/test/integration/features/steps/steps.py index 3f3a1ba57..e65ca2f29 100644 --- a/docker/test/integration/features/steps/steps.py +++ b/docker/test/integration/features/steps/steps.py @@ -39,7 +39,6 @@ import binascii import humanfriendly import OpenSSL.crypto -from kafka import KafkaProducer from confluent_kafka.admin import AdminClient, NewTopic from confluent_kafka import Producer import socket @@ -803,9 +802,9 @@ def step_impl(context, content, topic_name, semicolon_separated_headers): for header in semicolon_separated_headers.split(";"): kv = header.split(":") headers.append((kv[0].strip(), kv[1].strip().encode("utf-8"))) - producer = KafkaProducer(bootstrap_servers='localhost:29092') - future = producer.send(topic_name, content.encode("utf-8"), headers=headers) - assert future.get(timeout=60) + producer = Producer({"bootstrap.servers": "localhost:29092"}) + producer.produce(topic_name, content.encode("utf-8"), headers=headers) + producer.flush(10) @when("the Kafka consumer is registered in kafka broker") diff --git a/github_scripts/github_actions_cache_cleanup_tests.py b/github_scripts/github_actions_cache_cleanup_tests.py index 857bbd921..2f2f7772e 100755 --- a/github_scripts/github_actions_cache_cleanup_tests.py +++ b/github_scripts/github_actions_cache_cleanup_tests.py @@ -32,11 +32,11 @@ class TestGithubActionsCacheCleaner(unittest.TestCase): }, { "id": 11111, - "key": "ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", + "key": "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", }, { "id": 11112, - "key": "ubuntu-20.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487", + "key": "ubuntu-24.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487", }, { "id": 12345, @@ -52,11 +52,11 @@ class TestGithubActionsCacheCleaner(unittest.TestCase): }, { "id": 44444, - "key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456", + "key": "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456", }, { "id": 55555, - "key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567", + "key": "ubuntu-24.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567", } ] } @@ -77,11 +77,11 @@ class TestGithubActionsCacheCleaner(unittest.TestCase): }, { "id": 11111, - "key": "ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", + "key": "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", }, { "id": 11112, - "key": "ubuntu-20.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487", + "key": "ubuntu-24.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487", }, { "id": 12345, @@ -97,11 +97,11 @@ class TestGithubActionsCacheCleaner(unittest.TestCase): }, { "id": 44444, - "key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456", + "key": "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456", }, { "id": 55555, - "key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567", + "key": "ubuntu-24.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567", } ] } @@ -143,8 +143,8 @@ class TestGithubActionsCacheCleaner(unittest.TestCase): self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]), {"macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123", "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456", - "ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", - "ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"}) + "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", + "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"}) def test_cache_cleanup_with_zero_open_prs(self): cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp") @@ -152,11 +152,11 @@ class TestGithubActionsCacheCleaner(unittest.TestCase): cleaner.remove_obsolete_cache_entries() self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]), {"macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123", - "ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", - "ubuntu-20.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487", + "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", + "ubuntu-24.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487", "macos-xcode-ccache-refs/pull/227/merge-2d6d283f5bc894af8dfc295e5976a5f154753536", "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456", - "ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"}) + "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"}) def test_cache_cleanup_with_zero_action_caches(self): cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp")
