This is an automated email from the ASF dual-hosted git repository. martinzink pushed a commit to branch apache-rusty in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 9413e63e4f24ea9212f68a99b1595d49f191f8f3 Author: Martin Zink <[email protected]> AuthorDate: Thu Feb 12 16:53:10 2026 +0100 MINIFICPP-2718 Windows based docker tests Closes #2106 Signed-off-by: Martin Zink <[email protected]> --- .gitignore | 2 + behave_framework/pyproject.toml | 5 +- .../{container.py => container_linux.py} | 4 +- .../containers/container_protocol.py | 37 ++ .../containers/container_windows.py | 371 +++++++++++++++++++++ .../containers/http_proxy_container.py | 4 +- .../containers/minifi_container.py | 17 +- .../containers/minifi_controller.py | 79 +++++ .../containers/minifi_fhs_container.py | 119 +++++++ .../containers/minifi_linux_container.py | 119 +++++++ .../containers/minifi_protocol.py | 9 + .../containers/minifi_win_container.py | 59 ++++ .../containers/nifi_container.py | 4 +- .../src/minifi_test_framework/core/hooks.py | 12 +- .../core/minifi_test_context.py | 33 +- .../src/minifi_test_framework/core/ssl_utils.py | 222 ++++++------ docker/installed/win.Dockerfile | 24 ++ .../features/steps/kinesis_server_container.py | 4 +- .../tests/features/steps/s3_server_container.py | 4 +- .../tests/features/steps/azure_server_container.py | 4 +- .../features/steps/couchbase_server_container.py | 17 +- .../tests/features/steps/elastic_base_container.py | 4 +- .../features/steps/elasticsearch_container.py | 13 +- .../tests/features/steps/opensearch_container.py | 9 +- .../features/steps/fake_gcs_server_container.py | 4 +- .../tests/features/steps/grafana_loki_container.py | 13 +- .../features/steps/reverse_proxy_container.py | 4 +- .../tests/features/steps/kafka_server_container.py | 13 +- .../features/steps/opc_ua_server_container.py | 4 +- .../tests/features/steps/splunk_container.py | 13 +- .../features/steps/postgress_server_container.py | 4 +- .../tests/features/core_functionality.feature | 8 +- .../tests/features/replace_text.feature | 2 +- .../tests/features/steps/diag_slave_container.py | 4 +- .../features/steps/minifi_controller_steps.py | 24 +- .../tests/features/steps/syslog_container.py | 4 +- .../tests/features/steps/tcp_client_container.py | 4 +- run_flake8.sh | 2 +- 38 files changed, 1038 insertions(+), 240 deletions(-) diff --git a/.gitignore b/.gitignore index d30e23ff6..faaf21d60 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,8 @@ cmake_install.cmake install_manifest.txt CTestTestfile.cmake cmake-build-debug +venv +behave_venv # Generated files *flowfile_checkpoint diff --git a/behave_framework/pyproject.toml b/behave_framework/pyproject.toml index 263750d9f..5bf1f29f2 100644 --- a/behave_framework/pyproject.toml +++ b/behave_framework/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "minifi-test-framework" -version = "0.1.0" +version = "0.2.0" requires-python = ">= 3.10" description = "A testing framework for MiNiFi extensions." dependencies = [ @@ -8,8 +8,7 @@ dependencies = [ "docker==7.1.0", "PyYAML==6.0.3", "humanfriendly==10.0", - "m2crypto==0.46.2", - "pyopenssl==25.0.0", + "cryptography==46.0.5", "pyjks==20.0.0" ] diff --git a/behave_framework/src/minifi_test_framework/containers/container.py b/behave_framework/src/minifi_test_framework/containers/container_linux.py similarity index 99% rename from behave_framework/src/minifi_test_framework/containers/container.py rename to behave_framework/src/minifi_test_framework/containers/container_linux.py index 75660d785..28df65790 100644 --- a/behave_framework/src/minifi_test_framework/containers/container.py +++ b/behave_framework/src/minifi_test_framework/containers/container_linux.py @@ -27,6 +27,7 @@ import tarfile import docker from docker.models.networks import Network +from minifi_test_framework.containers.container_protocol import ContainerProtocol from minifi_test_framework.containers.directory import Directory from minifi_test_framework.containers.file import File from minifi_test_framework.containers.host_file import HostFile @@ -35,8 +36,9 @@ if TYPE_CHECKING: from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class Container: +class LinuxContainer(ContainerProtocol): def __init__(self, image_name: str, container_name: str, network: Network, command: str | None = None, entrypoint: str | None = None): + super().__init__() self.image_name: str = image_name self.container_name: str = container_name self.network: Network = network diff --git a/behave_framework/src/minifi_test_framework/containers/container_protocol.py b/behave_framework/src/minifi_test_framework/containers/container_protocol.py new file mode 100644 index 000000000..8713eff9c --- /dev/null +++ b/behave_framework/src/minifi_test_framework/containers/container_protocol.py @@ -0,0 +1,37 @@ +from typing import Protocol + + +class ContainerProtocol(Protocol): + def deploy(self, context) -> bool: + ... + + def clean_up(self): + ... + + def exec_run(self, command): + ... + + def directory_contains_file_with_content(self, directory_path: str, expected_content: str) -> bool: + ... + + def directory_contains_file_with_regex(self, directory_path: str, regex_str: str) -> bool: + ... + + def path_with_content_exists(self, path: str, content: str) -> bool: + ... + + def get_logs(self) -> str: + ... + + @property + def exited(self) -> bool: + ... + + def get_number_of_files(self, directory_path: str) -> int: + ... + + def verify_file_contents(self, directory_path: str, expected_contents: list[str]) -> bool: + ... + + def log_app_output(self) -> bool: + ... diff --git a/behave_framework/src/minifi_test_framework/containers/container_windows.py b/behave_framework/src/minifi_test_framework/containers/container_windows.py new file mode 100644 index 000000000..2418d3ccf --- /dev/null +++ b/behave_framework/src/minifi_test_framework/containers/container_windows.py @@ -0,0 +1,371 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations +import logging +import os +import tempfile +import base64 +import tarfile +import io +from typing import Union, Optional, Tuple, List, Dict, TYPE_CHECKING + +import docker +from docker.models.networks import Network +from docker.models.containers import Container + +from minifi_test_framework.containers.container_protocol import ContainerProtocol +from minifi_test_framework.containers.directory import Directory +from minifi_test_framework.containers.file import File +from minifi_test_framework.containers.host_file import HostFile + +if TYPE_CHECKING: + from minifi_test_framework.core.minifi_test_context import MinifiTestContext + + +class WindowsContainer(ContainerProtocol): + def __init__(self, image_name: str, container_name: str, network: Network, command: str | None = None, entrypoint: str | None = None): + super().__init__() + self.image_name: str = image_name + self.container_name: str = container_name + self.network: Network = network + + self.client: docker.DockerClient = docker.from_env() + self.container: Optional[Container] = None + self.files: List[File] = [] + self.dirs: List[Directory] = [] + self.host_files: List[HostFile] = [] + self.volumes: Dict = {} + self.command: str | None = command + self.entrypoint: str | None = entrypoint + self._temp_dir: Optional[tempfile.TemporaryDirectory] = None + self.ports: Optional[Dict[str, int]] = None + self.environment: List[str] = [] + + def _normalize_path(self, path: str) -> str: + """ + Converts paths to Windows format (C:\\...) + Handles leading slashes and ensures backslashes. + """ + clean_path = path.strip().replace("/", "\\") + if clean_path.startswith("\\"): + clean_path = clean_path[1:] + + # If it doesn't already have a drive letter, assume C: + if ":" not in clean_path: + return f"C:\\{clean_path}" + return clean_path + + def deploy(self, context: MinifiTestContext | None) -> bool: + # Cleanup previous temp dir if it exists to prevent leaks + if self._temp_dir: + self._temp_dir.cleanup() + self._temp_dir = tempfile.TemporaryDirectory() + + # 1. Prepare Volumes (Directory objects) + for directory in self.dirs: + # Mirror directory structure locally in temp + # usage of os.sep ensures we use host-native separators for the temp setup + rel_path = directory.path.strip("/\\") + temp_subdir = os.path.join(self._temp_dir.name, rel_path) + os.makedirs(temp_subdir, exist_ok=True) + + for file_name, content in directory.files.items(): + file_path = os.path.join(temp_subdir, file_name) + # Write with UTF-8 to ensure content is preserved on host + with open(file_path, "w", encoding="utf-8") as temp_file: + logging.info(f"writing content into {temp_file.name}") + temp_file.write(content) + + # Mount using Windows path format + container_bind_path = self._normalize_path(directory.path) + self.volumes[temp_subdir] = { + "bind": container_bind_path, + "mode": directory.mode + } + + for host_file in self.host_files: + container_bind_path = self._normalize_path(host_file.container_path) + self.volumes[container_bind_path] = {"bind": host_file.host_path, "mode": host_file.mode} + + # 3. Cleanup existing container + try: + existing_container = self.client.containers.get(self.container_name) + logging.warning(f"Found existing container '{self.container_name}'. Removing it first.") + existing_container.remove(force=True) + except docker.errors.NotFound: + pass + + # 4. Create and Start + try: + print(f"Creating and starting container '{self.container_name}'...") + self.container = self.client.containers.create( + image=self.image_name, + name=self.container_name, + ports=self.ports, + environment=self.environment, + volumes=self.volumes, + network=self.network.name, + command=self.command, + entrypoint=self.entrypoint, + detach=True, + tty=False + ) + + self.container.start() + + for file in self.files: + self._copy_content_to_container(file.content, file.path) + + except Exception as e: + logging.error(f"Error starting container: {e}") + self.clean_up() + raise + return True + + def _copy_content_to_container(self, content: str, target_path: str): + if not self.container: + return + + win_path = self._normalize_path(target_path) + dir_name = os.path.dirname(win_path) + file_name = os.path.basename(win_path) + + self._run_powershell(f"New-Item -ItemType Directory -Force -Path '{dir_name}'") + + tar_stream = io.BytesIO() + with tarfile.open(fileobj=tar_stream, mode='w') as tar: + encoded_data = content.encode('utf-8') + tarinfo = tarfile.TarInfo(name=file_name) + tarinfo.size = len(encoded_data) + tar.addfile(tarinfo, io.BytesIO(encoded_data)) + + tar_stream.seek(0) + + self.container.put_archive(path=dir_name, data=tar_stream) + + def clean_up(self): + if self._temp_dir: + try: + self._temp_dir.cleanup() + self._temp_dir = None + except Exception as e: + logging.warning(f"Failed to cleanup temp dir: {e}") + + if self.container: + try: + self.container.remove(force=True) + except docker.errors.NotFound: + pass + except Exception as e: + logging.warning(f"Failed to remove container: {e}") + finally: + self.container = None + + def exec_run(self, command: Union[str, list]) -> Tuple[int | None, str]: + logging.debug(f"Running command: {command}") + if self.container: + # Passing a list (if provided) bypasses shell parsing + (code, output) = self.container.exec_run(command, detach=False) + # errors='replace' handles non-utf8 characters often found in Windows logs + decoded_output = output.decode("utf-8", errors='replace') + logging.debug(f"Result {code}, output: {decoded_output}") + return code, decoded_output + return None, "Container not running." + + def _run_powershell(self, ps_script: str) -> Tuple[int | None, str]: + if not self.container: + return None, "Container not running" + + encoded_command = base64.b64encode(ps_script.encode('utf_16_le')).decode('utf-8') + + cmd_parts = ["powershell", "-NonInteractive", "-NoProfile", "-EncodedCommand", encoded_command] + + return self.exec_run(cmd_parts) + + def not_empty_dir_exists(self, directory_path: str) -> bool: + if not self.container: + return False + + win_path = self._normalize_path(directory_path) + ps_script = ( + f"if (Test-Path -Path '{win_path}' -PathType Container) {{ " + f" if (Get-ChildItem -Path '{win_path}') {{ exit 0 }} else {{ exit 1 }} " + f"}} else {{ exit 2 }}" + ) + + exit_code, _ = self._run_powershell(ps_script) + return exit_code == 0 + + def directory_contains_file_with_content(self, directory_path: str, expected_content: str) -> bool: + if not self.container: + return False + + win_path = self._normalize_path(directory_path) + escaped_content = expected_content.replace("'", "''") + + ps_script = ( + f"if (-not (Test-Path '{win_path}')) {{ exit 2 }}; " + f"$matches = Get-ChildItem -Path '{win_path}' -File -Depth 0 | " + f"Select-String -Pattern '{escaped_content}' -SimpleMatch -List; " + f"if ($matches) {{ exit 0 }} else {{ exit 1 }}" + ) + + exit_code, _ = self._run_powershell(ps_script) + return exit_code == 0 + + def directory_contains_file_with_regex(self, directory_path: str, regex_str: str) -> bool: + if not self.container: + return False + + win_path = self._normalize_path(directory_path) + escaped_regex = regex_str.replace("'", "''") + + ps_script = ( + f"if (-not (Test-Path '{win_path}')) {{ exit 2 }}; " + f"$matches = Get-ChildItem -Path '{win_path}' -File -Depth 0 | " + f"Select-String -Pattern '{escaped_regex}' -List; " + f"if ($matches) {{ exit 0 }} else {{ exit 1 }}" + ) + + exit_code, _ = self._run_powershell(ps_script) + return exit_code == 0 + + def path_with_content_exists(self, path: str, content: str) -> bool: + if not self.container: + return False + + win_path = self._normalize_path(path) + # Handle CRLF and escape quotes + escaped_content = content.replace("'", "''").replace("\n", "\r\n") + + ps_script = ( + f"try {{ " + f" $found = Select-String -Path '{win_path}' -Pattern '^{escaped_content}$'; " + f" if ($found.Count -eq 1) {{ exit 0 }} else {{ exit 1 }} " + f"}} catch {{ exit 2 }}" + ) + + exit_code, output = self._run_powershell(ps_script) + if exit_code != 0: + logging.debug(f"path_with_content_exists failed for {win_path}. Output: {output}") + + return exit_code == 0 + + def directory_has_single_file_with_content(self, directory_path: str, expected_content: str) -> bool: + if not self.container: + return False + + win_path = self._normalize_path(directory_path) + escaped_content = expected_content.strip().replace("'", "''").replace("\n", "\r\n") + + ps_script = ( + f"$files = Get-ChildItem -Path '{win_path}' -File -Depth 0; " + f"if ($files.Count -ne 1) {{ exit 1 }}; " + f"$actual_content = Get-Content -Path $files[0].FullName -Raw; " + f"if ($actual_content.Trim() -eq '{escaped_content}') {{ exit 0 }} else {{ exit 2 }};" + ) + + exit_code, output = self._run_powershell(ps_script) + if exit_code != 0: + logging.debug(f"Check for single file failed in {win_path}. Output: {output}") + + return exit_code == 0 + + def get_logs(self) -> str: + logging.debug("Getting logs from container '%s'", self.container_name) + if not self.container: + return "" + logs_as_bytes = self.container.logs() + # Windows logs may contain mixed encodings; replace errors to avoid crash + return logs_as_bytes.decode('utf-8', errors='replace') + + def log_app_output(self) -> bool: + logs = self.get_logs() + logging.info("Logs of container '%s':", self.container_name) + for line in logs.splitlines(): + logging.info(line) + return False + + @property + def exited(self) -> bool: + if not self.container: + return False + try: + self.container.reload() + return self.container.status == 'exited' + except docker.errors.NotFound: + self.container = None + return False + except Exception: + return False + + def get_number_of_files(self, directory_path: str) -> int: + if not self.container: + return -1 + + win_path = self._normalize_path(directory_path) + ps_script = f"(Get-ChildItem -Path '{win_path}' -File -Depth 0).Count" + + exit_code, output = self._run_powershell(ps_script) + + if exit_code != 0: + logging.error(f"Error counting files in '{win_path}': {output}") + return -1 + + try: + return int(output.strip()) + except (ValueError, IndexError): + return -1 + + def verify_file_contents(self, directory_path: str, expected_contents: List[str]) -> bool: + if not self.container: + return False + + win_path = self._normalize_path(directory_path) + + # 1. Get list of files + ps_list = f"Get-ChildItem -Path '{win_path}' -File -Depth 0 | Select-Object -ExpandProperty FullName" + exit_code, output = self._run_powershell(ps_list) + + if exit_code != 0: + logging.error(f"Error listing files in '{win_path}': {output}") + return False + + actual_filepaths = [path.strip() for path in output.splitlines() if path.strip()] + + if len(actual_filepaths) != len(expected_contents): + return False + + actual_file_contents = [] + for path in actual_filepaths: + # Read, normalize CRLF to LF, and trim + ps_read = ( + f"$c = Get-Content -Path '{path}' -Raw; " + f"if ($c) {{ ($c -replace '\\r\\n', '\\n' -replace '\\r', '\\n').Trim() }}" + ) + + exit_code, content = self._run_powershell(ps_read) + if exit_code != 0: + return False + actual_file_contents.append(content) + + # Normalize expected contents + normalized_expected = [ + s.strip().replace("\r\n", "\n").replace("\r", "\n") for s in expected_contents + ] + + return sorted(actual_file_contents) == sorted(normalized_expected) diff --git a/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py b/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py index 49ba2e766..c4b4d8823 100644 --- a/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py +++ b/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py @@ -17,13 +17,13 @@ from textwrap import dedent -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class HttpProxy(Container): +class HttpProxy(LinuxContainer): def __init__(self, test_context: MinifiTestContext): dockerfile = dedent("""\ FROM {base_image} diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_container.py index 75415017f..32728321f 100644 --- a/behave_framework/src/minifi_test_framework/containers/minifi_container.py +++ b/behave_framework/src/minifi_test_framework/containers/minifi_container.py @@ -18,13 +18,12 @@ import logging import os from pathlib import Path -from OpenSSL import crypto from minifi_test_framework.core.minifi_test_context import MinifiTestContext from minifi_test_framework.containers.file import File from minifi_test_framework.containers.host_file import HostFile from minifi_test_framework.minifi.minifi_flow_definition import MinifiFlowDefinition -from minifi_test_framework.core.ssl_utils import make_cert_without_extended_usage, make_client_cert, make_server_cert +from minifi_test_framework.core.ssl_utils import make_cert_without_extended_usage, make_client_cert, make_server_cert, dump_cert, dump_key from minifi_test_framework.core.helpers import wait_for_condition, retry_check from .container import Container @@ -37,18 +36,18 @@ class MinifiContainer(Container): self.log_properties: dict[str, str] = {} minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=self.container_name, ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) - self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert))) - self.files.append(File("/tmp/resources/root_ca.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert))) - self.files.append(File("/tmp/resources/minifi_client.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=minifi_client_cert))) - self.files.append(File("/tmp/resources/minifi_client.key", crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=minifi_client_key))) + self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", dump_cert(test_context.root_ca_cert))) + self.files.append(File("/tmp/resources/root_ca.crt", dump_cert(test_context.root_ca_cert))) + self.files.append(File("/tmp/resources/minifi_client.crt", dump_cert(minifi_client_cert))) + self.files.append(File("/tmp/resources/minifi_client.key", dump_key(minifi_client_key))) clientuser_cert, clientuser_key = make_client_cert("clientuser", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) - self.files.append(File("/tmp/resources/clientuser.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=clientuser_cert))) - self.files.append(File("/tmp/resources/clientuser.key", crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=clientuser_key))) + self.files.append(File("/tmp/resources/clientuser.crt", dump_cert(clientuser_cert))) + self.files.append(File("/tmp/resources/clientuser.key", dump_key(clientuser_key))) minifi_server_cert, minifi_server_key = make_server_cert(common_name=f"server-{test_context.scenario_id}", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) self.files.append(File("/tmp/resources/minifi_server.crt", - crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=minifi_server_cert) + crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=minifi_server_key))) + dump_cert(minifi_server_cert) + dump_key(minifi_server_key))) self.is_fhs = 'MINIFI_INSTALLATION_TYPE=FHS' in str(self.client.images.get(test_context.minifi_container_image).history()) if self.is_fhs: diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_controller.py b/behave_framework/src/minifi_test_framework/containers/minifi_controller.py new file mode 100644 index 000000000..0dc5b8330 --- /dev/null +++ b/behave_framework/src/minifi_test_framework/containers/minifi_controller.py @@ -0,0 +1,79 @@ +import logging +from minifi_test_framework.core.helpers import retry_check + + +class MinifiController: + def __init__(self, minifi_container, config_path): + self.minifi_container = minifi_container + self.config_path = config_path + + def set_controller_socket_properties(self): + self.minifi_container.properties["controller.socket.enable"] = "true" + self.minifi_container.properties["controller.socket.host"] = "localhost" + self.minifi_container.properties["controller.socket.port"] = "9998" + self.minifi_container.properties["controller.socket.local.any.interface"] = "false" + + def update_flow_config_through_controller(self): + self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--updateflow", "/tmp/resources/minifi-controller/config.yml"]) + + def updated_config_is_persisted(self) -> bool: + exit_code, output = self.minifi_container.exec_run(["cat", self.config_path]) + if exit_code != 0: + logging.error("Failed to read MiNiFi config file to check if updated config is persisted") + return False + return "2f2a3b47-f5ba-49f6-82b5-bc1c86b96f38" in output + + def stop_component_through_controller(self, component: str): + self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--stop", component]) + + def start_component_through_controller(self, component: str): + self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--start", component]) + + @retry_check(10, 1) + def is_component_running(self, component: str) -> bool: + (code, output) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--list", "components"]) + return code == 0 and component + ", running: true" in output + + def get_connections(self): + (_, output) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--list", "connections"]) + connections = [] + for line in output.split('\n'): + if not line.startswith('[') and not line.startswith('Connection Names'): + connections.append(line) + return connections + + @retry_check(10, 1) + def connection_found_through_controller(self, connection: str) -> bool: + return connection in self.get_connections() + + def get_full_connection_count(self) -> int: + (_, output) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--getfull"]) + for line in output.split('\n'): + if "are full" in line: + return int(line.split(' ')[0]) + return -1 + + def get_connection_size(self, connection: str): + (_, output) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--getsize", connection]) + for line in output.split('\n'): + if "Size/Max of " + connection in line: + size_and_max = line.split(connection)[1].split('/') + return (int(size_and_max[0].strip()), int(size_and_max[1].strip())) + return (-1, -1) + + def get_manifest(self) -> str: + (_, output) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--manifest"]) + manifest = "" + for line in output.split('\n'): + if not line.startswith('['): + manifest += line + return manifest + + def create_debug_bundle(self) -> bool: + (code, _) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--debug", "/tmp"]) + if code != 0: + logging.error("Minifi controller debug command failed with code: %d", code) + return False + + (code, _) = self.minifi_container.exec_run(["test", "-f", "/tmp/debug.tar.gz"]) + return code == 0 diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_fhs_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_fhs_container.py new file mode 100644 index 000000000..49c5236f0 --- /dev/null +++ b/behave_framework/src/minifi_test_framework/containers/minifi_fhs_container.py @@ -0,0 +1,119 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import os +from pathlib import Path + +from .container_linux import LinuxContainer + +from minifi_test_framework.core.minifi_test_context import MinifiTestContext +from minifi_test_framework.containers.file import File +from minifi_test_framework.containers.host_file import HostFile +from minifi_test_framework.minifi.minifi_flow_definition import MinifiFlowDefinition +from minifi_test_framework.core.ssl_utils import make_cert_without_extended_usage, make_client_cert, make_server_cert, dump_cert, dump_key +from minifi_test_framework.core.helpers import wait_for_condition + +from .minifi_protocol import MinifiProtocol +from .minifi_controller import MinifiController + + +class MinifiFhsContainer(LinuxContainer, MinifiProtocol): + def __init__(self, container_name: str, test_context: MinifiTestContext): + super().__init__(test_context.minifi_container_image, f"{container_name}-{test_context.scenario_id}", test_context.network) + self.flow_definition = MinifiFlowDefinition() + self.properties: dict[str, str] = {} + self.log_properties: dict[str, str] = {} + self.controller = MinifiController(self, "/etc/nifi-minifi-cpp/config.yml") + + minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=self.container_name, ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) + self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", dump_cert(test_context.root_ca_cert))) + self.files.append(File("/tmp/resources/root_ca.crt", dump_cert(test_context.root_ca_cert))) + self.files.append(File("/tmp/resources/minifi_client.crt", dump_cert(minifi_client_cert))) + self.files.append(File("/tmp/resources/minifi_client.key", dump_key(minifi_client_key))) + + clientuser_cert, clientuser_key = make_client_cert("clientuser", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) + self.files.append(File("/tmp/resources/clientuser.crt", dump_cert(clientuser_cert))) + self.files.append(File("/tmp/resources/clientuser.key", dump_key(clientuser_key))) + + minifi_server_cert, minifi_server_key = make_server_cert(common_name=f"server-{test_context.scenario_id}", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) + self.files.append(File("/tmp/resources/minifi_server.crt", + dump_cert(minifi_server_cert) + dump_key(minifi_server_key))) + + self.minifi_controller_path = '/usr/bin/minifi-controller' + + self._fill_default_properties() + self._fill_default_log_properties() + + def deploy(self, context: MinifiTestContext | None) -> bool: + flow_config = self.flow_definition.to_yaml() + logging.info(f"Deploying MiNiFi container '{self.container_name}' with flow configuration:\n{flow_config}") + self.files.append(File("/etc/nifi-minifi-cpp/config.yml", flow_config)) + self.files.append(File("/etc/nifi-minifi-cpp/minifi.properties", self._get_properties_file_content())) + self.files.append(File("/etc/nifi-minifi-cpp/minifi-log.properties", self._get_log_properties_file_content())) + resource_dir = Path(__file__).resolve().parent / "resources" / "minifi-controller" + self.host_files.append(HostFile("/tmp/resources/minifi-controller/config.yml", os.path.join(resource_dir, "config.yml"))) + + if not super().deploy(context): + return False + + finished_str = "MiNiFi started" + return wait_for_condition( + condition=lambda: finished_str in self.get_logs(), + timeout_seconds=15, + bail_condition=lambda: self.exited, + context=context) + + def set_property(self, key: str, value: str): + self.properties[key] = value + + def set_log_property(self, key: str, value: str): + self.log_properties[key] = value + + def enable_openssl_fips_mode(self): + self.properties["nifi.openssl.fips.support.enable"] = "true" + + def _fill_default_properties(self): + self.properties["nifi.flow.configuration.file"] = "/etc/nifi-minifi-cpp/config.yml" + self.properties["nifi.extension.path"] = "/usr/lib64/nifi-minifi-cpp/extensions/*" + self.properties["nifi.administrative.yield.duration"] = "1 sec" + self.properties["nifi.bored.yield.duration"] = "100 millis" + self.properties["nifi.openssl.fips.support.enable"] = "false" + self.properties["nifi.provenance.repository.class.name"] = "NoOpRepository" + + def _fill_default_log_properties(self): + self.log_properties["spdlog.pattern"] = "[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v" + + self.log_properties["appender.stderr"] = "stderr" + self.log_properties["logger.root"] = "DEBUG, stderr" + self.log_properties["logger.org::apache::nifi::minifi"] = "DEBUG, stderr" + + def _get_properties_file_content(self): + lines = (f"{key}={value}" for key, value in self.properties.items()) + return "\n".join(lines) + + def _get_log_properties_file_content(self): + lines = (f"{key}={value}" for key, value in self.log_properties.items()) + return "\n".join(lines) + + def get_memory_usage(self) -> int | None: + exit_code, output = self.exec_run(["awk", "/VmRSS/ { printf \"%d\\n\", $2 }", "/proc/1/status"]) + if exit_code != 0: + return None + memory_usage_in_bytes = int(output.strip()) * 1024 + logging.info(f"MiNiFi memory usage: {memory_usage_in_bytes} bytes") + return memory_usage_in_bytes diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_linux_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_linux_container.py new file mode 100644 index 000000000..f0279ebe2 --- /dev/null +++ b/behave_framework/src/minifi_test_framework/containers/minifi_linux_container.py @@ -0,0 +1,119 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import os +from pathlib import Path + +from .container_linux import LinuxContainer + +from minifi_test_framework.core.minifi_test_context import MinifiTestContext +from minifi_test_framework.containers.file import File +from minifi_test_framework.containers.host_file import HostFile +from minifi_test_framework.minifi.minifi_flow_definition import MinifiFlowDefinition +from minifi_test_framework.core.ssl_utils import make_cert_without_extended_usage, make_client_cert, make_server_cert, dump_cert, dump_key +from minifi_test_framework.core.helpers import wait_for_condition + +from .minifi_protocol import MinifiProtocol +from .minifi_controller import MinifiController + + +class MinifiLinuxContainer(LinuxContainer, MinifiProtocol): + def __init__(self, container_name: str, test_context: MinifiTestContext): + super().__init__(test_context.minifi_container_image, f"{container_name}-{test_context.scenario_id}", test_context.network) + self.flow_definition = MinifiFlowDefinition() + self.properties: dict[str, str] = {} + self.log_properties: dict[str, str] = {} + self.controller = MinifiController(self, "/opt/minifi/minifi-current/conf/config.yml") + + minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=self.container_name, ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) + self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", dump_cert(test_context.root_ca_cert))) + self.files.append(File("/tmp/resources/root_ca.crt", dump_cert(test_context.root_ca_cert))) + self.files.append(File("/tmp/resources/minifi_client.crt", dump_cert(minifi_client_cert))) + self.files.append(File("/tmp/resources/minifi_client.key", dump_key(minifi_client_key))) + + clientuser_cert, clientuser_key = make_client_cert("clientuser", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) + self.files.append(File("/tmp/resources/clientuser.crt", dump_cert(clientuser_cert))) + self.files.append(File("/tmp/resources/clientuser.key", dump_key(clientuser_key))) + + minifi_server_cert, minifi_server_key = make_server_cert(common_name=f"server-{test_context.scenario_id}", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) + self.files.append(File("/tmp/resources/minifi_server.crt", + dump_cert(cert=minifi_server_cert) + dump_key(minifi_server_key))) + + self.minifi_controller_path = '/opt/minifi/minifi-current/bin/minifi-controller' + + self._fill_default_properties() + self._fill_default_log_properties() + + def deploy(self, context: MinifiTestContext | None) -> bool: + flow_config = self.flow_definition.to_yaml() + logging.info(f"Deploying MiNiFi container '{self.container_name}' with flow configuration:\n{flow_config}") + self.files.append(File("/opt/minifi/minifi-current/conf/config.yml", flow_config)) + self.files.append(File("/opt/minifi/minifi-current/conf/minifi.properties", self._get_properties_file_content())) + self.files.append(File("/opt/minifi/minifi-current/conf/minifi-log.properties", self._get_log_properties_file_content())) + resource_dir = Path(__file__).resolve().parent / "resources" / "minifi-controller" + self.host_files.append(HostFile("/tmp/resources/minifi-controller/config.yml", os.path.join(resource_dir, "config.yml"))) + + if not super().deploy(context): + return False + + finished_str = "MiNiFi started" + return wait_for_condition( + condition=lambda: finished_str in self.get_logs(), + timeout_seconds=15, + bail_condition=lambda: self.exited, + context=context) + + def set_property(self, key: str, value: str): + self.properties[key] = value + + def set_log_property(self, key: str, value: str): + self.log_properties[key] = value + + def enable_openssl_fips_mode(self): + self.properties["nifi.openssl.fips.support.enable"] = "true" + + def _fill_default_properties(self): + self.properties["nifi.flow.configuration.file"] = "./conf/config.yml" + self.properties["nifi.extension.path"] = "../extensions/*" + self.properties["nifi.administrative.yield.duration"] = "1 sec" + self.properties["nifi.bored.yield.duration"] = "100 millis" + self.properties["nifi.openssl.fips.support.enable"] = "false" + self.properties["nifi.provenance.repository.class.name"] = "NoOpRepository" + + def _fill_default_log_properties(self): + self.log_properties["spdlog.pattern"] = "[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v" + + self.log_properties["appender.stderr"] = "stderr" + self.log_properties["logger.root"] = "DEBUG, stderr" + self.log_properties["logger.org::apache::nifi::minifi"] = "DEBUG, stderr" + + def _get_properties_file_content(self): + lines = (f"{key}={value}" for key, value in self.properties.items()) + return "\n".join(lines) + + def _get_log_properties_file_content(self): + lines = (f"{key}={value}" for key, value in self.log_properties.items()) + return "\n".join(lines) + + def get_memory_usage(self) -> int | None: + exit_code, output = self.exec_run(["awk", "/VmRSS/ { printf \"%d\\n\", $2 }", "/proc/1/status"]) + if exit_code != 0: + return None + memory_usage_in_bytes = int(output.strip()) * 1024 + logging.info(f"MiNiFi memory usage: {memory_usage_in_bytes} bytes") + return memory_usage_in_bytes diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_protocol.py b/behave_framework/src/minifi_test_framework/containers/minifi_protocol.py new file mode 100644 index 000000000..1c6a110d8 --- /dev/null +++ b/behave_framework/src/minifi_test_framework/containers/minifi_protocol.py @@ -0,0 +1,9 @@ +from typing import Protocol + + +class MinifiProtocol(Protocol): + def set_property(self, key: str, value: str): + ... + + def set_log_property(self, key: str, value: str): + ... diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_win_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_win_container.py new file mode 100644 index 000000000..2e8952cf3 --- /dev/null +++ b/behave_framework/src/minifi_test_framework/containers/minifi_win_container.py @@ -0,0 +1,59 @@ +from typing import Dict + +from minifi_test_framework.core.minifi_test_context import MinifiTestContext +from minifi_test_framework.minifi.minifi_flow_definition import MinifiFlowDefinition +from minifi_test_framework.containers.directory import Directory +from .container_windows import WindowsContainer +from .minifi_protocol import MinifiProtocol +import logging + + +class MinifiWindowsContainer(WindowsContainer, MinifiProtocol): + def __init__(self, container_name: str, test_context: MinifiTestContext): + super().__init__(test_context.minifi_container_image, f"{container_name}-{test_context.scenario_id}", test_context.network) + self.flow_config_str: str = "" + self.flow_definition = MinifiFlowDefinition() + self.properties: Dict[str, str] = {} + self.log_properties: Dict[str, str] = {} + + self._fill_default_properties() + self._fill_default_log_properties() + + def deploy(self, context: MinifiTestContext | None) -> bool: + logging.info(self.flow_definition.to_yaml()) + conf_dir = Directory("\\Program Files\\ApacheNiFiMiNiFi\\nifi-minifi-cpp\\conf") + conf_dir.add_file("config.yml", self.flow_definition.to_yaml()) + conf_dir.add_file("minifi.properties", self._get_properties_file_content()) + conf_dir.add_file("minifi-log.properties", self._get_log_properties_file_content()) + + self.dirs.append(conf_dir) + return super().deploy(context) + + def set_property(self, key: str, value: str): + self.properties[key] = value + + def set_log_property(self, key: str, value: str): + self.log_properties[key] = value + + def _fill_default_properties(self): + self.properties["nifi.flow.configuration.file"] = "./conf/config.yml" + self.properties["nifi.extension.path"] = "../extensions/*" + self.properties["nifi.administrative.yield.duration"] = "1 sec" + self.properties["nifi.bored.yield.duration"] = "100 millis" + self.properties["nifi.openssl.fips.support.enable"] = "false" + self.properties["nifi.provenance.repository.class.name"] = "NoOpRepository" + + def _fill_default_log_properties(self): + self.log_properties["spdlog.pattern"] = "[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v" + + self.log_properties["appender.stderr"] = "stderr" + self.log_properties["logger.root"] = "DEBUG, stderr" + self.log_properties["logger.org::apache::nifi::minifi"] = "DEBUG, stderr" + + def _get_properties_file_content(self): + lines = (f"{key}={value}" for key, value in self.properties.items()) + return "\n".join(lines) + + def _get_log_properties_file_content(self): + lines = (f"{key}={value}" for key, value in self.log_properties.items()) + return "\n".join(lines) diff --git a/behave_framework/src/minifi_test_framework/containers/nifi_container.py b/behave_framework/src/minifi_test_framework/containers/nifi_container.py index d40724de1..140b5d508 100644 --- a/behave_framework/src/minifi_test_framework/containers/nifi_container.py +++ b/behave_framework/src/minifi_test_framework/containers/nifi_container.py @@ -17,13 +17,13 @@ import io import gzip from typing import List, Optional from minifi_test_framework.containers.file import File -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext from minifi_test_framework.minifi.nifi_flow_definition import NifiFlowDefinition -class NifiContainer(Container): +class NifiContainer(LinuxContainer): NIFI_VERSION = '2.7.2' def __init__(self, test_context: MinifiTestContext, command: Optional[List[str]] = None, use_ssl: bool = False): diff --git a/behave_framework/src/minifi_test_framework/core/hooks.py b/behave_framework/src/minifi_test_framework/core/hooks.py index 298f27d85..576ac40f8 100644 --- a/behave_framework/src/minifi_test_framework/core/hooks.py +++ b/behave_framework/src/minifi_test_framework/core/hooks.py @@ -46,6 +46,10 @@ def inject_scenario_id(context: MinifiTestContext, step): def common_before_scenario(context: Context, scenario: Scenario): + if "SUPPORTS_WINDOWS" not in scenario.effective_tags and os.name == 'nt': + scenario.skip("No windows support") + return + if not hasattr(context, "minifi_container_image"): context.minifi_container_image = get_minifi_container_image() @@ -80,6 +84,8 @@ def common_before_scenario(context: Context, scenario: Scenario): def common_after_scenario(context: MinifiTestContext, scenario: Scenario): - for container in context.containers.values(): - container.clean_up() - context.network.remove() + if hasattr(context, 'containers'): + for container in context.containers.values(): + container.clean_up() + if hasattr(context, 'network'): + context.network.remove() diff --git a/behave_framework/src/minifi_test_framework/core/minifi_test_context.py b/behave_framework/src/minifi_test_framework/core/minifi_test_context.py index 959f2e1fa..5dd4db9cc 100644 --- a/behave_framework/src/minifi_test_framework/core/minifi_test_context.py +++ b/behave_framework/src/minifi_test_framework/core/minifi_test_context.py @@ -16,32 +16,45 @@ # from __future__ import annotations -from typing import TYPE_CHECKING + +import os +from typing import Any +import docker from behave.runner import Context from docker.models.networks import Network -from minifi_test_framework.containers.container import Container -from OpenSSL import crypto +from minifi_test_framework.containers.container_protocol import ContainerProtocol +from minifi_test_framework.containers.minifi_protocol import MinifiProtocol -if TYPE_CHECKING: - from minifi_test_framework.containers.minifi_container import MinifiContainer DEFAULT_MINIFI_CONTAINER_NAME = "minifi-primary" +class MinifiContainer(ContainerProtocol, MinifiProtocol): + pass + + class MinifiTestContext(Context): - containers: dict[str, Container] + containers: dict[str, ContainerProtocol] scenario_id: str network: Network minifi_container_image: str resource_dir: str | None - root_ca_key: crypto.PKey - root_ca_cert: crypto.X509 + root_ca_key: Any + root_ca_cert: Any def get_or_create_minifi_container(self, container_name: str) -> MinifiContainer: if container_name not in self.containers: - from minifi_test_framework.containers.minifi_container import MinifiContainer - self.containers[container_name] = MinifiContainer(container_name, self) + if os.name == 'nt': + from minifi_test_framework.containers.minifi_win_container import MinifiWindowsContainer + minifi_container = MinifiWindowsContainer(container_name, self) + elif 'MINIFI_INSTALLATION_TYPE=FHS' in str(docker.from_env().images.get(self.minifi_container_image).history()): + from minifi_test_framework.containers.minifi_fhs_container import MinifiFhsContainer + minifi_container = MinifiFhsContainer(container_name, self) + else: + from minifi_test_framework.containers.minifi_linux_container import MinifiLinuxContainer + minifi_container = MinifiLinuxContainer(container_name, self) + self.containers[container_name] = minifi_container return self.containers[container_name] def get_or_create_default_minifi_container(self) -> MinifiContainer: diff --git a/behave_framework/src/minifi_test_framework/core/ssl_utils.py b/behave_framework/src/minifi_test_framework/core/ssl_utils.py index d39cf9646..b11da3443 100644 --- a/behave_framework/src/minifi_test_framework/core/ssl_utils.py +++ b/behave_framework/src/minifi_test_framework/core/ssl_utils.py @@ -1,161 +1,125 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import time -import logging -import random - -from M2Crypto import X509, EVP, RSA, ASN1 -from OpenSSL import crypto +import datetime +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization def gen_cert(): - """ - Generate TLS certificate request for testing - """ - - req, key = gen_req() - pub_key = req.get_pubkey() - subject = req.get_subject() - cert = X509.X509() - # noinspection PyTypeChecker - cert.set_serial_number(1) - cert.set_version(2) - cert.set_subject(subject) - t = int(time.time()) - now = ASN1.ASN1_UTCTIME() - now.set_time(t) - now_plus_year = ASN1.ASN1_UTCTIME() - now_plus_year.set_time(t + 60 * 60 * 24 * 365) - cert.set_not_before(now) - cert.set_not_after(now_plus_year) - issuer = X509.X509_Name() - issuer.C = 'US' - issuer.CN = 'minifi-listen' - cert.set_issuer(issuer) - cert.set_pubkey(pub_key) - cert.sign(key, 'sha256') + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) - return cert, key - - -def rsa_gen_key_callback(): - pass - - -def gen_req(): - """ - Generate TLS certificate request for testing - """ + subject = issuer = x509.Name([ + x509.NameAttribute(NameOID.COUNTRY_NAME, u"US"), + x509.NameAttribute(NameOID.COMMON_NAME, u"minifi-listen"), + ]) - logging.info('Generating test certificate request') - key = EVP.PKey() - req = X509.Request() - rsa = RSA.gen_key(1024, 65537, rsa_gen_key_callback) - key.assign_rsa(rsa) - req.set_pubkey(key) - name = req.get_subject() - name.C = 'US' - name.CN = 'minifi-listen' - req.sign(key, 'sha256') + cert = x509.CertificateBuilder().subject_name( + subject + ).issuer_name( + issuer + ).public_key( + key.public_key() + ).serial_number( + x509.random_serial_number() + ).not_valid_before( + datetime.datetime.utcnow() + ).not_valid_after( + datetime.datetime.utcnow() + datetime.timedelta(days=365) + ).sign(key, hashes.SHA256()) - return req, key + return cert, key def make_self_signed_cert(common_name): - ca_key = crypto.PKey() - ca_key.generate_key(crypto.TYPE_RSA, 2048) + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) - ca_cert = crypto.X509() - ca_cert.set_version(2) - ca_cert.set_serial_number(random.randint(50000000, 100000000)) - - ca_subj = ca_cert.get_subject() - ca_subj.commonName = common_name - - ca_cert.add_extensions([ - crypto.X509Extension(b"subjectKeyIdentifier", False, b"hash", subject=ca_cert), - ]) - - ca_cert.add_extensions([ - crypto.X509Extension(b"authorityKeyIdentifier", False, b"keyid:always", issuer=ca_cert), + subject = issuer = x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, common_name), ]) - ca_cert.add_extensions([ - crypto.X509Extension(b"basicConstraints", False, b"CA:TRUE"), - crypto.X509Extension(b"keyUsage", False, b"keyCertSign, cRLSign"), - ]) - - ca_cert.set_issuer(ca_subj) - ca_cert.set_pubkey(ca_key) - - ca_cert.gmtime_adj_notBefore(0) - ca_cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) + cert = x509.CertificateBuilder().subject_name( + subject + ).issuer_name( + issuer + ).public_key( + key.public_key() + ).serial_number( + x509.random_serial_number() + ).not_valid_before( + datetime.datetime.utcnow() + ).not_valid_after( + datetime.datetime.utcnow() + datetime.timedelta(days=3650) + ).add_extension( + x509.SubjectKeyIdentifier.from_public_key(key.public_key()), + critical=False, + ).add_extension( + x509.BasicConstraints(ca=True, path_length=None), + critical=True, + ).sign(key, hashes.SHA256()) - ca_cert.sign(ca_key, 'sha256') - - return ca_cert, ca_key + return cert, key def _make_cert(common_name, ca_cert, ca_key, extended_key_usage=None): - key = crypto.PKey() - key.generate_key(crypto.TYPE_RSA, 2048) - - cert = crypto.X509() - cert.set_version(2) - cert.set_serial_number(random.randint(50000000, 100000000)) - - client_subj = cert.get_subject() - client_subj.commonName = common_name + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) - cert.add_extensions([ - crypto.X509Extension(b"basicConstraints", False, b"CA:FALSE"), - crypto.X509Extension(b"subjectKeyIdentifier", False, b"hash", subject=cert), + subject = x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, common_name), ]) - extensions = [crypto.X509Extension(b"authorityKeyIdentifier", False, b"keyid:always", issuer=ca_cert), - crypto.X509Extension(b"keyUsage", False, b"digitalSignature")] + builder = x509.CertificateBuilder().subject_name( + subject + ).issuer_name( + ca_cert.subject + ).public_key( + key.public_key() + ).serial_number( + x509.random_serial_number() + ).not_valid_before( + datetime.datetime.utcnow() + ).not_valid_after( + datetime.datetime.utcnow() + datetime.timedelta(days=3650) + ).add_extension( + x509.BasicConstraints(ca=False, path_length=None), + critical=True, + ).add_extension( + x509.SubjectKeyIdentifier.from_public_key(key.public_key()), + critical=False, + ).add_extension( + x509.SubjectAlternativeName([x509.DNSName(common_name)]), + critical=False, + ) if extended_key_usage: - extensions.append(crypto.X509Extension(b"extendedKeyUsage", False, extended_key_usage)) - - cert.add_extensions([ - crypto.X509Extension(b"subjectAltName", False, b"DNS.1:" + common_name.encode()) - ]) - - cert.add_extensions(extensions) - - cert.set_issuer(ca_cert.get_subject()) - cert.set_pubkey(key) - - cert.gmtime_adj_notBefore(0) - cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) - - cert.sign(ca_key, 'sha256') + builder = builder.add_extension( + x509.ExtendedKeyUsage(extended_key_usage), + critical=False + ) + cert = builder.sign(ca_key, hashes.SHA256()) return cert, key def make_client_cert(common_name, ca_cert, ca_key): - return _make_cert(common_name=common_name, ca_cert=ca_cert, ca_key=ca_key, extended_key_usage=b"clientAuth") + return _make_cert(common_name, ca_cert, ca_key, [x509.OID_CLIENT_AUTH]) def make_server_cert(common_name, ca_cert, ca_key): - return _make_cert(common_name=common_name, ca_cert=ca_cert, ca_key=ca_key, extended_key_usage=b"serverAuth") + return _make_cert(common_name, ca_cert, ca_key, [x509.OID_SERVER_AUTH]) def make_cert_without_extended_usage(common_name, ca_cert, ca_key): - return _make_cert(common_name=common_name, ca_cert=ca_cert, ca_key=ca_key, extended_key_usage=None) + return _make_cert(common_name, ca_cert, ca_key, None) + + +def dump_cert(cert, encoding_type=serialization.Encoding.PEM): + return cert.public_bytes(encoding_type) + + +def dump_key(key, encoding_type=serialization.Encoding.PEM): + return key.private_bytes( + encoding=encoding_type, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + ) diff --git a/docker/installed/win.Dockerfile b/docker/installed/win.Dockerfile new file mode 100644 index 000000000..b413b68b9 --- /dev/null +++ b/docker/installed/win.Dockerfile @@ -0,0 +1,24 @@ +#escape=` + +FROM mcr.microsoft.com/windows/servercore:ltsc2022 + +LABEL maintainer="Apache NiFi <[email protected]>" + +ARG MSI_SOURCE="nifi-minifi-cpp.msi" + +ENV MINIFI_HOME="C:\Program Files\ApacheNiFiMiNiFi\nifi-minifi-cpp" + +SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] + +RUN Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) + +RUN choco install git -y +RUN choco install vcredist140 -y + +COPY ${MSI_SOURCE} C:\temp\minifi.msi + +SHELL ["cmd", "/S", "/C"] + +RUN C:\Windows\System32\msiexec.exe /i C:\temp\minifi.msi /qn /norestart /L*V C:\minifi_install.log + +CMD ["C:\\Program Files\\ApacheNiFiMiNiFi\\nifi-minifi-cpp\\bin\\minifi.exe"] \ No newline at end of file diff --git a/extensions/aws/tests/features/steps/kinesis_server_container.py b/extensions/aws/tests/features/steps/kinesis_server_container.py index 01b6dced2..d97bb5c83 100644 --- a/extensions/aws/tests/features/steps/kinesis_server_container.py +++ b/extensions/aws/tests/features/steps/kinesis_server_container.py @@ -16,13 +16,13 @@ import logging from pathlib import Path -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.helpers import wait_for_condition, retry_check from minifi_test_framework.core.minifi_test_context import MinifiTestContext from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder -class KinesisServerContainer(Container): +class KinesisServerContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): builder = DockerImageBuilder( image_tag="minifi-kinesis-mock:latest", diff --git a/extensions/aws/tests/features/steps/s3_server_container.py b/extensions/aws/tests/features/steps/s3_server_container.py index 699617c71..a2c915743 100644 --- a/extensions/aws/tests/features/steps/s3_server_container.py +++ b/extensions/aws/tests/features/steps/s3_server_container.py @@ -16,12 +16,12 @@ import json import logging -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class S3ServerContainer(Container): +class S3ServerContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): super().__init__("adobe/s3mock:3.12.0", f"s3-server-{test_context.scenario_id}", test_context.network) self.environment.append("initialBuckets=test_bucket") diff --git a/extensions/azure/tests/features/steps/azure_server_container.py b/extensions/azure/tests/features/steps/azure_server_container.py index 4caeb786b..cb0d2e5eb 100644 --- a/extensions/azure/tests/features/steps/azure_server_container.py +++ b/extensions/azure/tests/features/steps/azure_server_container.py @@ -18,13 +18,13 @@ import logging from docker.errors import ContainerError -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.helpers import run_cmd_in_docker_image from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class AzureServerContainer(Container): +class AzureServerContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): super().__init__("mcr.microsoft.com/azure-storage/azurite:3.35.0", f"azure-storage-server-{test_context.scenario_id}", diff --git a/extensions/couchbase/tests/features/steps/couchbase_server_container.py b/extensions/couchbase/tests/features/steps/couchbase_server_container.py index 52228951d..c0471be50 100644 --- a/extensions/couchbase/tests/features/steps/couchbase_server_container.py +++ b/extensions/couchbase/tests/features/steps/couchbase_server_container.py @@ -15,26 +15,25 @@ import logging -from OpenSSL import crypto from minifi_test_framework.core.helpers import wait_for_condition, retry_check -from minifi_test_framework.core.ssl_utils import make_server_cert -from minifi_test_framework.containers.container import Container +from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.containers.file import File from minifi_test_framework.core.minifi_test_context import MinifiTestContext from docker.errors import ContainerError -class CouchbaseServerContainer(Container): +class CouchbaseServerContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): super().__init__("couchbase:enterprise-7.2.5", f"couchbase-server-{test_context.scenario_id}", test_context.network) couchbase_cert, couchbase_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) - root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert) + root_ca_content = dump_cert(test_context.root_ca_cert) self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/CA/root_ca.crt", root_ca_content, permissions=0o666)) - couchbase_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=couchbase_cert) + couchbase_cert_content = dump_cert(couchbase_cert) self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/chain.pem", couchbase_cert_content, permissions=0o666)) - couchbase_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=couchbase_key) + couchbase_key_content = dump_key(couchbase_key) self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/pkey.key", couchbase_key_content, permissions=0o666)) def deploy(self, context: MinifiTestContext | None) -> bool: @@ -66,7 +65,7 @@ class CouchbaseServerContainer(Container): return True - @retry_check(max_tries=12, retry_interval=5) + @retry_check(12, 5) def _run_couchbase_cli_command(self, command): (code, output) = self.exec_run(command) if code != 0: @@ -90,7 +89,7 @@ class CouchbaseServerContainer(Container): logging.error(f"Unexpected error while running python command '{command}' in couchbase helper docker: '{e}'") return False - @retry_check(max_tries=15, retry_interval=2) + @retry_check(15, 2) def _load_couchbase_certs(self): python_command = f""" import requests diff --git a/extensions/elasticsearch/tests/features/steps/elastic_base_container.py b/extensions/elasticsearch/tests/features/steps/elastic_base_container.py index 5db6e61dc..2287278c4 100644 --- a/extensions/elasticsearch/tests/features/steps/elastic_base_container.py +++ b/extensions/elasticsearch/tests/features/steps/elastic_base_container.py @@ -14,11 +14,11 @@ # limitations under the License. from minifi_test_framework.core.helpers import wait_for_condition, retry_check -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class ElasticBaseContainer(Container): +class ElasticBaseContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext, image: str, container_name: str): super().__init__(image, container_name, test_context.network) self.user = None diff --git a/extensions/elasticsearch/tests/features/steps/elasticsearch_container.py b/extensions/elasticsearch/tests/features/steps/elasticsearch_container.py index 037711d75..aec928026 100644 --- a/extensions/elasticsearch/tests/features/steps/elasticsearch_container.py +++ b/extensions/elasticsearch/tests/features/steps/elasticsearch_container.py @@ -19,8 +19,7 @@ import os from elastic_base_container import ElasticBaseContainer from pathlib import Path -from OpenSSL import crypto -from minifi_test_framework.core.ssl_utils import make_server_cert, make_cert_without_extended_usage +from minifi_test_framework.core.ssl_utils import make_server_cert, make_cert_without_extended_usage, dump_cert, dump_key from minifi_test_framework.containers.file import File from minifi_test_framework.containers.host_file import HostFile from minifi_test_framework.core.minifi_test_context import MinifiTestContext @@ -33,19 +32,19 @@ class ElasticsearchContainer(ElasticBaseContainer): http_cert, http_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) transport_cert, transport_key = make_cert_without_extended_usage(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) - root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert) + root_ca_content = dump_cert(test_context.root_ca_cert) self.files.append(File("/usr/share/elasticsearch/config/certs/root_ca.crt", root_ca_content, permissions=0o644)) - http_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=http_cert) + http_cert_content = dump_cert(http_cert) self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_http.crt", http_cert_content, permissions=0o644)) - http_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=http_key) + http_key_content = dump_key(http_key) self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_http.key", http_key_content, permissions=0o644)) - transport_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=transport_cert) + transport_cert_content = dump_cert(transport_cert) self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_transport.crt", transport_cert_content, permissions=0o644)) - transport_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=transport_key) + transport_key_content = dump_key(transport_key) self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_transport.key", transport_key_content, permissions=0o644)) features_dir = Path(__file__).resolve().parent.parent diff --git a/extensions/elasticsearch/tests/features/steps/opensearch_container.py b/extensions/elasticsearch/tests/features/steps/opensearch_container.py index cdf5008d5..234a26d12 100644 --- a/extensions/elasticsearch/tests/features/steps/opensearch_container.py +++ b/extensions/elasticsearch/tests/features/steps/opensearch_container.py @@ -18,8 +18,7 @@ import logging from elastic_base_container import ElasticBaseContainer from pathlib import Path -from OpenSSL import crypto -from minifi_test_framework.core.ssl_utils import make_server_cert +from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key from minifi_test_framework.containers.file import File from minifi_test_framework.containers.host_file import HostFile from minifi_test_framework.core.minifi_test_context import MinifiTestContext @@ -31,13 +30,13 @@ class OpensearchContainer(ElasticBaseContainer): admin_pem, admin_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) - root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert) + root_ca_content = dump_cert(test_context.root_ca_cert) self.files.append(File("/usr/share/opensearch/config/root-ca.pem", root_ca_content, permissions=0o644)) - admin_pem_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=admin_pem) + admin_pem_content = dump_cert(admin_pem) self.files.append(File("/usr/share/opensearch/config/admin.pem", admin_pem_content, permissions=0o644)) - admin_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=admin_key) + admin_key_content = dump_key(admin_key) self.files.append(File("/usr/share/opensearch/config/admin-key.pem", admin_key_content, permissions=0o644)) features_dir = Path(__file__).resolve().parent.parent diff --git a/extensions/gcp/tests/features/steps/fake_gcs_server_container.py b/extensions/gcp/tests/features/steps/fake_gcs_server_container.py index a9733ba4e..5689065bb 100644 --- a/extensions/gcp/tests/features/steps/fake_gcs_server_container.py +++ b/extensions/gcp/tests/features/steps/fake_gcs_server_container.py @@ -15,12 +15,12 @@ import logging from minifi_test_framework.core.helpers import wait_for_condition, retry_check -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.containers.directory import Directory from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class FakeGcsServerContainer(Container): +class FakeGcsServerContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): super().__init__("fsouza/fake-gcs-server:1.45.1", f"fake-gcs-server-{test_context.scenario_id}", test_context.network, command=f'-scheme http -host fake-gcs-server-{test_context.scenario_id}') diff --git a/extensions/grafana-loki/tests/features/steps/grafana_loki_container.py b/extensions/grafana-loki/tests/features/steps/grafana_loki_container.py index 1284b95b0..10c77defd 100644 --- a/extensions/grafana-loki/tests/features/steps/grafana_loki_container.py +++ b/extensions/grafana-loki/tests/features/steps/grafana_loki_container.py @@ -14,13 +14,12 @@ # limitations under the License. import logging -from OpenSSL import crypto from minifi_test_framework.core.helpers import wait_for_condition, retry_check -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.containers.file import File from minifi_test_framework.core.minifi_test_context import MinifiTestContext -from minifi_test_framework.core.ssl_utils import make_server_cert +from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key from docker.errors import ContainerError @@ -30,20 +29,20 @@ class GrafanaLokiOptions: self.enable_multi_tenancy = enable_multi_tenancy -class GrafanaLokiContainer(Container): +class GrafanaLokiContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext, options: GrafanaLokiOptions): super().__init__("grafana/loki:3.2.1", f"grafana-loki-server-{test_context.scenario_id}", test_context.network) extra_ssl_settings = "" if options.enable_ssl: grafana_loki_cert, grafana_loki_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) - root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert) + root_ca_content = dump_cert(test_context.root_ca_cert) self.files.append(File("/etc/loki/root_ca.crt", root_ca_content, permissions=0o644)) - grafana_loki_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=grafana_loki_cert) + grafana_loki_cert_content = dump_cert(grafana_loki_cert) self.files.append(File("/etc/loki/cert.pem", grafana_loki_cert_content, permissions=0o644)) - grafana_loki_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=grafana_loki_key) + grafana_loki_key_content = dump_key(grafana_loki_key) self.files.append(File("/etc/loki/key.pem", grafana_loki_key_content, permissions=0o644)) extra_ssl_settings = """ diff --git a/extensions/grafana-loki/tests/features/steps/reverse_proxy_container.py b/extensions/grafana-loki/tests/features/steps/reverse_proxy_container.py index 409a9b9e6..43efd49c5 100644 --- a/extensions/grafana-loki/tests/features/steps/reverse_proxy_container.py +++ b/extensions/grafana-loki/tests/features/steps/reverse_proxy_container.py @@ -14,11 +14,11 @@ # limitations under the License. from minifi_test_framework.core.minifi_test_context import MinifiTestContext -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.helpers import wait_for_condition -class ReverseProxyContainer(Container): +class ReverseProxyContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): super().__init__("minifi-reverse-proxy:latest", f"reverse-proxy-{test_context.scenario_id}", test_context.network) self.environment = [ diff --git a/extensions/kafka/tests/features/steps/kafka_server_container.py b/extensions/kafka/tests/features/steps/kafka_server_container.py index 30073186f..09e35e131 100644 --- a/extensions/kafka/tests/features/steps/kafka_server_container.py +++ b/extensions/kafka/tests/features/steps/kafka_server_container.py @@ -15,17 +15,18 @@ import logging import re + +from cryptography.hazmat.primitives import serialization import jks -from OpenSSL import crypto from minifi_test_framework.core.helpers import wait_for_condition -from minifi_test_framework.core.ssl_utils import make_server_cert -from minifi_test_framework.containers.container import Container +from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.containers.file import File from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class KafkaServer(Container): +class KafkaServer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): super().__init__("apache/kafka:4.1.0", f"kafka-server-{test_context.scenario_id}", test_context.network) self.environment.append("KAFKA_NODE_ID=1") @@ -55,7 +56,7 @@ class KafkaServer(Container): kafka_cert, kafka_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) - pke = jks.PrivateKeyEntry.new('kafka-broker-cert', [crypto.dump_certificate(crypto.FILETYPE_ASN1, kafka_cert)], crypto.dump_privatekey(crypto.FILETYPE_ASN1, kafka_key), 'rsa_raw') + pke = jks.PrivateKeyEntry.new('kafka-broker-cert', [dump_cert(kafka_cert, encoding_type=serialization.Encoding.DER)], dump_key(kafka_key, encoding_type=serialization.Encoding.DER), 'rsa_raw') server_keystore = jks.KeyStore.new('jks', [pke]) server_keystore_content = server_keystore.saves('abcdefgh') self.files.append(File("/etc/kafka/secrets/kafka.keystore.jks", server_keystore_content, permissions=0o644)) @@ -63,7 +64,7 @@ class KafkaServer(Container): trusted_cert = jks.TrustedCertEntry.new( 'root-ca', # Alias for the certificate - crypto.dump_certificate(crypto.FILETYPE_ASN1, test_context.root_ca_cert) + dump_cert(test_context.root_ca_cert, encoding_type=serialization.Encoding.DER) ) # Create a JKS keystore that will serve as a truststore with the trusted cert entry. diff --git a/extensions/opc/tests/features/steps/opc_ua_server_container.py b/extensions/opc/tests/features/steps/opc_ua_server_container.py index 868e54202..c9ba312ec 100644 --- a/extensions/opc/tests/features/steps/opc_ua_server_container.py +++ b/extensions/opc/tests/features/steps/opc_ua_server_container.py @@ -14,12 +14,12 @@ # limitations under the License. from typing import List, Optional -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class OPCUAServerContainer(Container): +class OPCUAServerContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext, command: Optional[List[str]] = None): super().__init__("lordgamez/open62541:1.4.10", f"opcua-server-{test_context.scenario_id}", test_context.network, command=command) diff --git a/extensions/splunk/tests/features/steps/splunk_container.py b/extensions/splunk/tests/features/steps/splunk_container.py index 4106bb63e..9a1488e63 100644 --- a/extensions/splunk/tests/features/steps/splunk_container.py +++ b/extensions/splunk/tests/features/steps/splunk_container.py @@ -15,15 +15,14 @@ import json -from OpenSSL import crypto -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.minifi_test_context import MinifiTestContext from minifi_test_framework.core.helpers import wait_for_condition, retry_check from minifi_test_framework.containers.file import File -from minifi_test_framework.core.ssl_utils import make_server_cert +from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key -class SplunkContainer(Container): +class SplunkContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): super().__init__("splunk/splunk:9.2.1-patch2", f"splunk-{test_context.scenario_id}", test_context.network) self.user = None @@ -43,9 +42,9 @@ splunk: self.files.append(File("/tmp/defaults/default.yml", splunk_config_content, mode="rw", permissions=0o644)) splunk_cert, splunk_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) - splunk_cert_content = crypto.dump_certificate(crypto.FILETYPE_PEM, splunk_cert) - splunk_key_content = crypto.dump_privatekey(crypto.FILETYPE_PEM, splunk_key) - root_ca_content = crypto.dump_certificate(crypto.FILETYPE_PEM, test_context.root_ca_cert) + splunk_cert_content = dump_cert(splunk_cert) + splunk_key_content = dump_key(splunk_key) + root_ca_content = dump_cert(test_context.root_ca_cert) self.files.append(File("/opt/splunk/etc/auth/splunk_cert.pem", splunk_cert_content.decode() + splunk_key_content.decode() + root_ca_content.decode(), permissions=0o644)) self.files.append(File("/opt/splunk/etc/auth/root_ca.pem", root_ca_content.decode(), permissions=0o644)) diff --git a/extensions/sql/tests/features/steps/postgress_server_container.py b/extensions/sql/tests/features/steps/postgress_server_container.py index 467eecc4b..a61209528 100644 --- a/extensions/sql/tests/features/steps/postgress_server_container.py +++ b/extensions/sql/tests/features/steps/postgress_server_container.py @@ -17,13 +17,13 @@ import logging from textwrap import dedent -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class PostgresContainer(Container): +class PostgresContainer(LinuxContainer): def __init__(self, context): dockerfile = dedent("""\ FROM {base_image} diff --git a/extensions/standard-processors/tests/features/core_functionality.feature b/extensions/standard-processors/tests/features/core_functionality.feature index 4b5f6b38b..0acdbb576 100644 --- a/extensions/standard-processors/tests/features/core_functionality.feature +++ b/extensions/standard-processors/tests/features/core_functionality.feature @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -@CORE +@CORE @SUPPORTS_WINDOWS Feature: Core flow functionalities Test core flow configuration functionalities @@ -33,8 +33,8 @@ Feature: Core flow functionalities When all instances start up - Then at least one file with the content "first_custom_text" is placed in the "/tmp/output" directory in less than 20 seconds - And at least one file with the content "second_custom_text" is placed in the "/tmp/output" directory in less than 20 seconds + Then at least one file with the content "first_custom_text" is placed in the "/tmp/output" directory in less than 200 seconds + And at least one file with the content "second_custom_text" is placed in the "/tmp/output" directory in less than 200 seconds Scenario: A funnel can be used as a terminator Given a GenerateFlowFile processor with the "Data Format" property set to "Text" @@ -77,7 +77,7 @@ Feature: Core flow functionalities And a non-sensitive parameter in the flow config called 'FILE_INPUT_PATH' with the value '/tmp/input' in the parameter context 'my-context' And a non-sensitive parameter in the flow config called 'FILE_OUTPUT_UPPER_PATH_ATTR' with the value 'upper_out_path_attr' in the parameter context 'my-context' And a GetFile processor with the "Input Directory" property set to "#{FILE_INPUT_PATH}" - And a file with filename "test_file_name" and content "test content" is present in "/tmp/input" + And a directory at "/tmp/input" has a file ("test_file_name") with the content "test content" And a UpdateAttribute processor with the "expr-lang-filename" property set to "#{FILENAME}" And the "is-upper-correct" property of the UpdateAttribute processor is set to "${#{FILENAME_IN_EXPRESSION}:toUpper():equals('TEST_FILE_NAME')}" And the "upper_out_path_attr" property of the UpdateAttribute processor is set to "/TMP/OUTPUT" diff --git a/extensions/standard-processors/tests/features/replace_text.feature b/extensions/standard-processors/tests/features/replace_text.feature index f8a3fdb05..b24a0e873 100644 --- a/extensions/standard-processors/tests/features/replace_text.feature +++ b/extensions/standard-processors/tests/features/replace_text.feature @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -@CORE +@CORE @SUPPORTS_WINDOWS Feature: Changing flowfile contents using the ReplaceText processor Scenario Outline: Replace text using Entire text mode diff --git a/extensions/standard-processors/tests/features/steps/diag_slave_container.py b/extensions/standard-processors/tests/features/steps/diag_slave_container.py index ba89e75c3..8defe490f 100644 --- a/extensions/standard-processors/tests/features/steps/diag_slave_container.py +++ b/extensions/standard-processors/tests/features/steps/diag_slave_container.py @@ -18,13 +18,13 @@ import logging from textwrap import dedent -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class DiagSlave(Container): +class DiagSlave(LinuxContainer): def __init__(self, test_context: MinifiTestContext): dockerfile = dedent("""\ FROM panterdsd/diagslave:latest diff --git a/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py b/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py index c65d67e0f..c53df9b02 100644 --- a/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py +++ b/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py @@ -22,52 +22,52 @@ from minifi_test_framework.core.minifi_test_context import MinifiTestContext @given('controller socket properties are set up') def step_impl(context: MinifiTestContext): - context.get_or_create_default_minifi_container().set_controller_socket_properties() + context.get_or_create_default_minifi_container().controller.set_controller_socket_properties() @when('MiNiFi config is updated through MiNiFi controller') def step_impl(context: MinifiTestContext): - context.get_or_create_default_minifi_container().update_flow_config_through_controller() + context.get_or_create_default_minifi_container().controller.update_flow_config_through_controller() @then('the updated config is persisted') def step_impl(context: MinifiTestContext): - assert context.get_or_create_default_minifi_container().updated_config_is_persisted() + assert context.get_or_create_default_minifi_container().controller.updated_config_is_persisted() @when('the {component} component is stopped through MiNiFi controller') def step_impl(context: MinifiTestContext, component: str): - context.get_or_create_default_minifi_container().stop_component_through_controller(component) + context.get_or_create_default_minifi_container().controller.stop_component_through_controller(component) @when('the {component} component is started through MiNiFi controller') def step_impl(context: MinifiTestContext, component: str): - context.get_or_create_default_minifi_container().start_component_through_controller(component) + context.get_or_create_default_minifi_container().controller.start_component_through_controller(component) @then('the {component} component is not running') def step_impl(context: MinifiTestContext, component: str): - assert not context.get_or_create_default_minifi_container().is_component_running(component) + assert not context.get_or_create_default_minifi_container().controller.is_component_running(component) @then('the {component} component is running') def step_impl(context: MinifiTestContext, component: str): - assert context.get_or_create_default_minifi_container().is_component_running(component) + assert context.get_or_create_default_minifi_container().controller.is_component_running(component) @then('connection \"{connection}\" can be seen through MiNiFi controller') def step_impl(context: MinifiTestContext, connection: str): - assert context.get_or_create_default_minifi_container().connection_found_through_controller(connection) + assert context.get_or_create_default_minifi_container().controller.connection_found_through_controller(connection) @then('{connection_count:d} connections can be seen full through MiNiFi controller') def step_impl(context: MinifiTestContext, connection_count: int): - assert context.get_or_create_default_minifi_container().get_full_connection_count() == connection_count + assert context.get_or_create_default_minifi_container().controller.get_full_connection_count() == connection_count @retry_check(10, 1) def check_connection_size_through_controller(context: MinifiTestContext, connection: str, size: int, max_size: int) -> bool: - return context.get_or_create_default_minifi_container().get_connection_size(connection) == (size, max_size) + return context.get_or_create_default_minifi_container().controller.get_connection_size(connection) == (size, max_size) @then('connection \"{connection}\" has {size:d} size and {max_size:d} max size through MiNiFi controller') @@ -77,7 +77,7 @@ def step_impl(context: MinifiTestContext, connection: str, size: int, max_size: @retry_check(10, 1) def manifest_can_be_retrieved_through_minifi_controller(context: MinifiTestContext) -> bool: - manifest = context.get_or_create_default_minifi_container().get_manifest() + manifest = context.get_or_create_default_minifi_container().controller.get_manifest() return '"agentManifest": {' in manifest and '"componentManifest": {' in manifest and '"agentType": "cpp"' in manifest @@ -88,4 +88,4 @@ def step_impl(context: MinifiTestContext): @then('debug bundle can be retrieved through MiNiFi controller') def step_impl(context: MinifiTestContext): - assert context.get_or_create_default_minifi_container().create_debug_bundle() + assert context.get_or_create_default_minifi_container().controller.create_debug_bundle() diff --git a/extensions/standard-processors/tests/features/steps/syslog_container.py b/extensions/standard-processors/tests/features/steps/syslog_container.py index f2da956f5..c831af6f1 100644 --- a/extensions/standard-processors/tests/features/steps/syslog_container.py +++ b/extensions/standard-processors/tests/features/steps/syslog_container.py @@ -15,10 +15,10 @@ # limitations under the License. # -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer -class SyslogContainer(Container): +class SyslogContainer(LinuxContainer): def __init__(self, protocol, context): super(SyslogContainer, self).__init__("ubuntu:24.04", f"syslog-{protocol}-{context.scenario_id}", context.network) self.command = f'/bin/bash -c "echo Syslog {protocol} client started; while true; do logger --{protocol} -n minifi-primary-{context.scenario_id} -P 514 sample_log; sleep 1; done"' diff --git a/extensions/standard-processors/tests/features/steps/tcp_client_container.py b/extensions/standard-processors/tests/features/steps/tcp_client_container.py index aca77dfae..0d90f3622 100644 --- a/extensions/standard-processors/tests/features/steps/tcp_client_container.py +++ b/extensions/standard-processors/tests/features/steps/tcp_client_container.py @@ -13,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class TcpClientContainer(Container): +class TcpClientContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): cmd = ( "/bin/sh -c 'apk add netcat-openbsd && " diff --git a/run_flake8.sh b/run_flake8.sh index 2288893bc..1effd79ae 100755 --- a/run_flake8.sh +++ b/run_flake8.sh @@ -19,4 +19,4 @@ set -euo pipefail directory=${1:-.} -flake8 --exclude .venv,venv,thirdparty,build,cmake-build-*,github_env --builtins log,REL_SUCCESS,REL_FAILURE,REL_ORIGINAL,raw_input --ignore E501,W503,F811 "${directory}" +flake8 --exclude .venv,venv,venv2,behave-venv,thirdparty,build,cmake-build-*,github_env --builtins log,REL_SUCCESS,REL_FAILURE,REL_ORIGINAL,raw_input --ignore E501,W503,F811 "${directory}"
