This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit f34569ee1bbc9bd424f8033a20c4bdd7f5ac4c6e Author: Gabor Gyimesi <[email protected]> AuthorDate: Thu Nov 20 14:33:13 2025 +0100 MINIFICPP-2665 Move OPC UA tests to modular docker tests - Allow using multiple minifi containers, and use dictionaries for naming and storing containers instead of a list - Allow using host file resources in test minifi containers Closes #2052 Signed-off-by: Marton Szasz <[email protected]> --- .../minifi_test_framework/containers/container.py | 22 +++-- .../minifi_test_framework/containers/host_file.py | 4 +- .../containers/minifi_container.py | 4 +- .../src/minifi_test_framework/core/helpers.py | 3 +- .../src/minifi_test_framework/core/hooks.py | 45 +++++---- .../core/minifi_test_context.py | 23 ++++- .../minifi_test_framework/steps/checking_steps.py | 59 ++++++------ .../steps/configuration_steps.py | 10 +- .../src/minifi_test_framework/steps/core_steps.py | 27 ++++-- .../steps/flow_building_steps.py | 101 ++++++++++++++------- docker/RunBehaveTests.sh | 3 +- docker/test/integration/cluster/ContainerStore.py | 9 -- .../cluster/DockerTestDirectoryBindings.py | 1 - docker/test/integration/features/http.feature | 2 +- docker/test/integration/features/https.feature | 2 +- docker/test/integration/features/steps/steps.py | 18 +--- extensions/aws/tests/features/steps/steps.py | 14 +-- extensions/azure/tests/features/steps/steps.py | 38 ++++---- extensions/opc/tests/features/environment.py | 28 ++++++ .../opc/tests}/features/opcua.feature | 84 ++++++++++------- .../features/resources}/opcua_client_cert.der | Bin .../tests/features/resources}/opcua_client_key.der | Bin .../features/steps/opc_ua_server_container.py | 32 +++---- extensions/opc/tests/features/steps/steps.py | 47 ++++++++++ extensions/sql/tests/features/steps/steps.py | 12 +-- .../tests/features/steps/steps.py | 4 +- .../tests/features/steps/syslog_container.py | 2 +- 27 files changed, 375 insertions(+), 219 deletions(-) diff --git a/behave_framework/src/minifi_test_framework/containers/container.py b/behave_framework/src/minifi_test_framework/containers/container.py index d3a8316df..dc0cf7353 100644 --- a/behave_framework/src/minifi_test_framework/containers/container.py +++ b/behave_framework/src/minifi_test_framework/containers/container.py @@ -29,22 +29,25 @@ from minifi_test_framework.containers.host_file import HostFile class Container: - def __init__(self, image_name: str, container_name: str, network: Network): + def __init__(self, image_name: str, container_name: str, network: Network, command: str | None = None): self.image_name: str = image_name self.container_name: str = container_name self.network: Network = network self.user: str = "0:0" - self.client = docker.from_env() + self.client: docker.DockerClient = docker.from_env() self.container = None self.files: list[File] = [] self.dirs: list[Directory] = [] self.host_files: list[HostFile] = [] self.volumes = {} - self.command = None - self._temp_dir = None - self.ports = None + self.command: str | None = command + self._temp_dir: tempfile.TemporaryDirectory | None = None + self.ports: dict[str, int] | None = None self.environment: list[str] = [] + def add_host_file(self, host_path: str, container_path: str, mode: str = "ro"): + self.host_files.append(HostFile(container_path, host_path, mode)) + def deploy(self) -> bool: self._temp_dir = tempfile.TemporaryDirectory() @@ -63,7 +66,7 @@ class Container: temp_file.write(content) self.volumes[temp_path] = {"bind": directory.path, "mode": directory.mode} for host_file in self.host_files: - self.volumes[host_file.container_path] = {"bind": host_file.host_path, "mode": host_file.mode} + self.volumes[host_file.host_path] = {"bind": host_file.container_path, "mode": host_file.mode} try: existing_container = self.client.containers.get(self.container_name) @@ -84,7 +87,10 @@ class Container: def clean_up(self): if self.container: - self.container.remove(force=True) + try: + self.container.remove(force=True) + except Exception as e: + logging.error(f"Error cleaning up container '{self.container_name}': {e}") def exec_run(self, command) -> tuple[int | None, str]: if self.container: @@ -110,7 +116,7 @@ class Container: quoted_content = shlex.quote(expected_content) command = "sh -c {}".format(shlex.quote(f"grep -l -F -- {quoted_content} {directory_path}/*")) - exit_code, output = self.exec_run(command) + exit_code, _ = self.exec_run(command) return exit_code == 0 diff --git a/behave_framework/src/minifi_test_framework/containers/host_file.py b/behave_framework/src/minifi_test_framework/containers/host_file.py index 9b59945d2..1e046f54c 100644 --- a/behave_framework/src/minifi_test_framework/containers/host_file.py +++ b/behave_framework/src/minifi_test_framework/containers/host_file.py @@ -16,7 +16,7 @@ # class HostFile: - def __init__(self, path, host_path): + def __init__(self, path, host_path, mode="ro"): self.container_path = path self.host_path = host_path - self.mode = "ro" + self.mode = mode diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_container.py index 7c2661d44..805b04227 100644 --- a/behave_framework/src/minifi_test_framework/containers/minifi_container.py +++ b/behave_framework/src/minifi_test_framework/containers/minifi_container.py @@ -23,8 +23,8 @@ from .container import Container class MinifiContainer(Container): - def __init__(self, image_name: str, scenario_id: str, network: Network): - super().__init__(image_name, f"minifi-{scenario_id}", network) + def __init__(self, image_name: str, container_name: str, scenario_id: str, network: Network): + super().__init__(image_name, f"{container_name}-{scenario_id}", network) self.flow_config_str: str = "" self.flow_definition = FlowDefinition() self.properties: dict[str, str] = {} diff --git a/behave_framework/src/minifi_test_framework/core/helpers.py b/behave_framework/src/minifi_test_framework/core/helpers.py index 01fa506b0..3c551b8d4 100644 --- a/behave_framework/src/minifi_test_framework/core/helpers.py +++ b/behave_framework/src/minifi_test_framework/core/helpers.py @@ -27,9 +27,8 @@ from minifi_test_framework.core.minifi_test_context import MinifiTestContext def log_due_to_failure(context: MinifiTestContext | None): if context is not None: - for container in context.containers: + for container in context.containers.values(): container.log_app_output() - context.minifi_container.log_app_output() def wait_for_condition(condition: Callable[[], bool], timeout_seconds: float, bail_condition: Callable[[], bool], diff --git a/behave_framework/src/minifi_test_framework/core/hooks.py b/behave_framework/src/minifi_test_framework/core/hooks.py index 0a4c33861..5513c9f51 100644 --- a/behave_framework/src/minifi_test_framework/core/hooks.py +++ b/behave_framework/src/minifi_test_framework/core/hooks.py @@ -17,13 +17,12 @@ import logging import os - import docker +import types + from behave.model import Scenario -from behave.model import Step from behave.runner import Context -from minifi_test_framework.containers.minifi_container import MinifiContainer from minifi_test_framework.core.minifi_test_context import MinifiTestContext @@ -35,40 +34,52 @@ def get_minifi_container_image(): return "apacheminificpp:behave" +def inject_scenario_id(context: MinifiTestContext, step): + if "${scenario_id}" in step.name: + step.name = step.name.replace("${scenario_id}", context.scenario_id) + if getattr(step, "table", None): + for row in step.table: + row.cells = [cell.replace("${scenario_id}", context.scenario_id) if "${scenario_id}" in cell else cell for cell in row.cells] + if hasattr(step, "text") and step.text and "${scenario_id}" in step.text: + step.text = step.text.replace("${scenario_id}", context.scenario_id) + + def common_before_scenario(context: Context, scenario: Scenario): if not hasattr(context, "minifi_container_image"): context.minifi_container_image = get_minifi_container_image() + method_map = { + "get_or_create_minifi_container": MinifiTestContext.get_or_create_minifi_container, + "get_or_create_default_minifi_container": MinifiTestContext.get_or_create_default_minifi_container, + "get_minifi_container": MinifiTestContext.get_minifi_container, + "get_default_minifi_container": MinifiTestContext.get_default_minifi_container, + } + for attr, method in method_map.items(): + if not hasattr(context, attr): + setattr(context, attr, types.MethodType(method, context)) + logging.info("Running scenario: %s", scenario) context.scenario_id = scenario.filename.rsplit("/", 1)[1].split(".")[0] + "-" + str( scenario.parent.scenarios.index(scenario)) network_name = f"{context.scenario_id}-net" docker_client = docker.client.from_env() + try: existing_network = docker_client.networks.get(network_name) logging.warning(f"Found existing network '{network_name}'. Removing it first.") existing_network.remove() except docker.errors.NotFound: pass # No existing network found, which is good. + context.network = docker_client.networks.create(network_name) - context.minifi_container = MinifiContainer(context.minifi_container_image, context.scenario_id, context.network) - context.containers = [] + context.containers = {} + context.resource_dir = None + for step in scenario.steps: inject_scenario_id(context, step) def common_after_scenario(context: MinifiTestContext, scenario: Scenario): - for container in context.containers: + for container in context.containers.values(): container.clean_up() - context.minifi_container.clean_up() context.network.remove() - - -def inject_scenario_id(context: MinifiTestContext, step: Step): - if "${scenario_id}" in step.name: - step.name = step.name.replace("${scenario_id}", context.scenario_id) - if step.table: - for row in step.table: - for i in range(len(row.cells)): - if "${scenario_id}" in row.cells[i]: - row.cells[i] = row.cells[i].replace("${scenario_id}", context.scenario_id) diff --git a/behave_framework/src/minifi_test_framework/core/minifi_test_context.py b/behave_framework/src/minifi_test_framework/core/minifi_test_context.py index 6c7892c05..ab9110de0 100644 --- a/behave_framework/src/minifi_test_framework/core/minifi_test_context.py +++ b/behave_framework/src/minifi_test_framework/core/minifi_test_context.py @@ -21,9 +21,28 @@ from docker.models.networks import Network from minifi_test_framework.containers.container import Container from minifi_test_framework.containers.minifi_container import MinifiContainer +DEFAULT_MINIFI_CONTAINER_NAME = "minifi-primary" + class MinifiTestContext(Context): - minifi_container: MinifiContainer - containers: list[Container] + containers: dict[str, Container] scenario_id: str network: Network + minifi_container_image: str + resource_dir: str | None + + def get_or_create_minifi_container(self, container_name: str) -> MinifiContainer: + if container_name not in self.containers: + self.containers[container_name] = MinifiContainer(self.minifi_container_image, container_name, self.scenario_id, self.network) + return self.containers[container_name] + + def get_or_create_default_minifi_container(self) -> MinifiContainer: + return self.get_or_create_minifi_container(DEFAULT_MINIFI_CONTAINER_NAME) + + def get_minifi_container(self, container_name: str) -> MinifiContainer: + if container_name not in self.containers: + raise KeyError(f"MiNiFi container '{container_name}' does not exist in the test context.") + return self.containers[container_name] + + def get_default_minifi_container(self) -> MinifiContainer: + return self.get_minifi_container(DEFAULT_MINIFI_CONTAINER_NAME) diff --git a/behave_framework/src/minifi_test_framework/steps/checking_steps.py b/behave_framework/src/minifi_test_framework/steps/checking_steps.py index e4d46c7b4..231a2be89 100644 --- a/behave_framework/src/minifi_test_framework/steps/checking_steps.py +++ b/behave_framework/src/minifi_test_framework/steps/checking_steps.py @@ -23,7 +23,7 @@ from behave import then, step from minifi_test_framework.containers.http_proxy_container import HttpProxy from minifi_test_framework.core.helpers import wait_for_condition -from minifi_test_framework.core.minifi_test_context import MinifiTestContext +from minifi_test_framework.core.minifi_test_context import DEFAULT_MINIFI_CONTAINER_NAME, MinifiTestContext @then('there is a file with "{content}" content at {path} in less than {duration}') @@ -31,8 +31,8 @@ def step_impl(context: MinifiTestContext, content: str, path: str, duration: str new_content = content.replace("\\n", "\n") timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( - condition=lambda: context.minifi_container.path_with_content_exists(path, new_content), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.minifi_container.exited, context=context) + condition=lambda: context.get_default_minifi_container().path_with_content_exists(path, new_content), + timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.get_default_minifi_container().exited, context=context) @then('there is a single file with "{content}" content in the "{directory}" directory in less than {duration}') @@ -40,47 +40,52 @@ def step_impl(context: MinifiTestContext, content: str, directory: str, duration new_content = content.replace("\\n", "\n") timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( - condition=lambda: context.minifi_container.directory_has_single_file_with_content(directory, new_content), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.minifi_container.exited, context=context) + condition=lambda: context.get_default_minifi_container().directory_has_single_file_with_content(directory, new_content), + timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.get_default_minifi_container().exited, context=context) -@then('at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') -def step_impl(context: MinifiTestContext, content: str, directory: str, duration: str): +@then('in the "{container_name}" container at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') +def step_impl(context: MinifiTestContext, container_name: str, content: str, directory: str, duration: str): timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( - condition=lambda: context.minifi_container.directory_contains_file_with_content(directory, content), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.minifi_container.exited, context=context) + condition=lambda: context.get_minifi_container(container_name).directory_contains_file_with_content(directory, content), + timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.get_minifi_container(container_name).exited, context=context) + + +@then('at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') +def step_impl(context: MinifiTestContext, content: str, directory: str, duration: str): + context.execute_steps(f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') @then('the Minifi logs do not contain the following message: "{message}" after {duration}') def step_impl(context: MinifiTestContext, message: str, duration: str): duration_seconds = humanfriendly.parse_timespan(duration) time.sleep(duration_seconds) - assert message not in context.minifi_container.get_logs() + assert message not in context.get_default_minifi_container().get_logs() @then("the Minifi logs do not contain errors") def step_impl(context: MinifiTestContext): - assert "[error]" not in context.minifi_container.get_logs() or context.minifi_container.log_app_output() + assert "[error]" not in context.get_default_minifi_container().get_logs() or context.get_default_minifi_container().log_app_output() @then("the Minifi logs do not contain warnings") def step_impl(context: MinifiTestContext): - assert "[warning]" not in context.minifi_container.get_logs() or context.minifi_container.log_app_output() + assert "[warning]" not in context.get_default_minifi_container().get_logs() or context.get_default_minifi_container().log_app_output() @then("the Minifi logs contain the following message: '{message}' in less than {duration}") @then('the Minifi logs contain the following message: "{message}" in less than {duration}') def step_impl(context: MinifiTestContext, message: str, duration: str): duration_seconds = humanfriendly.parse_timespan(duration) - assert wait_for_condition(condition=lambda: message in context.minifi_container.get_logs(), - timeout_seconds=duration_seconds, bail_condition=lambda: context.minifi_container.exited, + assert wait_for_condition(condition=lambda: message in context.get_default_minifi_container().get_logs(), + timeout_seconds=duration_seconds, bail_condition=lambda: context.get_default_minifi_container().exited, context=context) @step('no errors were generated on the http-proxy regarding "{url}"') def step_impl(context: MinifiTestContext, url: str): - http_proxy_container = next(container for container in context.containers if isinstance(container, HttpProxy)) + http_proxy_container = next(container for container in context.containers.values() if isinstance(container, HttpProxy)) assert http_proxy_container.check_http_proxy_access(url) or http_proxy_container.log_app_output() @@ -88,16 +93,16 @@ def step_impl(context: MinifiTestContext, url: str): @then('there is {num_str} file in the "{directory}" directory in less than {duration}') def step_impl(context: MinifiTestContext, num_str: str, directory: str, duration: str): duration_seconds = humanfriendly.parse_timespan(duration) - assert wait_for_condition(condition=lambda: context.minifi_container.get_number_of_files(directory) == int(num_str), - timeout_seconds=duration_seconds, bail_condition=lambda: context.minifi_container.exited, + assert wait_for_condition(condition=lambda: context.get_default_minifi_container().get_number_of_files(directory) == int(num_str), + timeout_seconds=duration_seconds, bail_condition=lambda: context.get_default_minifi_container().exited, context=context) @then('there are at least {num_str} files is in the "{directory}" directory in less than {duration}') def step_impl(context: MinifiTestContext, num_str: str, directory: str, duration: str): duration_seconds = humanfriendly.parse_timespan(duration) - assert wait_for_condition(condition=lambda: context.minifi_container.get_number_of_files(directory) >= int(num_str), - timeout_seconds=duration_seconds, bail_condition=lambda: context.minifi_container.exited, + assert wait_for_condition(condition=lambda: context.get_default_minifi_container().get_number_of_files(directory) >= int(num_str), + timeout_seconds=duration_seconds, bail_condition=lambda: context.get_default_minifi_container().exited, context=context) @@ -105,8 +110,8 @@ def step_impl(context: MinifiTestContext, num_str: str, directory: str, duration def step_impl(context: MinifiTestContext, directory: str, regex_str: str, duration: str): duration_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( - condition=lambda: context.minifi_container.directory_contains_file_with_regex(directory, regex_str), - timeout_seconds=duration_seconds, bail_condition=lambda: context.minifi_container.exited, context=context) + condition=lambda: context.get_default_minifi_container().directory_contains_file_with_regex(directory, regex_str), + timeout_seconds=duration_seconds, bail_condition=lambda: context.get_default_minifi_container().exited, context=context) @then('the contents of {directory} in less than {timeout} are: "{content_one}" and "{content_two}"') @@ -115,8 +120,8 @@ def step_impl(context: MinifiTestContext, directory: str, timeout: str, content_ c1 = content_one.replace("\\n", "\n") c2 = content_two.replace("\\n", "\n") contents_arr = [c1, c2] - assert wait_for_condition(condition=lambda: context.minifi_container.verify_file_contents(directory, contents_arr), - timeout_seconds=timeout_seconds, bail_condition=lambda: context.minifi_container.exited, + assert wait_for_condition(condition=lambda: context.get_default_minifi_container().verify_file_contents(directory, contents_arr), + timeout_seconds=timeout_seconds, bail_condition=lambda: context.get_default_minifi_container().exited, context=context) @@ -125,8 +130,8 @@ def step_impl(context: MinifiTestContext, directory: str, timeout: str, contents timeout_seconds = humanfriendly.parse_timespan(timeout) new_contents = contents.replace("\\n", "\n") contents_arr = new_contents.split(",") - assert wait_for_condition(condition=lambda: context.minifi_container.verify_file_contents(directory, contents_arr), - timeout_seconds=timeout_seconds, bail_condition=lambda: context.minifi_container.exited, + assert wait_for_condition(condition=lambda: context.get_default_minifi_container().verify_file_contents(directory, contents_arr), + timeout_seconds=timeout_seconds, bail_condition=lambda: context.get_default_minifi_container().exited, context=context) @@ -135,5 +140,5 @@ def step_impl(context: MinifiTestContext, directory: str, timeout: str, contents def step_impl(context: MinifiTestContext, content: str, directory: str, duration: str): timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( - condition=lambda: context.minifi_container.verify_path_with_json_content(directory, content), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.minifi_container.exited, context=context) + condition=lambda: context.get_default_minifi_container().verify_path_with_json_content(directory, content), + timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.get_default_minifi_container().exited, context=context) diff --git a/behave_framework/src/minifi_test_framework/steps/configuration_steps.py b/behave_framework/src/minifi_test_framework/steps/configuration_steps.py index 5eaeaa326..a9794936f 100644 --- a/behave_framework/src/minifi_test_framework/steps/configuration_steps.py +++ b/behave_framework/src/minifi_test_framework/steps/configuration_steps.py @@ -22,16 +22,16 @@ from minifi_test_framework.core.minifi_test_context import MinifiTestContext @step('MiNiFi configuration "{config_key}" is set to "{config_value}"') def step_impl(context: MinifiTestContext, config_key: str, config_value: str): - context.minifi_container.set_property(config_key, config_value) + context.get_or_create_default_minifi_container().set_property(config_key, config_value) @step("log metrics publisher is enabled in MiNiFi") def step_impl(context): - context.minifi_container.set_property("nifi.metrics.publisher.LogMetricsPublisher.metrics", "RepositoryMetrics") - context.minifi_container.set_property("nifi.metrics.publisher.LogMetricsPublisher.logging.interval", "1s") - context.minifi_container.set_property("nifi.metrics.publisher.class", "LogMetricsPublisher") + context.get_or_create_default_minifi_container().set_property("nifi.metrics.publisher.LogMetricsPublisher.metrics", "RepositoryMetrics") + context.get_or_create_default_minifi_container().set_property("nifi.metrics.publisher.LogMetricsPublisher.logging.interval", "1s") + context.get_or_create_default_minifi_container().set_property("nifi.metrics.publisher.class", "LogMetricsPublisher") @step('log property "{log_property_key}" is set to "{log_property_value}"') def step_impl(context: MinifiTestContext, log_property_key: str, log_property_value: str): - context.minifi_container.set_log_property(log_property_key, log_property_value) + context.get_or_create_default_minifi_container().set_log_property(log_property_key, log_property_value) diff --git a/behave_framework/src/minifi_test_framework/steps/core_steps.py b/behave_framework/src/minifi_test_framework/steps/core_steps.py index f4d55357b..552ea2d19 100644 --- a/behave_framework/src/minifi_test_framework/steps/core_steps.py +++ b/behave_framework/src/minifi_test_framework/steps/core_steps.py @@ -21,24 +21,24 @@ import string import os import humanfriendly -from behave import when, step +from behave import when, step, given + from minifi_test_framework.containers.directory import Directory from minifi_test_framework.containers.file import File -from minifi_test_framework.core.minifi_test_context import MinifiTestContext +from minifi_test_framework.core.minifi_test_context import DEFAULT_MINIFI_CONTAINER_NAME, MinifiTestContext @when("both instances start up") @when("all instances start up") def step_impl(context: MinifiTestContext): - for container in context.containers: - assert container.deploy() - assert context.minifi_container.deploy() or context.minifi_container.log_app_output() + for container in context.containers.values(): + assert container.deploy() or container.log_app_output() logging.debug("All instances started up") @when("the MiNiFi instance starts up") def step_impl(context): - assert context.minifi_container.deploy() + assert context.get_or_create_default_minifi_container().deploy() logging.debug("MiNiFi instance started up") @@ -48,10 +48,21 @@ def step_impl(context: MinifiTestContext, directory: str, size: str): content = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(size)) new_dir = Directory(directory) new_dir.files["input.txt"] = content - context.minifi_container.dirs.append(new_dir) + context.get_or_create_default_minifi_container().dirs.append(new_dir) @step('a file with filename "{file_name}" and content "{content}" is present in "{path}"') def step_impl(context: MinifiTestContext, file_name: str, content: str, path: str): new_content = content.replace("\\n", "\n") - context.minifi_container.files.append(File(os.path.join(path, file_name), new_content)) + context.get_or_create_default_minifi_container().files.append(File(os.path.join(path, file_name), new_content)) + + +@given('a host resource file "{filename}" is bound to the "{container_path}" path in the MiNiFi container "{container_name}"') +def step_impl(context: MinifiTestContext, filename: str, container_path: str, container_name: str): + path = os.path.join(context.resource_dir, filename) + context.get_or_create_minifi_container(container_name).add_host_file(path, container_path) + + +@given('a host resource file "{filename}" is bound to the "{container_path}" path in the MiNiFi container') +def step_impl(context: MinifiTestContext, filename: str, container_path: str): + context.execute_steps(f"given a host resource file \"{filename}\" is bound to the \"{container_path}\" path in the MiNiFi container \"{DEFAULT_MINIFI_CONTAINER_NAME}\"") diff --git a/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py b/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py index a01ec10f9..a41d096c7 100644 --- a/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py +++ b/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py @@ -19,7 +19,7 @@ from behave import given, step from minifi_test_framework.containers.directory import Directory from minifi_test_framework.containers.http_proxy_container import HttpProxy -from minifi_test_framework.core.minifi_test_context import MinifiTestContext +from minifi_test_framework.core.minifi_test_context import DEFAULT_MINIFI_CONTAINER_NAME, MinifiTestContext from minifi_test_framework.minifi.connection import Connection from minifi_test_framework.minifi.controller_service import ControllerService from minifi_test_framework.minifi.funnel import Funnel @@ -30,8 +30,8 @@ from minifi_test_framework.minifi.processor import Processor @given("a transient MiNiFi flow with a LogOnDestructionProcessor processor") def step_impl(context: MinifiTestContext): - context.minifi_container.command = ["/bin/sh", "-c", "timeout 10s ./bin/minifi.sh run && sleep 100"] - context.minifi_container.flow_definition.add_processor( + context.get_or_create_default_minifi_container().command = ["/bin/sh", "-c", "timeout 10s ./bin/minifi.sh run && sleep 100"] + context.get_or_create_default_minifi_container().flow_definition.add_processor( Processor("LogOnDestructionProcessor", "LogOnDestructionProcessor")) @@ -41,7 +41,7 @@ def step_impl(context: MinifiTestContext, processor_type: str, processor_name: s property_value: str): processor = Processor(processor_type, processor_name) processor.add_property(property_name, property_value) - context.minifi_container.flow_definition.add_processor(processor) + context.get_or_create_default_minifi_container().flow_definition.add_processor(processor) @step('a {processor_type} processor with the "{property_name}" property set to "{property_value}"') @@ -50,75 +50,102 @@ def step_impl(context: MinifiTestContext, processor_type: str, property_name: st f'Given a {processor_type} processor with the name "{processor_type}" and the "{property_name}" property set to "{property_value}"') +@step('a {processor_type} processor with the "{property_name}" property set to "{property_value}" in the "{minifi_container_name}" flow') +def step_impl(context: MinifiTestContext, processor_type: str, property_name: str, property_value: str, minifi_container_name: str): + processor = Processor(processor_type, processor_type) + processor.add_property(property_name, property_value) + context.get_or_create_minifi_container(minifi_container_name).flow_definition.add_processor(processor) + + @given('a {processor_type} processor with the name "{processor_name}"') def step_impl(context: MinifiTestContext, processor_type: str, processor_name: str): processor = Processor(processor_type, processor_name) - context.minifi_container.flow_definition.add_processor(processor) + context.get_or_create_default_minifi_container().flow_definition.add_processor(processor) + + +@given("a {processor_type} processor in the \"{minifi_container_name}\" flow") +def step_impl(context: MinifiTestContext, processor_type: str, minifi_container_name: str): + processor = Processor(processor_type, processor_type) + context.get_or_create_minifi_container(minifi_container_name).flow_definition.add_processor(processor) @given("a {processor_type} processor") def step_impl(context: MinifiTestContext, processor_type: str): - processor = Processor(processor_type, processor_type) - context.minifi_container.flow_definition.add_processor(processor) + context.execute_steps(f'given a {processor_type} processor in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') @step('the "{property_name}" property of the {processor_name} processor is set to "{property_value}"') def step_impl(context: MinifiTestContext, property_name: str, processor_name: str, property_value: str): - processor = context.minifi_container.flow_definition.get_processor(processor_name) + processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name) processor.add_property(property_name, property_value) @step('a Funnel with the name "{funnel_name}" is set up') def step_impl(context: MinifiTestContext, funnel_name: str): - context.minifi_container.flow_definition.add_funnel(Funnel(funnel_name)) + context.get_or_create_default_minifi_container().flow_definition.add_funnel(Funnel(funnel_name)) + + +@step('in the "{minifi_container_name}" flow the "{relationship_name}" relationship of the {source} processor is connected to the {target}') +def step_impl(context: MinifiTestContext, relationship_name: str, source: str, target: str, minifi_container_name: str): + connection = Connection(source_name=source, source_relationship=relationship_name, target_name=target) + context.get_or_create_minifi_container(minifi_container_name).flow_definition.add_connection(connection) @step('the "{relationship_name}" relationship of the {source} processor is connected to the {target}') def step_impl(context: MinifiTestContext, relationship_name: str, source: str, target: str): - connection = Connection(source_name=source, source_relationship=relationship_name, target_name=target) - context.minifi_container.flow_definition.add_connection(connection) + context.execute_steps(f'given in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow the "{relationship_name}" relationship of the {source} processor is connected to the {target}') @step('the Funnel with the name "{funnel_name}" is connected to the {target}') def step_impl(context: MinifiTestContext, funnel_name: str, target: str): connection = Connection(source_name=funnel_name, source_relationship="success", target_name=target) - context.minifi_container.flow_definition.add_connection(connection) + context.get_or_create_default_minifi_container().flow_definition.add_connection(connection) + + +@step("{processor_name}'s {relationship} relationship is auto-terminated in the \"{minifi_container_name}\" flow") +def step_impl(context: MinifiTestContext, processor_name: str, relationship: str, minifi_container_name: str): + context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor(processor_name).auto_terminated_relationships.append( + relationship) @step("{processor_name}'s {relationship} relationship is auto-terminated") def step_impl(context: MinifiTestContext, processor_name: str, relationship: str): - context.minifi_container.flow_definition.get_processor(processor_name).auto_terminated_relationships.append( - relationship) + context.execute_steps(f'given {processor_name}\'s {relationship} relationship is auto-terminated in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') @given("a transient MiNiFi flow is set up") def step_impl(context: MinifiTestContext): - context.minifi_container.command = ["/bin/sh", "-c", "timeout 10s ./bin/minifi.sh run && sleep 100"] + context.get_or_create_default_minifi_container().command = ["/bin/sh", "-c", "timeout 10s ./bin/minifi.sh run && sleep 100"] @step('the scheduling period of the {processor_name} processor is set to "{duration_str}"') def step_impl(context: MinifiTestContext, processor_name: str, duration_str: str): - context.minifi_container.flow_definition.get_processor(processor_name).scheduling_period = duration_str + context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name).scheduling_period = duration_str @given("parameter context name is set to '{context_name}'") def step_impl(context: MinifiTestContext, context_name: str): - context.minifi_container.flow_definition.parameter_contexts.append(ParameterContext(context_name)) + context.get_or_create_default_minifi_container().flow_definition.parameter_contexts.append(ParameterContext(context_name)) @step( "a non-sensitive parameter in the flow config called '{parameter_name}' with the value '{parameter_value}' in the parameter context '{context_name}'") def step_impl(context: MinifiTestContext, parameter_name: str, parameter_value: str, context_name: str): - parameter_context = context.minifi_container.flow_definition.get_parameter_context(context_name) + parameter_context = context.get_or_create_default_minifi_container().flow_definition.get_parameter_context(context_name) parameter_context.parameters.append(Parameter(parameter_name, parameter_value, False)) -@step('a directory at "{directory}" has a file with the content "{content}"') -def step_impl(context: MinifiTestContext, directory: str, content: str): +@step('a directory at "{directory}" has a file with the content "{content}" in the "{flow_name}" flow') +def step_impl(context: MinifiTestContext, directory: str, content: str, flow_name: str): new_content = content.replace("\\n", "\n") new_dir = Directory(directory) new_dir.files["input.txt"] = new_content - context.minifi_container.dirs.append(new_dir) + context.get_or_create_minifi_container(flow_name).dirs.append(new_dir) + + +@step('a directory at "{directory}" has a file with the content "{content}"') +def step_impl(context: MinifiTestContext, directory: str, content: str): + context.execute_steps(f'given a directory at "{directory}" has a file with the content "{content}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') @step('a directory at "{directory}" has a file ("{file_name}") with the content "{content}"') @@ -126,19 +153,26 @@ def step_impl(context: MinifiTestContext, directory: str, file_name: str, conten new_content = content.replace("\\n", "\n") new_dir = Directory(directory) new_dir.files[file_name] = new_content - context.minifi_container.dirs.append(new_dir) + context.get_or_create_default_minifi_container().dirs.append(new_dir) + + +@given("these processor properties are set in the \"{minifi_container_name}\" flow") +def step_impl(context: MinifiTestContext, minifi_container_name: str): + for row in context.table: + processor = context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor(row["processor name"]) + processor.add_property(row["property name"], row["property value"]) @given("these processor properties are set") def step_impl(context: MinifiTestContext): for row in context.table: - processor = context.minifi_container.flow_definition.get_processor(row["processor name"]) + processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(row["processor name"]) processor.add_property(row["property name"], row["property value"]) @step("the http proxy server is set up") def step_impl(context): - context.containers.append(HttpProxy(context)) + context.containers["http-proxy"] = HttpProxy(context) @step("the processors are connected up as described here") @@ -148,23 +182,28 @@ def step_impl(context: MinifiTestContext): dest_proc_name = row["destination name"] relationship = row["relationship name"] if dest_proc_name == "auto-terminated": - context.minifi_container.flow_definition.get_processor( + context.get_or_create_default_minifi_container().flow_definition.get_processor( source_proc_name).auto_terminated_relationships.append(relationship) else: connection = Connection(source_name=row["source name"], source_relationship=relationship, target_name=row["destination name"]) - context.minifi_container.flow_definition.add_connection(connection) + context.get_or_create_default_minifi_container().flow_definition.add_connection(connection) + + +@step("{processor_name} is EVENT_DRIVEN in the \"{minifi_container_name}\" flow") +def step_impl(context: MinifiTestContext, processor_name: str, minifi_container_name: str): + processor = context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor(processor_name) + processor.scheduling_strategy = "EVENT_DRIVEN" @step("{processor_name} is EVENT_DRIVEN") def step_impl(context: MinifiTestContext, processor_name: str): - processor = context.minifi_container.flow_definition.get_processor(processor_name) - processor.scheduling_strategy = "EVENT_DRIVEN" + context.execute_steps(f'given {processor_name} is EVENT_DRIVEN in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') @step("{processor_name} is TIMER_DRIVEN with {scheduling_period} scheduling period") def step_impl(context: MinifiTestContext, processor_name: str, scheduling_period: int): - processor = context.minifi_container.flow_definition.get_processor(processor_name) + processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name) processor.scheduling_strategy = "TIMER_DRIVEN" processor.scheduling_period = scheduling_period @@ -173,11 +212,11 @@ def step_impl(context: MinifiTestContext, processor_name: str, scheduling_period @given("an {service_name} controller service is set up") def step_impl(context: MinifiTestContext, service_name: str): controller_service = ControllerService(class_name=service_name, service_name=service_name) - context.minifi_container.flow_definition.controller_services.append(controller_service) + context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) @given('a {service_name} controller service is set up and the "{property_name}" property set to "{property_value}"') def step_impl(context: MinifiTestContext, service_name: str, property_name: str, property_value: str): controller_service = ControllerService(class_name=service_name, service_name=service_name) controller_service.add_property(property_name, property_value) - context.minifi_container.flow_definition.controller_services.append(controller_service) + context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) diff --git a/docker/RunBehaveTests.sh b/docker/RunBehaveTests.sh index 4bac967f0..89d59f076 100755 --- a/docker/RunBehaveTests.sh +++ b/docker/RunBehaveTests.sh @@ -198,4 +198,5 @@ exec \ "${docker_dir}/../extensions/aws/tests/features" \ "${docker_dir}/../extensions/azure/tests/features" \ "${docker_dir}/../extensions/sql/tests/features" \ - "${docker_dir}/../extensions/llamacpp/tests/features" + "${docker_dir}/../extensions/llamacpp/tests/features" \ + "${docker_dir}/../extensions/opc/tests/features" diff --git a/docker/test/integration/cluster/ContainerStore.py b/docker/test/integration/cluster/ContainerStore.py index 9b5cba6b3..a78b0712c 100644 --- a/docker/test/integration/cluster/ContainerStore.py +++ b/docker/test/integration/cluster/ContainerStore.py @@ -26,7 +26,6 @@ from .containers.FakeGcsServerContainer import FakeGcsServerContainer from .containers.HttpProxyContainer import HttpProxyContainer from .containers.PostgreSQLServerContainer import PostgreSQLServerContainer from .containers.MqttBrokerContainer import MqttBrokerContainer -from .containers.OPCUAServerContainer import OPCUAServerContainer from .containers.SplunkContainer import SplunkContainer from .containers.ElasticsearchContainer import ElasticsearchContainer from .containers.OpensearchContainer import OpensearchContainer @@ -184,14 +183,6 @@ class ContainerStore: network=self.network, image_store=self.image_store, command=command)) - elif engine == 'opcua-server': - return self.containers.setdefault(container_name, - OPCUAServerContainer(feature_context=feature_context, - name=container_name, - vols=self.vols, - network=self.network, - image_store=self.image_store, - command=command)) elif engine == 'splunk': return self.containers.setdefault(container_name, SplunkContainer(feature_context=feature_context, diff --git a/docker/test/integration/cluster/DockerTestDirectoryBindings.py b/docker/test/integration/cluster/DockerTestDirectoryBindings.py index 25add90ae..9834bc524 100644 --- a/docker/test/integration/cluster/DockerTestDirectoryBindings.py +++ b/docker/test/integration/cluster/DockerTestDirectoryBindings.py @@ -55,7 +55,6 @@ class DockerTestDirectoryBindings: # Add resources test_dir = os.environ['TEST_DIRECTORY'] # Based on DockerVerify.sh shutil.copytree(test_dir + "/resources/python", self.data_directories[self.feature_id]["resources_dir"] + "/python") - shutil.copytree(test_dir + "/resources/opcua", self.data_directories[self.feature_id]["resources_dir"] + "/opcua") shutil.copytree(test_dir + "/resources/lua", self.data_directories[self.feature_id]["resources_dir"] + "/lua") shutil.copytree(test_dir + "/resources/minifi", self.data_directories[self.feature_id]["minifi_config_dir"], dirs_exist_ok=True) shutil.copytree(test_dir + "/resources/minifi-controller", self.data_directories[self.feature_id]["resources_dir"] + "/minifi-controller") diff --git a/docker/test/integration/features/http.feature b/docker/test/integration/features/http.feature index 0cd01fbbb..19a9d3745 100644 --- a/docker/test/integration/features/http.feature +++ b/docker/test/integration/features/http.feature @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -@CORE +@ENABLE_CIVET Feature: Sending data using InvokeHTTP to a receiver using ListenHTTP In order to send and receive data via HTTP As a user of MiNiFi diff --git a/docker/test/integration/features/https.feature b/docker/test/integration/features/https.feature index f4ff9d246..863cc782e 100644 --- a/docker/test/integration/features/https.feature +++ b/docker/test/integration/features/https.feature @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -@CORE +@ENABLE_CIVET Feature: Transfer data from and to MiNiFi using HTTPS Background: diff --git a/docker/test/integration/features/steps/steps.py b/docker/test/integration/features/steps/steps.py index 590fa06d0..87285e409 100644 --- a/docker/test/integration/features/steps/steps.py +++ b/docker/test/integration/features/steps/steps.py @@ -680,17 +680,6 @@ def step_impl(context): context.test.acquire_container(context=context, name="postgresql-server", engine="postgresql-server") -# OPC UA -@given("an OPC UA server is set up") -def step_impl(context): - context.test.acquire_container(context=context, name="opcua-server", engine="opcua-server") - - -@given("an OPC UA server is set up with access control") -def step_impl(context): - context.test.acquire_container(context=context, name="opcua-server", engine="opcua-server", command=["/opt/open62541/examples/access_control_server"]) - - @when("the MiNiFi instance starts up") @when("both instances start up") @when("all instances start up") @@ -917,7 +906,7 @@ def step_impl(context, blob_and_snapshot_count, timeout_seconds): # SQL @then("the query \"{query}\" returns {number_of_rows:d} rows in less than {timeout_seconds:d} seconds on the PostgreSQL server") -def step_impl(context, query, number_of_rows, timeout_seconds): +def step_impl(context, query: str, number_of_rows: int, timeout_seconds: int): context.test.check_query_results(context.test.get_container_name_with_postfix("postgresql-server"), query, number_of_rows, timeout_seconds) @@ -943,11 +932,6 @@ def step_impl(context, regex, duration): context.test.check_minifi_log_matches_regex(regex, humanfriendly.parse_timespan(duration)) -@then("the OPC UA server logs contain the following message: \"{log_message}\" in less than {duration}") -def step_impl(context, log_message, duration): - context.test.check_container_log_contents("opcua-server", log_message, humanfriendly.parse_timespan(duration)) - - # MQTT @then("the MQTT broker has a log line matching \"{log_pattern}\"") def step_impl(context, log_pattern): diff --git a/extensions/aws/tests/features/steps/steps.py b/extensions/aws/tests/features/steps/steps.py index e1d6c4cd7..6f8d1d757 100644 --- a/extensions/aws/tests/features/steps/steps.py +++ b/extensions/aws/tests/features/steps/steps.py @@ -46,32 +46,32 @@ def step_impl(context: MinifiTestContext, processor_name: str): processor.add_property('Proxy Username', '') processor.add_property('Proxy Password', '') - context.minifi_container.flow_definition.add_processor(processor) + context.get_or_create_default_minifi_container().flow_definition.add_processor(processor) @step('a s3 server is set up in correspondence with the {processor_name}') @step('an s3 server is set up in correspondence with the {processor_name}') def step_impl(context: MinifiTestContext, processor_name: str): - context.containers.append(S3ServerContainer(context)) + context.containers["s3-server"] = S3ServerContainer(context) @step('the object on the s3 server is "{object_data}"') def step_impl(context: MinifiTestContext, object_data: str): - s3_server_container = context.containers[0] + s3_server_container = context.containers["s3-server"] assert isinstance(s3_server_container, S3ServerContainer) assert s3_server_container.check_s3_server_object_data(object_data) @step('the object content type on the s3 server is "{content_type}" and the object metadata matches use metadata') def step_impl(context: MinifiTestContext, content_type: str): - s3_server_container = context.containers[0] + s3_server_container = context.containers["s3-server"] assert isinstance(s3_server_container, S3ServerContainer) assert s3_server_container.check_s3_server_object_metadata(content_type) @step("the object bucket on the s3 server is empty in less than 10 seconds") def step_impl(context): - s3_server_container = context.containers[0] + s3_server_container = context.containers["s3-server"] assert isinstance(s3_server_container, S3ServerContainer) assert wait_for_condition( condition=lambda: s3_server_container.is_s3_bucket_empty(), @@ -80,7 +80,7 @@ def step_impl(context): @step("the object on the s3 server is present and matches the original hash") def step_impl(context): - s3_server_container = context.containers[0] + s3_server_container = context.containers["s3-server"] assert isinstance(s3_server_container, S3ServerContainer) assert s3_server_container.check_s3_server_object_hash(context.original_hash) @@ -98,5 +98,5 @@ def step_impl(context): content = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(size)) new_dir = Directory("/tmp/input") new_dir.files["input.txt"] = content - context.minifi_container.dirs.append(new_dir) + context.get_or_create_default_minifi_container().dirs.append(new_dir) context.original_hash = computeMD5hash(content) diff --git a/extensions/azure/tests/features/steps/steps.py b/extensions/azure/tests/features/steps/steps.py index 38721f99b..3331d7e71 100644 --- a/extensions/azure/tests/features/steps/steps.py +++ b/extensions/azure/tests/features/steps/steps.py @@ -37,41 +37,39 @@ def step_impl(context: MinifiTestContext, processor_name: str): f'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={hostname}:10000/devstoreaccount1;QueueEndpoint={hostname}:10001/devstoreaccount1;') processor.add_property('Blob', 'test-blob') processor.add_property('Create Container', 'true') - context.minifi_container.flow_definition.add_processor(processor) + context.get_or_create_default_minifi_container().flow_definition.add_processor(processor) @step("an Azure storage server is set up") def step_impl(context): - context.containers.append(AzureServerContainer(context)) - azure_server_container = context.containers[0] - assert isinstance(azure_server_container, AzureServerContainer) - assert azure_server_container.deploy() + context.containers["azure-storage-server"] = AzureServerContainer(context) + assert context.containers["azure-storage-server"].deploy() @then('the object on the Azure storage server is "{object_data}"') def step_impl(context: MinifiTestContext, object_data: str): - azure_server_container = context.containers[0] + azure_server_container = context.containers["azure-storage-server"] assert isinstance(azure_server_container, AzureServerContainer) assert azure_server_container.check_azure_storage_server_data(object_data) @step('test blob "{blob_name}" with the content "{data}" is created on Azure blob storage') def step_impl(context: MinifiTestContext, blob_name: str, data: str): - azure_server_container = context.containers[0] + azure_server_container = context.containers["azure-storage-server"] assert isinstance(azure_server_container, AzureServerContainer) assert azure_server_container.add_test_blob(blob_name, content=data) @step('test blob "{blob_name}" is created on Azure blob storage') def step_impl(context: MinifiTestContext, blob_name: str): - azure_server_container = context.containers[0] + azure_server_container = context.containers["azure-storage-server"] assert isinstance(azure_server_container, AzureServerContainer) assert azure_server_container.add_test_blob(blob_name) @step('test blob "{blob_name}" is created on Azure blob storage with a snapshot') def step_impl(context: MinifiTestContext, blob_name: str): - azure_server_container = context.containers[0] + azure_server_container = context.containers["azure-storage-server"] assert isinstance(azure_server_container, AzureServerContainer) assert azure_server_container.add_test_blob(blob_name, with_snapshot=True) @@ -79,20 +77,22 @@ def step_impl(context: MinifiTestContext, blob_name: str): @then("the Azure blob storage becomes empty in {timeout_str}") def step_impl(context: MinifiTestContext, timeout_str: str): timeout_in_seconds = humanfriendly.parse_timespan(timeout_str) - azure_server_container = context.containers[0] + azure_server_container = context.containers["azure-storage-server"] assert isinstance(azure_server_container, AzureServerContainer) - assert wait_for_condition(condition=lambda: azure_server_container.check_azure_blob_storage_is_empty(), - timeout_seconds=timeout_in_seconds, - bail_condition=lambda: context.minifi_container.exited, - context=context) + assert wait_for_condition( + condition=lambda: azure_server_container.check_azure_blob_storage_is_empty(), + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: azure_server_container.exited, + context=context) @then("the blob and snapshot count becomes 1 in {timeout_str}") def step_impl(context: MinifiTestContext, timeout_str: str): timeout_in_seconds = humanfriendly.parse_timespan(timeout_str) - azure_server_container = context.containers[0] + azure_server_container = context.containers["azure-storage-server"] assert isinstance(azure_server_container, AzureServerContainer) - assert wait_for_condition(condition=lambda: azure_server_container.check_azure_blob_and_snapshot_count(1), - timeout_seconds=timeout_in_seconds, - bail_condition=lambda: context.minifi_container.exited, - context=context) + assert wait_for_condition( + condition=lambda: azure_server_container.check_azure_blob_and_snapshot_count(1), + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: azure_server_container.exited, + context=context) diff --git a/extensions/opc/tests/features/environment.py b/extensions/opc/tests/features/environment.py new file mode 100644 index 000000000..a02cce4c3 --- /dev/null +++ b/extensions/opc/tests/features/environment.py @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from minifi_test_framework.core.hooks import common_before_scenario +from minifi_test_framework.core.hooks import common_after_scenario + + +def before_scenario(context, scenario): + common_before_scenario(context, scenario) + context.resource_dir = os.path.join(os.path.dirname(__file__), 'resources') + + +def after_scenario(context, scenario): + common_after_scenario(context, scenario) diff --git a/docker/test/integration/features/opcua.feature b/extensions/opc/tests/features/opcua.feature similarity index 69% rename from docker/test/integration/features/opcua.feature rename to extensions/opc/tests/features/opcua.feature index 8f561f356..064e32d32 100644 --- a/docker/test/integration/features/opcua.feature +++ b/extensions/opc/tests/features/opcua.feature @@ -19,16 +19,16 @@ Feature: Putting and fetching data to OPC UA server As a user of MiNiFi I need to have PutOPCProcessor and FetchOPCProcessor - Background: - Given the content of "/tmp/output" is monitored - Scenario Outline: Create and fetch data from an OPC UA node Given a GetFile processor with the "Input Directory" property set to "/tmp/input" in the "create-opc-ua-node" flow - And a file with the content "<Value>" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "<Value>" in the "create-opc-ua-node" flow And a PutOPCProcessor processor in the "create-opc-ua-node" flow + And PutOPCProcessor is EVENT_DRIVEN in the "create-opc-ua-node" flow And a FetchOPCProcessor processor in the "fetch-opc-ua-node" flow And a PutFile processor with the "Directory" property set to "/tmp/output" in the "fetch-opc-ua-node" flow - And these processor properties are set: + And PutFile's success relationship is auto-terminated in the "fetch-opc-ua-node" flow + And PutFile is EVENT_DRIVEN in the "fetch-opc-ua-node" flow + And these processor properties are set in the "create-opc-ua-node" flow | processor name | property name | property value | | PutOPCProcessor | Parent node ID | 85 | | PutOPCProcessor | Parent node ID type | Int | @@ -36,21 +36,23 @@ Feature: Putting and fetching data to OPC UA server | PutOPCProcessor | Target node ID type | Int | | PutOPCProcessor | Target node namespace index | 1 | | PutOPCProcessor | Value type | <Value Type> | - | PutOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${feature_id}:4840/ | + | PutOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${scenario_id}:4840/ | | PutOPCProcessor | Target node browse name | testnodename | + And these processor properties are set in the "fetch-opc-ua-node" flow + | processor name | property name | property value | | FetchOPCProcessor | Node ID | 9999 | | FetchOPCProcessor | Node ID type | Int | | FetchOPCProcessor | Namespace index | 1 | - | FetchOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${feature_id}:4840/ | + | FetchOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${scenario_id}:4840/ | | FetchOPCProcessor | Max depth | 1 | - And the "success" relationship of the GetFile processor is connected to the PutOPCProcessor - And the "success" relationship of the FetchOPCProcessor processor is connected to the PutFile + And in the "create-opc-ua-node" flow the "success" relationship of the GetFile processor is connected to the PutOPCProcessor + And in the "fetch-opc-ua-node" flow the "success" relationship of the FetchOPCProcessor processor is connected to the PutFile And an OPC UA server is set up When all instances start up - Then at least one flowfile with the content "<Value>" is placed in the monitored directory in less than 60 seconds + Then in the "fetch-opc-ua-node" container at least one file with the content "<Value>" is placed in the "/tmp/output" directory in less than 60 seconds Examples: Topic names and formats to test | Value Type | Value | @@ -61,11 +63,14 @@ Feature: Putting and fetching data to OPC UA server Scenario Outline: Update and fetch data from an OPC UA node Given a GetFile processor with the "Input Directory" property set to "/tmp/input" in the "update-opc-ua-node" flow - And a file with the content "<Value>" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "<Value>" in the "update-opc-ua-node" flow And a PutOPCProcessor processor in the "update-opc-ua-node" flow + And PutOPCProcessor is EVENT_DRIVEN in the "update-opc-ua-node" flow And a FetchOPCProcessor processor in the "fetch-opc-ua-node" flow And a PutFile processor with the "Directory" property set to "/tmp/output" in the "fetch-opc-ua-node" flow - And these processor properties are set: + And PutFile's success relationship is auto-terminated in the "fetch-opc-ua-node" flow + And PutFile is EVENT_DRIVEN in the "fetch-opc-ua-node" flow + And these processor properties are set in the "update-opc-ua-node" flow | processor name | property name | property value | | PutOPCProcessor | Parent node ID | 85 | | PutOPCProcessor | Parent node ID type | Int | @@ -73,21 +78,23 @@ Feature: Putting and fetching data to OPC UA server | PutOPCProcessor | Target node ID type | <Node ID Type> | | PutOPCProcessor | Target node namespace index | 1 | | PutOPCProcessor | Value type | <Value Type> | - | PutOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${feature_id}:4840/ | + | PutOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${scenario_id}:4840/ | | PutOPCProcessor | Target node browse name | testnodename | + And these processor properties are set in the "fetch-opc-ua-node" flow + | processor name | property name | property value | | FetchOPCProcessor | Node ID | <Node ID> | | FetchOPCProcessor | Node ID type | <Node ID Type> | | FetchOPCProcessor | Namespace index | 1 | - | FetchOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${feature_id}:4840/ | + | FetchOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${scenario_id}:4840/ | | FetchOPCProcessor | Max depth | 1 | - And the "success" relationship of the GetFile processor is connected to the PutOPCProcessor - And the "success" relationship of the FetchOPCProcessor processor is connected to the PutFile + And in the "update-opc-ua-node" flow the "success" relationship of the GetFile processor is connected to the PutOPCProcessor + And in the "fetch-opc-ua-node" flow the "success" relationship of the FetchOPCProcessor processor is connected to the PutFile And an OPC UA server is set up When all instances start up - Then at least one flowfile with the content "<Value>" is placed in the monitored directory in less than 60 seconds + Then in the "fetch-opc-ua-node" container at least one file with the content "<Value>" is placed in the "/tmp/output" directory in less than 60 seconds # Node ids starting from 51000 are pre-defined demo node ids in the test server application (server_ctt) of the open62541 docker image. There is one nodeid defined # for each type supported by OPC UA. These demo nodes can be used for testing purposes. "the.answer" is also a pre-defined string id for the same testing purposes. @@ -101,11 +108,18 @@ Feature: Putting and fetching data to OPC UA server Scenario: Create and fetch data from an OPC UA node through secure connection Given a GetFile processor with the "Input Directory" property set to "/tmp/input" in the "create-opc-ua-node" flow - And a file with the content "Test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "Test" in the "create-opc-ua-node" flow And a PutOPCProcessor processor in the "create-opc-ua-node" flow + And PutOPCProcessor is EVENT_DRIVEN in the "create-opc-ua-node" flow And a FetchOPCProcessor processor in the "fetch-opc-ua-node" flow And a PutFile processor with the "Directory" property set to "/tmp/output" in the "fetch-opc-ua-node" flow - And these processor properties are set: + And PutFile's success relationship is auto-terminated in the "fetch-opc-ua-node" flow + And PutFile is EVENT_DRIVEN in the "fetch-opc-ua-node" flow + And a host resource file "opcua_client_cert.der" is bound to the "/tmp/resources/opcua/opcua_client_cert.der" path in the MiNiFi container "create-opc-ua-node" + And a host resource file "opcua_client_key.der" is bound to the "/tmp/resources/opcua/opcua_client_key.der" path in the MiNiFi container "create-opc-ua-node" + And a host resource file "opcua_client_cert.der" is bound to the "/tmp/resources/opcua/opcua_client_cert.der" path in the MiNiFi container "fetch-opc-ua-node" + And a host resource file "opcua_client_key.der" is bound to the "/tmp/resources/opcua/opcua_client_key.der" path in the MiNiFi container "fetch-opc-ua-node" + And these processor properties are set in the "create-opc-ua-node" flow | processor name | property name | property value | | PutOPCProcessor | Parent node ID | 85 | | PutOPCProcessor | Parent node ID type | Int | @@ -113,38 +127,44 @@ Feature: Putting and fetching data to OPC UA server | PutOPCProcessor | Target node ID type | Int | | PutOPCProcessor | Target node namespace index | 1 | | PutOPCProcessor | Value type | String | - | PutOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${feature_id}:4840/ | + | PutOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${scenario_id}:4840/ | | PutOPCProcessor | Target node browse name | testnodename | | PutOPCProcessor | Certificate path | /tmp/resources/opcua/opcua_client_cert.der | | PutOPCProcessor | Key path | /tmp/resources/opcua/opcua_client_key.der | | PutOPCProcessor | Trusted server certificate path | /tmp/resources/opcua/opcua_client_cert.der | | PutOPCProcessor | Application URI | urn:open62541.server.application | + And these processor properties are set in the "fetch-opc-ua-node" flow + | processor name | property name | property value | | FetchOPCProcessor | Node ID | 9999 | | FetchOPCProcessor | Node ID type | Int | | FetchOPCProcessor | Namespace index | 1 | - | FetchOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${feature_id}:4840/ | + | FetchOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${scenario_id}:4840/ | | FetchOPCProcessor | Max depth | 1 | | FetchOPCProcessor | Certificate path | /tmp/resources/opcua/opcua_client_cert.der | | FetchOPCProcessor | Key path | /tmp/resources/opcua/opcua_client_key.der | | FetchOPCProcessor | Trusted server certificate path | /tmp/resources/opcua/opcua_client_cert.der | | FetchOPCProcessor | Application URI | urn:open62541.server.application | - And the "success" relationship of the GetFile processor is connected to the PutOPCProcessor - And the "success" relationship of the FetchOPCProcessor processor is connected to the PutFile + And in the "create-opc-ua-node" flow the "success" relationship of the GetFile processor is connected to the PutOPCProcessor + And in the "fetch-opc-ua-node" flow the "success" relationship of the FetchOPCProcessor processor is connected to the PutFile And an OPC UA server is set up When all instances start up - Then at least one flowfile with the content "Test" is placed in the monitored directory in less than 60 seconds + + Then in the "fetch-opc-ua-node" container at least one file with the content "Test" is placed in the "/tmp/output" directory in less than 60 seconds And the OPC UA server logs contain the following message: "SecureChannel opened with SecurityPolicy http://opcfoundation.org/UA/SecurityPolicy#Aes128_Sha256_RsaOaep"; in less than 5 seconds Scenario: Create and fetch data from an OPC UA node through username and password authenticated connection Given a GetFile processor with the "Input Directory" property set to "/tmp/input" in the "create-opc-ua-node" flow - And a file with the content "Test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "Test" in the "create-opc-ua-node" flow And a PutOPCProcessor processor in the "create-opc-ua-node" flow + And PutOPCProcessor is EVENT_DRIVEN in the "create-opc-ua-node" flow And a FetchOPCProcessor processor in the "fetch-opc-ua-node" flow And a PutFile processor with the "Directory" property set to "/tmp/output" in the "fetch-opc-ua-node" flow - And these processor properties are set: + And PutFile's success relationship is auto-terminated in the "fetch-opc-ua-node" flow + And PutFile is EVENT_DRIVEN in the "fetch-opc-ua-node" flow + And these processor properties are set in the "create-opc-ua-node" flow | processor name | property name | property value | | PutOPCProcessor | Parent node ID | 85 | | PutOPCProcessor | Parent node ID type | Int | @@ -152,22 +172,24 @@ Feature: Putting and fetching data to OPC UA server | PutOPCProcessor | Target node ID type | Int | | PutOPCProcessor | Target node namespace index | 1 | | PutOPCProcessor | Value type | String | - | PutOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${feature_id}:4840/ | + | PutOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${scenario_id}:4840/ | | PutOPCProcessor | Target node browse name | testnodename | | PutOPCProcessor | Username | peter | | PutOPCProcessor | Password | peter123 | + And these processor properties are set in the "fetch-opc-ua-node" flow + | processor name | property name | property value | | FetchOPCProcessor | Node ID | 9999 | | FetchOPCProcessor | Node ID type | Int | | FetchOPCProcessor | Namespace index | 1 | - | FetchOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${feature_id}:4840/ | + | FetchOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${scenario_id}:4840/ | | FetchOPCProcessor | Max depth | 1 | | FetchOPCProcessor | Username | peter | | FetchOPCProcessor | Password | peter123 | - And the "success" relationship of the GetFile processor is connected to the PutOPCProcessor - And the "success" relationship of the FetchOPCProcessor processor is connected to the PutFile + And in the "create-opc-ua-node" flow the "success" relationship of the GetFile processor is connected to the PutOPCProcessor + And in the "fetch-opc-ua-node" flow the "success" relationship of the FetchOPCProcessor processor is connected to the PutFile And an OPC UA server is set up with access control When all instances start up - Then at least one flowfile with the content "Test" is placed in the monitored directory in less than 60 seconds + Then in the "fetch-opc-ua-node" container at least one file with the content "Test" is placed in the "/tmp/output" directory in less than 60 seconds diff --git a/docker/test/integration/resources/opcua/opcua_client_cert.der b/extensions/opc/tests/features/resources/opcua_client_cert.der similarity index 100% rename from docker/test/integration/resources/opcua/opcua_client_cert.der rename to extensions/opc/tests/features/resources/opcua_client_cert.der diff --git a/docker/test/integration/resources/opcua/opcua_client_key.der b/extensions/opc/tests/features/resources/opcua_client_key.der similarity index 100% rename from docker/test/integration/resources/opcua/opcua_client_key.der rename to extensions/opc/tests/features/resources/opcua_client_key.der diff --git a/docker/test/integration/cluster/containers/OPCUAServerContainer.py b/extensions/opc/tests/features/steps/opc_ua_server_container.py similarity index 52% rename from docker/test/integration/cluster/containers/OPCUAServerContainer.py rename to extensions/opc/tests/features/steps/opc_ua_server_container.py index fefb8759c..4973b9046 100644 --- a/docker/test/integration/cluster/containers/OPCUAServerContainer.py +++ b/extensions/opc/tests/features/steps/opc_ua_server_container.py @@ -13,27 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. - -import logging -from .Container import Container +from typing import List, Optional +from minifi_test_framework.containers.container import Container +from minifi_test_framework.core.helpers import wait_for_condition +from minifi_test_framework.core.minifi_test_context import MinifiTestContext class OPCUAServerContainer(Container): - def __init__(self, feature_context, name, vols, network, image_store, command=None): - super().__init__(feature_context, name, 'opcua-server', vols, network, image_store, command) - - def get_startup_finished_log_entry(self): - return "New DiscoveryUrl added: opc.tcp://" + def __init__(self, test_context: MinifiTestContext, command: Optional[List[str]] = None): + super().__init__("lordgamez/open62541:1.4.10", f"opcua-server-{test_context.scenario_id}", test_context.network, command=command) def deploy(self): - if not self.set_deployed(): - return - - logging.info('Creating and running OPC UA server docker container...') - self.client.containers.run( - "lordgamez/open62541:1.4.10", - detach=True, - name=self.name, - network=self.network.name, - entrypoint=self.command) - logging.info('Added container \'%s\'', self.name) + super().deploy() + finished_str = "New DiscoveryUrl added: opc.tcp://" + return wait_for_condition( + condition=lambda: finished_str in self.get_logs(), + timeout_seconds=15, + bail_condition=lambda: self.exited, + context=None) diff --git a/extensions/opc/tests/features/steps/steps.py b/extensions/opc/tests/features/steps/steps.py new file mode 100644 index 000000000..f9db97c8a --- /dev/null +++ b/extensions/opc/tests/features/steps/steps.py @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import humanfriendly +from behave import step, then + +from minifi_test_framework.steps import checking_steps # noqa: F401 +from minifi_test_framework.steps import configuration_steps # noqa: F401 +from minifi_test_framework.steps import core_steps # noqa: F401 +from minifi_test_framework.steps import flow_building_steps # noqa: F401 +from minifi_test_framework.core.minifi_test_context import MinifiTestContext +from minifi_test_framework.core.helpers import wait_for_condition +from opc_ua_server_container import OPCUAServerContainer + + +@step("an OPC UA server is set up") +def step_impl(context: MinifiTestContext): + context.containers["opcua-server"] = OPCUAServerContainer(context) + + +@step("an OPC UA server is set up with access control") +def step_impl(context: MinifiTestContext): + context.containers["opcua-server-access"] = OPCUAServerContainer(context, command=["/opt/open62541/examples/access_control_server"]) + + +@then("the OPC UA server logs contain the following message: \"{log_message}\" in less than {duration}") +def step_impl(context, log_message, duration): + timeout_seconds = humanfriendly.parse_timespan(duration) + opcua_container = context.containers["opcua-server"] + assert isinstance(opcua_container, OPCUAServerContainer) + assert wait_for_condition( + condition=lambda: log_message in opcua_container.get_logs(), + timeout_seconds=timeout_seconds, + bail_condition=lambda: opcua_container.exited, + context=context) diff --git a/extensions/sql/tests/features/steps/steps.py b/extensions/sql/tests/features/steps/steps.py index 60a2059c3..db2f555e4 100644 --- a/extensions/sql/tests/features/steps/steps.py +++ b/extensions/sql/tests/features/steps/steps.py @@ -33,20 +33,20 @@ def step_impl(context: MinifiTestContext, processor_name: str, service_name: str odb_service = ControllerService(class_name="ODBCService", service_name=service_name) postgres_server_hostname = f"postgres-server-{context.scenario_id}" odb_service.add_property("Connection String", f"Driver={{PostgreSQL ANSI}};Server={postgres_server_hostname};Port=5432;Database=postgres;Uid=postgres;Pwd=password;") - context.minifi_container.flow_definition.controller_services.append(odb_service) - processor = context.minifi_container.flow_definition.get_processor(processor_name) + context.get_or_create_default_minifi_container().flow_definition.controller_services.append(odb_service) + processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name) processor.add_property("DB Controller Service", "ODBCService") @step("a PostgreSQL server is set up") def step_impl(context): - context.containers.append(PostgresContainer(context)) + context.containers["postgres-server"] = PostgresContainer(context) -@then('the query "{query}" returns {rows} rows in less than {timeout_str} on the PostgreSQL server') -def step_impl(context, query: str, rows: str, timeout_str: str): +@then('the query "{query}" returns {rows:d} rows in less than {timeout_str} on the PostgreSQL server') +def step_impl(context, query: str, rows: int, timeout_str: str): timeout_seconds = humanfriendly.parse_timespan(timeout_str) - postgres_container = context.containers[0] + postgres_container = context.containers["postgres-server"] assert isinstance(postgres_container, PostgresContainer) assert wait_for_condition( condition=lambda: postgres_container.check_query_results(query, int(rows)), diff --git a/extensions/standard-processors/tests/features/steps/steps.py b/extensions/standard-processors/tests/features/steps/steps.py index df76b292b..a53631e2e 100644 --- a/extensions/standard-processors/tests/features/steps/steps.py +++ b/extensions/standard-processors/tests/features/steps/steps.py @@ -27,9 +27,9 @@ from syslog_container import SyslogContainer @step("a Syslog client with TCP protocol is setup to send logs to minifi") def step_impl(context: MinifiTestContext): - context.containers.append(SyslogContainer("tcp", context)) + context.containers["syslog-tcp"] = SyslogContainer("tcp", context) @step("a Syslog client with UDP protocol is setup to send logs to minifi") def step_impl(context): - context.containers.append(SyslogContainer("udp", context)) + context.containers["syslog-udp"] = SyslogContainer("udp", context) diff --git a/extensions/standard-processors/tests/features/steps/syslog_container.py b/extensions/standard-processors/tests/features/steps/syslog_container.py index 15a3c9551..f2da956f5 100644 --- a/extensions/standard-processors/tests/features/steps/syslog_container.py +++ b/extensions/standard-processors/tests/features/steps/syslog_container.py @@ -21,4 +21,4 @@ from minifi_test_framework.containers.container import Container class SyslogContainer(Container): def __init__(self, protocol, context): super(SyslogContainer, self).__init__("ubuntu:24.04", f"syslog-{protocol}-{context.scenario_id}", context.network) - self.command = f'/bin/bash -c "echo Syslog {protocol} client started; while true; do logger --{protocol} -n minifi-{context.scenario_id} -P 514 sample_log; sleep 1; done"' + self.command = f'/bin/bash -c "echo Syslog {protocol} client started; while true; do logger --{protocol} -n minifi-primary-{context.scenario_id} -P 514 sample_log; sleep 1; done"'
