This is an automated email from the ASF dual-hosted git repository. lordgamez pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit d24d3240030f25655710bd056b3e9c38e12c2299 Author: Gabor Gyimesi <[email protected]> AuthorDate: Tue Jan 20 14:13:54 2026 +0100 MINIFICPP-2683 Move C2 docker tests to modular docker tests Signed-off-by: Gabor Gyimesi <[email protected]> This closes #2089 --- .../containers/minifi_container.py | 10 ++ .../minifi_test_framework/steps/checking_steps.py | 12 +-- .../src/minifi_test_framework/steps/core_steps.py | 11 +- docker/test/integration/cluster/ContainerStore.py | 28 ------ .../cluster/containers/MinifiC2ServerContainer.py | 112 --------------------- .../cluster/containers/MinifiContainer.py | 31 +----- .../features/MiNiFi_integration_test_driver.py | 5 - docker/test/integration/features/steps/steps.py | 32 ------ .../tests}/features/minifi_c2_server.feature | 15 ++- .../features/steps/minifi_c2_server_container.py | 76 ++++++++++++++ .../resources/minifi-c2-server/authorizations.yaml | 0 .../resources/minifi-c2-server/c2.properties | 0 .../steps}/resources/minifi-c2-server/config.yml | 0 .../tests/features/steps/steps.py | 54 +++++++++- 14 files changed, 161 insertions(+), 225 deletions(-) diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_container.py index be72155e9..8d2b5f992 100644 --- a/behave_framework/src/minifi_test_framework/containers/minifi_container.py +++ b/behave_framework/src/minifi_test_framework/containers/minifi_container.py @@ -35,6 +35,7 @@ class MinifiContainer(Container): self.flow_definition = MinifiFlowDefinition() self.properties: dict[str, str] = {} self.log_properties: dict[str, str] = {} + self.scenario_id = test_context.scenario_id minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=self.container_name, ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert))) @@ -96,6 +97,15 @@ class MinifiContainer(Container): def enable_openssl_fips_mode(self): self.properties["nifi.openssl.fips.support.enable"] = "true" + def fetch_flow_config_from_flow_url(self): + self.properties["nifi.c2.flow.url"] = f"http://minifi-c2-server-{self.scenario_id}:10090/c2/config?class=minifi-test-class"; + + def set_up_ssl_properties(self): + self.properties["nifi.remote.input.secure"] = "true" + self.properties["nifi.security.client.certificate"] = "/tmp/resources/minifi_client.crt" + self.properties["nifi.security.client.private.key"] = "/tmp/resources/minifi_client.key" + self.properties["nifi.security.client.ca.certificate"] = "/tmp/resources/root_ca.crt" + def _fill_default_properties(self): if self.is_fhs: self.properties["nifi.flow.configuration.file"] = "/etc/nifi-minifi-cpp/config.yml" diff --git a/behave_framework/src/minifi_test_framework/steps/checking_steps.py b/behave_framework/src/minifi_test_framework/steps/checking_steps.py index 8a2eafae1..ef5b1a338 100644 --- a/behave_framework/src/minifi_test_framework/steps/checking_steps.py +++ b/behave_framework/src/minifi_test_framework/steps/checking_steps.py @@ -196,7 +196,7 @@ def step_impl(context: MinifiTestContext, content: str, directory: str, duration @then('after a wait of {duration}, at least {lower_bound:d} and at most {upper_bound:d} files are produced and placed in the "{directory}" directory') -def step_impl(context, lower_bound, upper_bound, duration, directory): +def step_impl(context: MinifiTestContext, lower_bound: int, upper_bound: int, duration: str, directory: str): duration_seconds = humanfriendly.parse_timespan(duration) assert check_condition_after_wait(condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files(directory) >= lower_bound and context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files(directory) <= upper_bound, @@ -204,7 +204,7 @@ def step_impl(context, lower_bound, upper_bound, duration, directory): @then('exactly these files are in the "{directory}" directory in less than {duration}: "{contents}"') -def step_impl(context, directory, duration, contents): +def step_impl(context: MinifiTestContext, directory: str, duration: str, contents: str): if not contents: context.execute_steps(f'then no files are placed in the "{directory}" directory in {duration} of running time') return @@ -221,7 +221,7 @@ def step_impl(context, directory, duration): @then("at least one empty file is placed in the \"{directory}\" directory in less than {duration}") -def step_impl(context, directory, duration): +def step_impl(context: MinifiTestContext, directory: str, duration: str): timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].directory_contains_empty_file(directory), @@ -229,7 +229,7 @@ def step_impl(context, directory, duration): @then("in the \"{container_name}\" container at least one empty file is placed in the \"{directory}\" directory in less than {duration}") -def step_impl(context, container_name, directory, duration): +def step_impl(context: MinifiTestContext, container_name: str, directory: str, duration: str): timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( condition=lambda: context.containers[container_name].directory_contains_empty_file(directory), @@ -237,7 +237,7 @@ def step_impl(context, container_name, directory, duration): @then("in the \"{container_name}\" container at least one file with minimum size of \"{size}\" is placed in the \"{directory}\" directory in less than {duration}") -def step_impl(context, container_name: str, directory: str, size: str, duration: str): +def step_impl(context: MinifiTestContext, container_name: str, directory: str, size: str, duration: str): timeout_in_seconds = humanfriendly.parse_timespan(duration) size_in_bytes = humanfriendly.parse_size(size) assert wait_for_condition( @@ -246,6 +246,6 @@ def step_impl(context, container_name: str, directory: str, size: str, duration: @then("at least one file with minimum size of \"{size}\" is placed in the \"{directory}\" directory in less than {duration}") -def step_impl(context, directory: str, size: str, duration: str): +def step_impl(context: MinifiTestContext, directory: str, size: str, duration: str): context.execute_steps( f'Then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container at least one file with minimum size of "{size}" is placed in the "{directory}" directory in less than {duration}') diff --git a/behave_framework/src/minifi_test_framework/steps/core_steps.py b/behave_framework/src/minifi_test_framework/steps/core_steps.py index 836b58834..6b488587c 100644 --- a/behave_framework/src/minifi_test_framework/steps/core_steps.py +++ b/behave_framework/src/minifi_test_framework/steps/core_steps.py @@ -100,15 +100,20 @@ def step_impl(context: MinifiTestContext): @given("OpenSSL FIPS mode is enabled in MiNiFi") -def step_impl(context): +def step_impl(context: MinifiTestContext): context.get_or_create_default_minifi_container().enable_openssl_fips_mode() @step("the http proxy server is set up") -def step_impl(context): +def step_impl(context: MinifiTestContext): context.containers["http-proxy"] = HttpProxy(context) @step("a NiFi container is set up") -def step_impl(context): +def step_impl(context: MinifiTestContext): context.containers["nifi"] = NifiContainer(context) + + +@given("flow configuration path is set up in flow url property") +def step_impl(context: MinifiTestContext): + context.get_or_create_default_minifi_container().fetch_flow_config_from_flow_url() diff --git a/docker/test/integration/cluster/ContainerStore.py b/docker/test/integration/cluster/ContainerStore.py index 084bbc45d..f139acd21 100644 --- a/docker/test/integration/cluster/ContainerStore.py +++ b/docker/test/integration/cluster/ContainerStore.py @@ -26,7 +26,6 @@ from .containers.SyslogUdpClientContainer import SyslogUdpClientContainer from .containers.SyslogTcpClientContainer import SyslogTcpClientContainer from .containers.MinifiAsPodInKubernetesCluster import MinifiAsPodInKubernetesCluster from .containers.PrometheusContainer import PrometheusContainer -from .containers.MinifiC2ServerContainer import MinifiC2ServerContainer from .FeatureContext import FeatureContext @@ -172,24 +171,6 @@ class ContainerStore: image_store=self.image_store, command=command, ssl=True)) - elif engine == "minifi-c2-server": - return self.containers.setdefault(container_name, - MinifiC2ServerContainer(feature_context=feature_context, - name=container_name, - vols=self.vols, - network=self.network, - image_store=self.image_store, - command=command, - ssl=False)) - elif engine == "minifi-c2-server-ssl": - return self.containers.setdefault(container_name, - MinifiC2ServerContainer(feature_context=feature_context, - name=container_name, - vols=self.vols, - network=self.network, - image_store=self.image_store, - command=command, - ssl=True)) else: raise Exception('invalid flow engine: \'%s\'' % engine) @@ -231,15 +212,6 @@ class ContainerStore: def enable_provenance_repository_in_minifi(self): self.minifi_options.enable_provenance = True - def enable_c2_in_minifi(self): - self.minifi_options.enable_c2 = True - - def enable_c2_with_ssl_in_minifi(self): - self.minifi_options.enable_c2_with_ssl = True - - def fetch_flow_config_from_c2_url_in_minifi(self): - self.minifi_options.use_flow_config_from_url = True - def set_ssl_context_properties_in_minifi(self): self.minifi_options.set_ssl_context_properties = True diff --git a/docker/test/integration/cluster/containers/MinifiC2ServerContainer.py b/docker/test/integration/cluster/containers/MinifiC2ServerContainer.py deleted file mode 100644 index 9be1cdeb1..000000000 --- a/docker/test/integration/cluster/containers/MinifiC2ServerContainer.py +++ /dev/null @@ -1,112 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import tempfile - -import docker.types -import jks - -from .Container import Container -from OpenSSL import crypto -from cryptography.hazmat.primitives.serialization import pkcs12, BestAvailableEncryption, load_pem_private_key -from cryptography import x509 -from ssl_utils.SSL_cert_utils import make_server_cert - - -class MinifiC2ServerContainer(Container): - def __init__(self, feature_context, name, vols, network, image_store, command=None, ssl=False): - engine = "minifi-c2-server-ssl" if ssl else "minifi-c2-server" - super().__init__(feature_context, name, engine, vols, network, image_store, command) - self.ssl = ssl - if ssl: - c2_cert, c2_key = make_server_cert(f"minifi-c2-server-{feature_context.id}", feature_context.root_ca_cert, feature_context.root_ca_key) - pke = jks.PrivateKeyEntry.new('c2-server-cert', [crypto.dump_certificate(crypto.FILETYPE_ASN1, c2_cert)], crypto.dump_privatekey(crypto.FILETYPE_ASN1, c2_key), 'rsa_raw') - server_keystore = jks.KeyStore.new('jks', [pke]) - self.server_keystore_file = tempfile.NamedTemporaryFile(delete=False) - server_keystore.save(self.server_keystore_file.name, 'abcdefgh') - os.chmod(self.server_keystore_file.name, 0o644) - - self.server_truststore_file = tempfile.NamedTemporaryFile(delete=False) - private_key_pem = crypto.dump_privatekey(crypto.FILETYPE_PEM, feature_context.root_ca_key) - private_key = load_pem_private_key(private_key_pem, password=None) - certificate_pem = crypto.dump_certificate(crypto.FILETYPE_PEM, feature_context.root_ca_cert) - certificate = x509.load_pem_x509_certificate(certificate_pem) - pkcs12_data = pkcs12.serialize_key_and_certificates( - name=None, - key=private_key, - cert=certificate, - cas=None, - encryption_algorithm=BestAvailableEncryption(b'abcdefgh') - ) - self.server_truststore_file.write(pkcs12_data) - self.server_truststore_file.close() - os.chmod(self.server_truststore_file.name, 0o644) - - self.authorities_file = tempfile.NamedTemporaryFile(delete=False) - authorities_file_content = """ -CN=minifi-cpp-flow-{feature_id}: - - CLASS_MINIFI_CPP -""".format(feature_id=self.feature_context.id) - self.authorities_file.write(authorities_file_content.encode()) - self.authorities_file.close() - os.chmod(self.authorities_file.name, 0o644) - - def get_startup_finished_log_entry(self): - return "Server Started" - - def deploy(self): - if not self.set_deployed(): - return - - test_dir = os.environ['TEST_DIRECTORY'] # Based on DockerVerify.sh - mounts = [docker.types.Mount( - type='bind', - source=os.path.join(test_dir, "resources/minifi-c2-server/config.yml"), - target='/opt/minifi-c2/minifi-c2-current/files/minifi-test-class/config.text.yml.v1' - )] - - if self.ssl: - mounts.append(docker.types.Mount( - type='bind', - source=self.authorities_file.name, - target='/opt/minifi-c2/minifi-c2-current/conf/authorities.yaml' - )) - mounts.append(docker.types.Mount( - type='bind', - source=os.path.join(test_dir, "resources/minifi-c2-server/authorizations.yaml"), - target='/opt/minifi-c2/minifi-c2-current/conf/authorizations.yaml' - )) - mounts.append(docker.types.Mount( - type='bind', - source=os.path.join(test_dir, "resources/minifi-c2-server/c2.properties"), - target='/opt/minifi-c2/minifi-c2-current/conf/c2.properties' - )) - mounts.append(docker.types.Mount( - type='bind', - source=self.server_keystore_file.name, - target='/opt/minifi-c2/minifi-c2-current/certs/minifi-c2-server-keystore.jks' - )) - mounts.append(docker.types.Mount( - type='bind', - source=self.server_truststore_file.name, - target='/opt/minifi-c2/minifi-c2-current/certs/minifi-c2-server-truststore.p12' - )) - self.docker_container = self.client.containers.run( - image='apache/nifi-minifi-c2:1.27.0', - detach=True, - name=self.name, - network=self.network.name, - mounts=mounts, - entrypoint=self.command) diff --git a/docker/test/integration/cluster/containers/MinifiContainer.py b/docker/test/integration/cluster/containers/MinifiContainer.py index 15e88021a..fcb73c0a4 100644 --- a/docker/test/integration/cluster/containers/MinifiContainer.py +++ b/docker/test/integration/cluster/containers/MinifiContainer.py @@ -27,8 +27,6 @@ from minifi.flow_serialization.Minifi_flow_json_serializer import Minifi_flow_js class MinifiOptions: def __init__(self): - self.enable_c2 = False - self.enable_c2_with_ssl = False self.enable_provenance = False self.enable_prometheus = False self.enable_prometheus_with_ssl = False @@ -39,7 +37,6 @@ class MinifiOptions: self.remove_python_requirements_txt = False self.use_nifi_python_processors_without_dependencies = False self.config_format = "json" - self.use_flow_config_from_url = False self.set_ssl_context_properties = False self.enable_log_metrics_publisher = False self.enable_example_minifi_python_processors = False @@ -133,25 +130,6 @@ class MinifiContainer(FlowContainer): f.write("nifi.database.content.repository.directory.default={minifi_home}/content_repository\n".format(minifi_home=MinifiContainer.MINIFI_LOCATIONS.minifi_home)) f.write("nifi.python.processor.dir={minifi_home}/minifi-python\n".format(minifi_home=MinifiContainer.MINIFI_LOCATIONS.minifi_home)) - if self.options.enable_c2: - f.write("nifi.c2.enable=true\n") - f.write(f"nifi.c2.rest.url=http://minifi-c2-server-{self.feature_context.id}:10090/c2/config/heartbeat\n";) - f.write(f"nifi.c2.rest.url.ack=http://minifi-c2-server-{self.feature_context.id}:10090/c2/config/acknowledge\n";) - f.write(f"nifi.c2.flow.base.url=http://minifi-c2-server-{self.feature_context.id}:10090/c2/config/\n";) - f.write("nifi.c2.root.classes=DeviceInfoNode,AgentInformation,FlowInformation,AssetInformation\n") - f.write("nifi.c2.full.heartbeat=false\n") - f.write("nifi.c2.agent.class=minifi-test-class\n") - f.write("nifi.c2.agent.identifier=minifi-test-id\n") - elif self.options.enable_c2_with_ssl: - f.write("nifi.c2.enable=true\n") - f.write(f"nifi.c2.rest.url=https://minifi-c2-server-{self.feature_context.id}:10090/c2/config/heartbeat\n";) - f.write(f"nifi.c2.rest.url.ack=https://minifi-c2-server-{self.feature_context.id}:10090/c2/config/acknowledge\n";) - f.write(f"nifi.c2.flow.base.url=https://minifi-c2-server-{self.feature_context.id}:10090/c2/config/\n";) - f.write("nifi.c2.root.classes=DeviceInfoNode,AgentInformation,FlowInformation,AssetInformation\n") - f.write("nifi.c2.full.heartbeat=false\n") - f.write("nifi.c2.agent.class=minifi-test-class\n") - f.write("nifi.c2.agent.identifier=minifi-test-id\n") - if self.options.set_ssl_context_properties: f.write("nifi.remote.input.secure=true\n") f.write("nifi.security.client.certificate=/tmp/resources/minifi_client.crt\n") @@ -180,9 +158,6 @@ class MinifiContainer(FlowContainer): if metrics_publisher_classes: f.write("nifi.metrics.publisher.class=" + ",".join(metrics_publisher_classes) + "\n") - if self.options.use_flow_config_from_url: - f.write(f"nifi.c2.flow.url=http://minifi-c2-server-{self.feature_context.id}:10090/c2/config?class=minifi-test-class\n";) - if self.options.use_nifi_python_processors_with_virtualenv or self.options.remove_python_requirements_txt or self.options.use_nifi_python_processors_without_dependencies: f.write("nifi.python.virtualenv.directory={minifi_python_venv_parent}/venv\n".format(minifi_python_venv_parent=MinifiContainer.MINIFI_LOCATIONS.minifi_python_venv_parent)) elif self.options.use_nifi_python_processors_with_virtualenv_packages_installed: @@ -198,10 +173,8 @@ class MinifiContainer(FlowContainer): def _setup_config(self): self._create_properties() - if not self.options.use_flow_config_from_url: - self._create_config() - self.vols[os.path.join(self.container_specific_config_dir, 'config.yml')] = {"bind": MinifiContainer.MINIFI_LOCATIONS.config_path, "mode": "rw"} - + self._create_config() + self.vols[os.path.join(self.container_specific_config_dir, 'config.yml')] = {"bind": MinifiContainer.MINIFI_LOCATIONS.config_path, "mode": "rw"} self.vols[os.path.join(self.container_specific_config_dir, 'minifi.properties')] = {"bind": MinifiContainer.MINIFI_LOCATIONS.properties_path, "mode": "rw"} self.vols[os.path.join(self.container_specific_config_dir, 'minifi-log.properties')] = {"bind": MinifiContainer.MINIFI_LOCATIONS.log_properties_path, "mode": "rw"} diff --git a/docker/test/integration/features/MiNiFi_integration_test_driver.py b/docker/test/integration/features/MiNiFi_integration_test_driver.py index 73941ecda..7a36309cd 100644 --- a/docker/test/integration/features/MiNiFi_integration_test_driver.py +++ b/docker/test/integration/features/MiNiFi_integration_test_driver.py @@ -63,11 +63,6 @@ class MiNiFi_integration_test: def acquire_transient_minifi(self, context, name, engine='minifi-cpp'): return self.cluster.acquire_transient_minifi(context=context, name=name, engine=engine) - def start_minifi_c2_server(self, context): - self.cluster.acquire_container(context=context, name="minifi-c2-server", engine="minifi-c2-server") - self.cluster.deploy_container('minifi-c2-server') - assert self.cluster.wait_for_container_startup_to_finish('minifi-c2-server') or self.cluster.log_app_output() - def start_nifi(self, context): self.cluster.acquire_container(context=context, name='nifi', engine='nifi') self.cluster.deploy_container('nifi') diff --git a/docker/test/integration/features/steps/steps.py b/docker/test/integration/features/steps/steps.py index 606d5d76d..ce222141b 100644 --- a/docker/test/integration/features/steps/steps.py +++ b/docker/test/integration/features/steps/steps.py @@ -827,43 +827,11 @@ def step_impl(context): context.test.check_all_prometheus_metric_types_are_defined_once() -# MiNiFi C2 Server -@given("ssl properties are set up for MiNiFi C2 server") -def step_impl(context): - context.test.enable_c2_with_ssl_in_minifi() - context.test.set_ssl_context_properties_in_minifi() - - @given("SSL properties are set in MiNiFi") def step_impl(context): context.test.set_ssl_context_properties_in_minifi() -@given(u'a MiNiFi C2 server is set up') -def step_impl(context): - context.test.acquire_container(context=context, name="minifi-c2-server", engine="minifi-c2-server") - - -@given(u'a MiNiFi C2 server is started') -def step_impl(context): - context.test.start_minifi_c2_server(context) - - -@then("the MiNiFi C2 server logs contain the following message: \"{log_message}\" in less than {duration}") -def step_impl(context, log_message, duration): - context.test.check_container_log_contents("minifi-c2-server", log_message, humanfriendly.parse_timespan(duration)) - - -@then("the MiNiFi C2 SSL server logs contain the following message: \"{log_message}\" in less than {duration}") -def step_impl(context, log_message, duration): - context.test.check_container_log_contents("minifi-c2-server-ssl", log_message, humanfriendly.parse_timespan(duration)) - - -@given(u'a MiNiFi C2 server is set up with SSL') -def step_impl(context): - context.test.acquire_container(context=context, name="minifi-c2-server", engine="minifi-c2-server-ssl") - - @given(u'flow configuration path is set up in flow url property') def step_impl(context): context.test.acquire_container(context=context, name="minifi-cpp-flow", engine="minifi-cpp") diff --git a/docker/test/integration/features/minifi_c2_server.feature b/extensions/standard-processors/tests/features/minifi_c2_server.feature similarity index 82% rename from docker/test/integration/features/minifi_c2_server.feature rename to extensions/standard-processors/tests/features/minifi_c2_server.feature index 2bbffc9f4..29d11295a 100644 --- a/docker/test/integration/features/minifi_c2_server.feature +++ b/extensions/standard-processors/tests/features/minifi_c2_server.feature @@ -16,9 +16,6 @@ @CORE Feature: MiNiFi can communicate with Apache NiFi MiNiFi C2 server - Background: - Given the content of "/tmp/output" is monitored - Scenario: MiNiFi flow config is updated from MiNiFi C2 server Given a GetFile processor with the name "GetFile1" and the "Input Directory" property set to "/tmp/non-existent" And C2 is enabled in MiNiFi @@ -26,7 +23,7 @@ Feature: MiNiFi can communicate with Apache NiFi MiNiFi C2 server And a MiNiFi C2 server is set up When all instances start up Then the MiNiFi C2 server logs contain the following message: "acknowledged with a state of FULLY_APPLIED(DONE)" in less than 30 seconds - And a flowfile with the content "test" is placed in the monitored directory in less than 10 seconds + And a single file with the content "test" is placed in the "/tmp/output" directory in less than 10 seconds And the Minifi logs do not contain the following message: "Failed to parse json response: The document is empty. at 0" after 0 seconds Scenario: MiNiFi can get flow config from C2 server through flow url when it is not available at start @@ -34,9 +31,9 @@ Feature: MiNiFi can communicate with Apache NiFi MiNiFi C2 server And C2 is enabled in MiNiFi And a file with the content "test" is present in "/tmp/input" And a MiNiFi C2 server is started - When all instances start up + When the MiNiFi instance starts up Then the MiNiFi C2 server logs contain the following message: "acknowledged with a state of FULLY_APPLIED(DONE)" in less than 30 seconds - And a flowfile with the content "test" is placed in the monitored directory in less than 10 seconds + And a single file with the content "test" is placed in the "/tmp/output" directory in less than 10 seconds Scenario: MiNiFi flow config is updated from MiNiFi C2 server through SSL with SSL properties Given a file with the content "test" is present in "/tmp/input" @@ -44,8 +41,8 @@ Feature: MiNiFi can communicate with Apache NiFi MiNiFi C2 server And ssl properties are set up for MiNiFi C2 server And a MiNiFi C2 server is set up with SSL When all instances start up - Then the MiNiFi C2 SSL server logs contain the following message: "acknowledged with a state of FULLY_APPLIED(DONE)" in less than 60 seconds - And a flowfile with the content "test" is placed in the monitored directory in less than 10 seconds + Then the MiNiFi C2 server logs contain the following message: "acknowledged with a state of FULLY_APPLIED(DONE)" in less than 60 seconds + And a single file with the content "test" is placed in the "/tmp/output" directory in less than 10 seconds Scenario: MiNiFi flow config is updated from MiNiFi C2 server with overriden parameter context Given a GetFile processor with the name "GetFile1" and the "Input Directory" property set to "${INPUT_DIR}" @@ -56,5 +53,5 @@ Feature: MiNiFi can communicate with Apache NiFi MiNiFi C2 server And a MiNiFi C2 server is set up When all instances start up Then the MiNiFi C2 server logs contain the following message: "acknowledged with a state of FULLY_APPLIED(DONE)" in less than 30 seconds - And a flowfile with the content "test" is placed in the monitored directory in less than 10 seconds + And a single file with the content "test" is placed in the "/tmp/output" directory in less than 10 seconds And the Minifi logs do not contain the following message: "Failed to parse json response: The document is empty. at 0" after 0 seconds diff --git a/extensions/standard-processors/tests/features/steps/minifi_c2_server_container.py b/extensions/standard-processors/tests/features/steps/minifi_c2_server_container.py new file mode 100644 index 000000000..6c25d7c77 --- /dev/null +++ b/extensions/standard-processors/tests/features/steps/minifi_c2_server_container.py @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import jks +import os +from OpenSSL import crypto +from cryptography.hazmat.primitives.serialization import pkcs12, BestAvailableEncryption, load_pem_private_key +from cryptography import x509 +from pathlib import Path + +from minifi_test_framework.containers.container import Container +from minifi_test_framework.core.helpers import wait_for_condition +from minifi_test_framework.core.minifi_test_context import MinifiTestContext +from minifi_test_framework.core.ssl_utils import make_server_cert +from minifi_test_framework.containers.file import File +from minifi_test_framework.containers.host_file import HostFile + + +class MinifiC2Server(Container): + def __init__(self, test_context: MinifiTestContext, ssl: bool = False): + super().__init__("apache/nifi-minifi-c2:1.27.0", f"minifi-c2-server-{test_context.scenario_id}", test_context.network) + if ssl: + c2_cert, c2_key = make_server_cert(f"minifi-c2-server-{test_context.scenario_id}", test_context.root_ca_cert, test_context.root_ca_key) + pke = jks.PrivateKeyEntry.new('c2-server-cert', [crypto.dump_certificate(crypto.FILETYPE_ASN1, c2_cert)], crypto.dump_privatekey(crypto.FILETYPE_ASN1, c2_key), 'rsa_raw') + server_keystore = jks.KeyStore.new('jks', [pke]) + server_keystore_content = server_keystore.saves('abcdefgh') + self.files.append(File("/opt/minifi-c2/minifi-c2-current/certs/minifi-c2-server-keystore.jks", server_keystore_content, permissions=0o644)) + + private_key_pem = crypto.dump_privatekey(crypto.FILETYPE_PEM, test_context.root_ca_key) + private_key = load_pem_private_key(private_key_pem, password=None) + certificate_pem = crypto.dump_certificate(crypto.FILETYPE_PEM, test_context.root_ca_cert) + certificate = x509.load_pem_x509_certificate(certificate_pem) + pkcs12_data = pkcs12.serialize_key_and_certificates( + name=None, + key=private_key, + cert=certificate, + cas=None, + encryption_algorithm=BestAvailableEncryption(b'abcdefgh') + ) + self.files.append(File("/opt/minifi-c2/minifi-c2-current/certs/minifi-c2-server-truststore.p12", pkcs12_data, permissions=0o644)) + + authorities_file_content = """ + CN=minifi-primary-{scenario_id}: + - CLASS_MINIFI_CPP + """.format(scenario_id=test_context.scenario_id) + self.files.append(File("/opt/minifi-c2/minifi-c2-current/conf/authorities.yaml", authorities_file_content, permissions=0o644)) + + resource_dir = Path(__file__).resolve().parent / "resources" / "minifi-c2-server" + self.host_files.append(HostFile("/opt/minifi-c2/minifi-c2-current/files/minifi-test-class/config.text.yml.v1", os.path.join(resource_dir, "config.yml"))) + if ssl: + self.host_files.append(HostFile("/opt/minifi-c2/minifi-c2-current/conf/authorizations.yaml", os.path.join(resource_dir, "authorizations.yaml"))) + self.host_files.append(HostFile("/opt/minifi-c2/minifi-c2-current/conf/c2.properties", os.path.join(resource_dir, "c2.properties"))) + + def deploy(self): + super().deploy() + finished_str = "Server Started" + return wait_for_condition( + condition=lambda: finished_str in self.get_logs(), + timeout_seconds=60, + bail_condition=lambda: self.exited, + context=None + ) diff --git a/docker/test/integration/resources/minifi-c2-server/authorizations.yaml b/extensions/standard-processors/tests/features/steps/resources/minifi-c2-server/authorizations.yaml similarity index 100% rename from docker/test/integration/resources/minifi-c2-server/authorizations.yaml rename to extensions/standard-processors/tests/features/steps/resources/minifi-c2-server/authorizations.yaml diff --git a/docker/test/integration/resources/minifi-c2-server/c2.properties b/extensions/standard-processors/tests/features/steps/resources/minifi-c2-server/c2.properties similarity index 100% rename from docker/test/integration/resources/minifi-c2-server/c2.properties rename to extensions/standard-processors/tests/features/steps/resources/minifi-c2-server/c2.properties diff --git a/docker/test/integration/resources/minifi-c2-server/config.yml b/extensions/standard-processors/tests/features/steps/resources/minifi-c2-server/config.yml similarity index 100% rename from docker/test/integration/resources/minifi-c2-server/config.yml rename to extensions/standard-processors/tests/features/steps/resources/minifi-c2-server/config.yml diff --git a/extensions/standard-processors/tests/features/steps/steps.py b/extensions/standard-processors/tests/features/steps/steps.py index 85158cb9e..517bff039 100644 --- a/extensions/standard-processors/tests/features/steps/steps.py +++ b/extensions/standard-processors/tests/features/steps/steps.py @@ -15,16 +15,19 @@ # limitations under the License. # -from behave import step +import humanfriendly +from behave import step, then, given from minifi_test_framework.steps import checking_steps # noqa: F401 from minifi_test_framework.steps import configuration_steps # noqa: F401 from minifi_test_framework.steps import core_steps # noqa: F401 from minifi_test_framework.steps import flow_building_steps # noqa: F401 from minifi_test_framework.core.minifi_test_context import MinifiTestContext +from minifi_test_framework.core.helpers import wait_for_condition from syslog_container import SyslogContainer from diag_slave_container import DiagSlave from tcp_client_container import TcpClientContainer +from minifi_c2_server_container import MinifiC2Server @step("a Syslog client with TCP protocol is setup to send logs to minifi") @@ -51,3 +54,52 @@ def step_impl(context: MinifiTestContext, modbus_cmd: str): @step('a TCP client is set up to send a test TCP message to minifi') def step_impl(context: MinifiTestContext): context.containers["tcp-client"] = TcpClientContainer(context) + + +@given("C2 is enabled in MiNiFi") +def step_impl(context: MinifiTestContext): + context.get_or_create_default_minifi_container().set_property("nifi.c2.enable", "true") + context.get_or_create_default_minifi_container().set_property("nifi.c2.rest.url", f"http://minifi-c2-server-{context.scenario_id}:10090/c2/config/heartbeat";) + context.get_or_create_default_minifi_container().set_property("nifi.c2.rest.url.ack", f"http://minifi-c2-server-{context.scenario_id}:10090/c2/config/acknowledge";) + context.get_or_create_default_minifi_container().set_property("nifi.c2.flow.base.url", f"http://minifi-c2-server-{context.scenario_id}:10090/c2/config/";) + context.get_or_create_default_minifi_container().set_property("nifi.c2.root.classes", "DeviceInfoNode,AgentInformation,FlowInformation,AssetInformation") + context.get_or_create_default_minifi_container().set_property("nifi.c2.full.heartbeat", "false") + context.get_or_create_default_minifi_container().set_property("nifi.c2.agent.class", "minifi-test-class") + context.get_or_create_default_minifi_container().set_property("nifi.c2.agent.identifier", "minifi-test-id") + + +@given("ssl properties are set up for MiNiFi C2 server") +def step_impl(context: MinifiTestContext): + context.get_or_create_default_minifi_container().set_property("nifi.c2.enable", "true") + context.get_or_create_default_minifi_container().set_property("nifi.c2.rest.url", f"https://minifi-c2-server-{context.scenario_id}:10090/c2/config/heartbeat";) + context.get_or_create_default_minifi_container().set_property("nifi.c2.rest.url.ack", f"https://minifi-c2-server-{context.scenario_id}:10090/c2/config/acknowledge";) + context.get_or_create_default_minifi_container().set_property("nifi.c2.flow.base.url", f"https://minifi-c2-server-{context.scenario_id}:10090/c2/config/";) + context.get_or_create_default_minifi_container().set_property("nifi.c2.root.classes", "DeviceInfoNode,AgentInformation,FlowInformation,AssetInformation") + context.get_or_create_default_minifi_container().set_property("nifi.c2.full.heartbeat", "false") + context.get_or_create_default_minifi_container().set_property("nifi.c2.agent.class", "minifi-test-class") + context.get_or_create_default_minifi_container().set_property("nifi.c2.agent.identifier", "minifi-test-id") + context.get_or_create_default_minifi_container().set_up_ssl_properties() + + +@given("a MiNiFi C2 server is set up") +def step_impl(context: MinifiTestContext): + context.containers["minifi-c2-server"] = MinifiC2Server(context) + + +@given("a MiNiFi C2 server is set up with SSL") +def step_impl(context: MinifiTestContext): + context.containers["minifi-c2-server"] = MinifiC2Server(context, ssl=True) + + +@given("a MiNiFi C2 server is started") +def step_impl(context: MinifiTestContext): + context.containers["minifi-c2-server"] = MinifiC2Server(context) + assert context.containers["minifi-c2-server"].deploy() + + +@then("the MiNiFi C2 server logs contain the following message: \"{log_message}\" in less than {duration}") +def step_impl(context: MinifiTestContext, log_message: str, duration: str): + duration_seconds = humanfriendly.parse_timespan(duration) + assert wait_for_condition(condition=lambda: log_message in context.containers["minifi-c2-server"].get_logs(), + timeout_seconds=duration_seconds, bail_condition=lambda: context.containers["minifi-c2-server"].exited, + context=context)
