This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 7771c651aa942cc9584f044aea42d994d71c9a05 Author: Martin Zink <[email protected]> AuthorDate: Wed Mar 29 06:08:45 2023 +0200 MINIFICPP-2072 Docker test for ListenSyslog/ReplaceText workflow Closes #1529 Signed-off-by: Marton Szasz <[email protected]> --- .../integration/MiNiFi_integration_test_driver.py | 6 ++++ .../integration/features/syslog_listener.feature | 12 ++++--- .../filesystem_validation/FileSystemObserver.py | 2 +- .../minifi/validators/FileOutputValidator.py | 17 ++++++++++ .../SingleOrMultiFileOutputRegexValidator.py | 38 ++++++++++++++++++++++ docker/test/integration/steps/steps.py | 5 +++ 6 files changed, 75 insertions(+), 5 deletions(-) diff --git a/docker/test/integration/MiNiFi_integration_test_driver.py b/docker/test/integration/MiNiFi_integration_test_driver.py index e23a8f2ab..34dbfb502 100644 --- a/docker/test/integration/MiNiFi_integration_test_driver.py +++ b/docker/test/integration/MiNiFi_integration_test_driver.py @@ -29,6 +29,7 @@ from minifi.validators.NoFileOutPutValidator import NoFileOutPutValidator from minifi.validators.SingleFileOutputValidator import SingleFileOutputValidator from minifi.validators.MultiFileOutputValidator import MultiFileOutputValidator from minifi.validators.SingleOrMultiFileOutputValidator import SingleOrMultiFileOutputValidator +from minifi.validators.SingleOrMultiFileOutputRegexValidator import SingleOrMultiFileOutputRegexValidator from minifi.validators.NoContentCheckFileNumberValidator import NoContentCheckFileNumberValidator from minifi.validators.NumFileRangeValidator import NumFileRangeValidator from minifi.validators.SingleJSONFileOutputValidator import SingleJSONFileOutputValidator @@ -172,6 +173,11 @@ class MiNiFi_integration_test: output_validator.set_output_dir(self.file_system_observer.get_output_dir()) self.__check_output(timeout_seconds, output_validator, file_count) + def check_for_at_least_one_file_with_matching_content(self, regex, timeout_seconds): + output_validator = SingleOrMultiFileOutputRegexValidator(regex) + output_validator.set_output_dir(self.file_system_observer.get_output_dir()) + self.__check_output(timeout_seconds, output_validator) + def check_for_at_least_one_file_with_content_generated(self, content, timeout_seconds): output_validator = SingleOrMultiFileOutputValidator(decode_escaped_str(content)) output_validator.set_output_dir(self.file_system_observer.get_output_dir()) diff --git a/docker/test/integration/features/syslog_listener.feature b/docker/test/integration/features/syslog_listener.feature index 7ffe70e9a..0e4a4d264 100644 --- a/docker/test/integration/features/syslog_listener.feature +++ b/docker/test/integration/features/syslog_listener.feature @@ -22,18 +22,22 @@ Feature: Minifi C++ can act as a syslog listener Given a ListenSyslog processor And a PutFile processor with the "Directory" property set to "/tmp/output" And the "Protocol" property of the ListenSyslog processor is set to "UDP" - And the "Parse Messages" property of the ListenSyslog processor is set to "yes" + And the "Parse Messages" property of the ListenSyslog processor is set to "true" And a Syslog client with UDP protocol is setup to send logs to minifi - And the "success" relationship of the ListenSyslog processor is connected to the PutFile + And a ReplaceText processor with the "Evaluation Mode" property set to "Entire text" + And the "Replacement Strategy" property of the ReplaceText processor is set to "Always Replace" + And the "Replacement Value" property of the ReplaceText processor is set to "${syslog.timestamp} ${syslog.priority} ${reverseDnsLookup(${syslog.sender}, 100)} ${syslog.msg}" + And the "success" relationship of the ListenSyslog processor is connected to the ReplaceText + And the "success" relationship of the ReplaceText processor is connected to the PutFile When both instances start up - Then at least one flowfile is placed in the monitored directory in less than 10 seconds + Then at least one flowfile's content match the following regex: "^((?:(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}:\d{2}(?:\.\d+)?))(Z|[\+-]\d{2}:\d{2})?)\t13\t.*\tsample_log$" in less than 10 seconds Scenario: A syslog client can send messages to Minifi over TCP Given a ListenSyslog processor And a PutFile processor with the "Directory" property set to "/tmp/output" And the "Protocol" property of the ListenSyslog processor is set to "TCP" - And the "Parse Messages" property of the ListenSyslog processor is set to "yes" + And the "Parse Messages" property of the ListenSyslog processor is set to "true" And a Syslog client with TCP protocol is setup to send logs to minifi And the "success" relationship of the ListenSyslog processor is connected to the PutFile diff --git a/docker/test/integration/filesystem_validation/FileSystemObserver.py b/docker/test/integration/filesystem_validation/FileSystemObserver.py index 629bd6eae..98874622f 100644 --- a/docker/test/integration/filesystem_validation/FileSystemObserver.py +++ b/docker/test/integration/filesystem_validation/FileSystemObserver.py @@ -56,7 +56,7 @@ class FileSystemObserver(object): while True: # Note: The timing on Event.wait() is inaccurate self.done_event.wait(timeout_seconds - time.perf_counter() + wait_start_time) - if self.done_event.isSet(): + if self.done_event.is_set(): self.done_event.clear() if max_files and max_files <= self.event_handler.get_num_files_created(): return output_validator.validate() diff --git a/docker/test/integration/minifi/validators/FileOutputValidator.py b/docker/test/integration/minifi/validators/FileOutputValidator.py index f4d59003d..5cc693f2c 100644 --- a/docker/test/integration/minifi/validators/FileOutputValidator.py +++ b/docker/test/integration/minifi/validators/FileOutputValidator.py @@ -16,6 +16,7 @@ import logging import os +import re from os import listdir from os.path import join @@ -28,6 +29,22 @@ class FileOutputValidator(OutputValidator): def set_output_dir(self, output_dir): self.output_dir = output_dir + @staticmethod + def num_files_matching_regex_in_dir(dir_path: str, expected_content_regex: str): + listing = listdir(dir_path) + if not listing: + return 0 + files_of_matching_content_found = 0 + for file_name in listing: + full_path = join(dir_path, file_name) + if not os.path.isfile(full_path) or is_temporary_output_file(full_path): + continue + with open(full_path, 'r') as out_file: + content = out_file.read() + if re.search(expected_content_regex, content): + files_of_matching_content_found += 1 + return files_of_matching_content_found + @staticmethod def num_files_matching_content_in_dir(dir_path, expected_content): listing = listdir(dir_path) diff --git a/docker/test/integration/minifi/validators/SingleOrMultiFileOutputRegexValidator.py b/docker/test/integration/minifi/validators/SingleOrMultiFileOutputRegexValidator.py new file mode 100644 index 000000000..db4c73485 --- /dev/null +++ b/docker/test/integration/minifi/validators/SingleOrMultiFileOutputRegexValidator.py @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import os + +from .FileOutputValidator import FileOutputValidator + + +class SingleOrMultiFileOutputRegexValidator(FileOutputValidator): + """ + Validates the content of a single or multiple files in the given directory. + """ + + def __init__(self, expected_content_regex: str): + self.expected_content_regex = expected_content_regex + + def validate(self): + full_dir = os.path.join(self.output_dir) + logging.info("Output folder: %s", full_dir) + + if not os.path.isdir(full_dir): + return False + + return 0 < self.num_files_matching_regex_in_dir(full_dir, self.expected_content_regex) diff --git a/docker/test/integration/steps/steps.py b/docker/test/integration/steps/steps.py index cce5eaf30..9dce360d7 100644 --- a/docker/test/integration/steps/steps.py +++ b/docker/test/integration/steps/steps.py @@ -762,6 +762,11 @@ def step_impl(context, content, duration): context.test.check_for_single_json_file_with_content_generated(content, humanfriendly.parse_timespan(duration)) +@then("at least one flowfile's content match the following regex: \"{regex}\" in less than {duration}") +def step_impl(context, regex: str, duration: str): + context.test.check_for_at_least_one_file_with_matching_content(regex, humanfriendly.parse_timespan(duration)) + + @then("at least one flowfile with the content \"{content}\" is placed in the monitored directory in less than {duration}") @then("at least one flowfile with the content '{content}' is placed in the monitored directory in less than {duration}") def step_impl(context, content, duration):
