This is an automated email from the ASF dual-hosted git repository. fgerlits pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 4d7c19eaac51f535016ba00c473dcb178cfdeec1 Author: Gabor Gyimesi <[email protected]> AuthorDate: Thu Oct 2 17:03:34 2025 +0200 MINIFICPP-2645 Enable NiFi python processors using both EL and validators Signed-off-by: Ferenc Gerlits <[email protected]> Closes #2041 --- docker/test/integration/cluster/ImageStore.py | 4 +- docker/test/integration/features/python.feature | 20 ++++++++ .../ExpressionLanguagePropertyWithValidator.py | 26 +++++++++++ .../ExpressionLanguagePropertyWithValidator.py | 53 ++++++++++++++++++++++ .../pythonprocessors/nifiapi/processorbase.py | 9 +++- 5 files changed, 109 insertions(+), 3 deletions(-) diff --git a/docker/test/integration/cluster/ImageStore.py b/docker/test/integration/cluster/ImageStore.py index 1b0802df2..9fe772dcb 100644 --- a/docker/test/integration/cluster/ImageStore.py +++ b/docker/test/integration/cluster/ImageStore.py @@ -213,6 +213,7 @@ class ImageStore: COPY TestStateManager.py {minifi_python_dir}/nifi_python_processors/TestStateManager.py COPY NifiStyleLogDynamicProperties.py {minifi_python_dir}/nifi_python_processors/NifiStyleLogDynamicProperties.py COPY LogDynamicProperties.py {minifi_python_dir}/LogDynamicProperties.py + COPY ExpressionLanguagePropertyWithValidator.py {minifi_python_dir}/nifi_python_processors/ExpressionLanguagePropertyWithValidator.py RUN python3 -m venv {minifi_python_venv_parent}/venv """.format(base_image='apacheminificpp:' + MinifiContainer.MINIFI_TAG_PREFIX + MinifiContainer.MINIFI_VERSION, pip3_install_command=pip3_install_command, @@ -237,7 +238,8 @@ class ImageStore: build_full_python_resource_path("SetRecordField.py"), build_full_python_resource_path("TestStateManager.py"), build_full_python_resource_path("NifiStyleLogDynamicProperties.py"), - build_full_python_resource_path("LogDynamicProperties.py") + build_full_python_resource_path("LogDynamicProperties.py"), + build_full_python_resource_path("ExpressionLanguagePropertyWithValidator.py") ]) def __build_minifi_cpp_image_with_llamacpp_model(self): diff --git a/docker/test/integration/features/python.feature b/docker/test/integration/features/python.feature index 966f9d81b..08b8dd1c4 100644 --- a/docker/test/integration/features/python.feature +++ b/docker/test/integration/features/python.feature @@ -269,3 +269,23 @@ Feature: MiNiFi can use python processors in its flows When all instances start up Then the Minifi logs contain the following message: "Static Property value: static value" in less than 60 seconds And the Minifi logs contain the following message: "Dynamic Property value: dynamic value" in less than 60 seconds + + Scenario: MiNiFi C++ allows NiFi python processors to use property validators for expression language enabled properties + Given a GenerateFlowFile processor with the "File Size" property set to "0B" + And a ExpressionLanguagePropertyWithValidator processor with the "Integer Property" property set to "42" + And the "success" relationship of the GenerateFlowFile processor is connected to the ExpressionLanguagePropertyWithValidator + And python processors without dependencies are present on the MiNiFi agent + + When all instances start up + Then the Minifi logs contain the following message: "Integer Property value: 42" in less than 60 seconds + + Scenario: MiNiFi C++ rolls back session if expression language cannot be evaluated as integer in NiFi python processors + Given a GenerateFlowFile processor with the "File Size" property set to "0B" + And a UpdateAttribute processor with the "my.integer" property set to "invalid" + And a ExpressionLanguagePropertyWithValidator processor with the "Integer Property" property set to "${my.integer}" + And the "success" relationship of the GenerateFlowFile processor is connected to the UpdateAttribute + And the "success" relationship of the UpdateAttribute processor is connected to the ExpressionLanguagePropertyWithValidator + And python processors without dependencies are present on the MiNiFi agent + + When all instances start up + Then the Minifi logs contain the following message: "ProcessSession rollback for ExpressionLanguagePropertyWithValidator" in less than 60 seconds diff --git a/docker/test/integration/minifi/processors/ExpressionLanguagePropertyWithValidator.py b/docker/test/integration/minifi/processors/ExpressionLanguagePropertyWithValidator.py new file mode 100644 index 000000000..42bdede34 --- /dev/null +++ b/docker/test/integration/minifi/processors/ExpressionLanguagePropertyWithValidator.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from ..core.Processor import Processor + + +class ExpressionLanguagePropertyWithValidator(Processor): + def __init__(self, context): + super(ExpressionLanguagePropertyWithValidator, self).__init__( + context=context, + clazz='ExpressionLanguagePropertyWithValidator', + class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.', + properties={}, + schedule={'scheduling strategy': 'EVENT_DRIVEN'}, + auto_terminate=[]) diff --git a/docker/test/integration/resources/python/ExpressionLanguagePropertyWithValidator.py b/docker/test/integration/resources/python/ExpressionLanguagePropertyWithValidator.py new file mode 100644 index 000000000..346f1ede4 --- /dev/null +++ b/docker/test/integration/resources/python/ExpressionLanguagePropertyWithValidator.py @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult +from nifiapi.properties import ExpressionLanguageScope, PropertyDescriptor, StandardValidators + + +class ExpressionLanguagePropertyWithValidator(FlowFileTransform): + + class Java: + implements = ['org.apache.nifi.python.processor.FlowFileTransform'] + + class ProcessorDetails: + version = '1.2.3' + description = "Test processor" + dependencies = [] + + INTEGER_PROPERTY = PropertyDescriptor( + name="Integer Property", + description="A dummy integer property", + required=True, + validators=[StandardValidators.INTEGER_VALIDATOR], + expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES + ) + + def __init__(self, **kwargs): + pass + + def getPropertyDescriptors(self): + return [self.INTEGER_PROPERTY] + + def getDynamicPropertyDescriptor(self, propertyname): + return PropertyDescriptor(name=propertyname, + description="A user-defined property", + dynamic=True) + + def transform(self, context, flow_file): + integer_value = context.getProperty(self.INTEGER_PROPERTY).asInteger() + self.logger.info("Integer Property value: {}".format(integer_value)) + + return FlowFileTransformResult('success', contents="content") diff --git a/extensions/python/pythonprocessors/nifiapi/processorbase.py b/extensions/python/pythonprocessors/nifiapi/processorbase.py index 7a884aeed..1ef376c4a 100644 --- a/extensions/python/pythonprocessors/nifiapi/processorbase.py +++ b/extensions/python/pythonprocessors/nifiapi/processorbase.py @@ -15,7 +15,7 @@ from abc import ABC, abstractmethod from typing import List -from .properties import ExpressionLanguageScope, PropertyDescriptor, translateStandardValidatorToMiNiFiPropertype +from .properties import ExpressionLanguageScope, PropertyDescriptor, translateStandardValidatorToMiNiFiPropertype, MinifiPropertyTypes from .properties import ProcessContext as ProcessContextProxy from minifi_native import OutputStream, Processor, ProcessContext, ProcessSession @@ -54,8 +54,13 @@ class ProcessorBase(ABC): self.supports_dynamic_properties = False for property in self.getPropertyDescriptors(): - property_type_code = translateStandardValidatorToMiNiFiPropertype(property.validators) expression_language_supported = True if property.expressionLanguageScope != ExpressionLanguageScope.NONE else False + property_type_code = translateStandardValidatorToMiNiFiPropertype(property.validators) + + # MiNiFi C++ does not support validators for expression language enabled properties + if expression_language_supported and property_type_code is not None and property_type_code != MinifiPropertyTypes.NON_BLANK_TYPE: + self.logger.warn("Property '{}' has validators defined, but since it also supports Expression Language, the validators will be ignored.".format(property.name)) + property_type_code = None # MiNiFi C++ does not support dependant properties, so if a property depends on another property, it should not be required is_required = True if property.required and not property.dependencies else False
