This is an automated email from the ASF dual-hosted git repository.
fgerlits pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
from 61d347da7 MINIFICPP-2562 Readd setting current working directory at
startup
new 8602b76d2 MINIFICPP-2556 Create llama.cpp processor for LLM inference
new 6611bea39 MINIFICPP-2518 PutKinesisStream
new ccccbef80 MINIFICPP-2525 Handle errors when enabling controllers
new 241c56e68 MINIFICPP-2542 ConsumeKafka late offset commit
new 03eb5df50 MINIFICPP-2564 - Resolve asset path references during flow
parsing
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci.yml | 7 +-
LICENSE | 25 ++
METRICS.md | 16 +
NOTICE | 1 +
PROCESSORS.md | 90 ++++
README.md | 5 +-
bootstrap.sh | 2 +
bootstrap/minifi_option.py | 2 +-
bstrp_functions.sh | 2 +
cmake/BundledAwsSdkCpp.cmake | 14 +-
cmake/CppVersion.cmake | 4 +-
cmake/LlamaCpp.cmake | 49 +++
cmake/MiNiFiOptions.cmake | 1 +
cmake/Spdlog.cmake | 4 +-
cmake/fmt.cmake | 1 +
controller/tests/ControllerTests.cpp | 2 +-
docker/requirements.txt | 3 +-
docker/test/integration/cluster/ContainerStore.py | 22 +-
.../test/integration/cluster/DockerTestCluster.py | 17 +-
docker/test/integration/cluster/ImageStore.py | 15 +
.../integration/cluster/checkers/AwsChecker.py | 5 +
.../integration/cluster/checkers/KafkaHelper.py | 58 +++
.../cluster/containers/KafkaBrokerContainer.py | 113 +++--
...ProxyContainer.py => KinesisServerContainer.py} | 12 +-
.../cluster/containers/MinifiContainer.py | 3 +
.../cluster/containers/ZookeeperContainer.py | 1 -
.../features/MiNiFi_integration_test_driver.py | 45 +-
.../integration/features/consume_kafka.feature | 321 ++++++++++++++
docker/test/integration/features/kafka.feature | 468 --------------------
...{attributes_to_json.feature => kinesis.feature} | 29 +-
.../{lua_script.feature => llamacpp.feature} | 22 +-
.../integration/features/publish_kafka.feature | 228 ++++++++++
.../test/integration/features/steps/kafka_steps.py | 188 ++++++++
docker/test/integration/features/steps/steps.py | 250 +++--------
.../{DeleteS3Object.py => PutKinesisStream.py} | 14 +-
...{PutOPCProcessor.py => RunLlamaCppInference.py} | 6 +-
.../integration/resources/kafka_broker/Dockerfile | 4 -
.../resources/kafka_broker/conf/server.properties | 180 --------
.../integration/resources/kinesis-mock/Dockerfile | 8 +
.../resources/kinesis-mock/consumer/consumer.js | 64 +++
.../resources/kinesis-mock/consumer/package.json | 3 +
.../integration/resources/kinesis-mock/server.json | 6 +
.../include/utils/ProcessorConfigUtils.h | 38 +-
extensions/aws/AWSCredentialsProvider.cpp | 2 -
extensions/aws/AWSCredentialsProvider.h | 2 -
extensions/aws/CMakeLists.txt | 10 +-
.../{S3Processor.cpp => AwsProcessor.cpp} | 49 +--
.../processors/{S3Processor.h => AwsProcessor.h} | 35 +-
extensions/aws/processors/DeleteS3Object.cpp | 21 +-
extensions/aws/processors/DeleteS3Object.h | 5 +-
extensions/aws/processors/FetchS3Object.cpp | 19 +-
extensions/aws/processors/FetchS3Object.h | 5 +-
extensions/aws/processors/ListS3.cpp | 6 +-
extensions/aws/processors/PutKinesisStream.cpp | 208 +++++++++
extensions/aws/processors/PutKinesisStream.h | 139 ++++++
extensions/aws/processors/PutS3Object.cpp | 35 +-
extensions/aws/processors/PutS3Object.h | 11 +-
extensions/aws/processors/S3Processor.cpp | 113 +----
extensions/aws/processors/S3Processor.h | 155 +------
extensions/aws/s3/S3ClientRequestSender.cpp | 2 -
extensions/aws/s3/S3ClientRequestSender.h | 2 -
extensions/aws/s3/S3RequestSender.h | 9 -
extensions/aws/s3/S3Wrapper.cpp | 2 -
extensions/aws/s3/S3Wrapper.h | 24 +-
extensions/aws/tests/PutKinesisStreamTests.cpp | 318 ++++++++++++++
extensions/aws/utils/AWSInitializer.cpp | 19 +-
extensions/aws/utils/AWSInitializer.h | 2 -
extensions/aws/utils/AWSSdkLogger.cpp | 2 -
extensions/aws/utils/AWSSdkLogger.h | 2 -
.../aws/utils/ProxyOptions.h | 16 +-
extensions/civetweb/tests/ListenHTTPTests.cpp | 50 +--
.../controllerservices/CouchbaseClusterService.h | 4 +-
extensions/kafka/ConsumeKafka.cpp | 340 +++++++++------
extensions/kafka/ConsumeKafka.h | 473 ++++++++++++---------
extensions/kafka/rdkafka_utils.h | 14 +
extensions/{mqtt => llamacpp}/CMakeLists.txt | 17 +-
.../llamacpp/processors/DefaultLlamaContext.cpp | 157 +++++++
.../llamacpp/processors/DefaultLlamaContext.h | 44 ++
.../processors/LlamaBackendInitializer.cpp} | 17 +-
.../llamacpp/processors/LlamaBackendInitializer.h | 32 +-
extensions/llamacpp/processors/LlamaContext.h | 66 +++
.../llamacpp/processors/RunLlamaCppInference.cpp | 155 +++++++
.../llamacpp/processors/RunLlamaCppInference.h | 198 +++++++++
.../{kafka => llamacpp}/tests/CMakeLists.txt | 20 +-
.../llamacpp/tests/RunLlamaCppInferenceTests.cpp | 345 +++++++++++++++
extensions/mqtt/tests/ConsumeMQTTTests.cpp | 6 +-
extensions/mqtt/tests/PublishMQTTTests.cpp | 9 +-
.../tests/SmbConnectionControllerServiceTests.cpp | 2 +-
extensions/standard-processors/modbus/Error.h | 12 +-
.../tests/unit/SegmentContentTests.cpp | 4 +-
.../tests/unit/SplitContentTests.cpp | 2 +-
.../tests/unit/SplitTextTests.cpp | 2 +-
libminifi/include/core/FlowConfiguration.h | 6 +-
libminifi/include/core/ReferenceParser.h | 135 ++++++
.../ForwardingControllerServiceProvider.h | 4 +-
.../controller/StandardControllerServiceNode.h | 26 +-
.../controller/StandardControllerServiceProvider.h | 74 +---
.../include/core/state/nodes/SupportedOperations.h | 2 +-
libminifi/include/utils/file/AssetManager.h | 2 +
libminifi/src/core/FlowConfiguration.cpp | 19 +-
.../core/controller/ControllerServiceNodeMap.cpp | 1 +
.../controller/StandardControllerServiceNode.cpp | 51 ++-
.../StandardControllerServiceProvider.cpp | 128 ++++++
.../src/core/flow/StructuredConfiguration.cpp | 20 +-
.../src/core/state/nodes/SupportedOperations.cpp | 35 +-
.../{ParsingErrors.cpp => AttributeErrors.cpp} | 10 +-
libminifi/src/utils/file/AssetManager.cpp | 10 +
.../integration/C2ControllerEnableFailureTest.cpp | 197 +++++++++
libminifi/test/libtest/unit/ProvenanceTestHelper.h | 54 ---
libminifi/test/libtest/unit/TestBase.cpp | 2 +-
...heduleRetry.yml => TestC2InvalidController.yml} | 24 +-
...ScheduleRetry.yml => TestC2ValidController.yml} | 24 +-
libminifi/test/unit/AssetResolutionTests.cpp | 88 ++++
libminifi/test/unit/ExpectedTest.cpp | 2 +-
libminifi/test/unit/SchedulingAgentTests.cpp | 1 -
minifi-api/include/minifi-cpp/Exception.h | 3 +-
.../core/controller/ControllerServiceProvider.h | 2 +-
minifi_main/MiNiFiMain.cpp | 7 +-
thirdparty/aws-sdk-cpp/dll-export-injection.patch | 185 +++++---
thirdparty/llamacpp/lu8_macro_fix.patch | 17 +
utils/include/core/ProcessContext.h | 16 +-
utils/include/core/ProcessorMetrics.h | 1 +
.../utils/{ParsingErrors.h => AttributeErrors.h} | 35 +-
utils/include/utils/Error.h | 10 -
utils/include/utils/OptionalUtils.h | 3 +-
utils/include/utils/ParsingErrors.h | 15 +-
utils/include/utils/ParsingUtils.h | 3 +-
utils/include/utils/PropertyErrors.h | 35 +-
utils/include/utils/expected.h | 54 ++-
utils/src/utils/ParsingUtils.cpp | 8 +
130 files changed, 4716 insertions(+), 2123 deletions(-)
create mode 100644 cmake/LlamaCpp.cmake
create mode 100644 docker/test/integration/cluster/checkers/KafkaHelper.py
copy docker/test/integration/cluster/containers/{HttpProxyContainer.py =>
KinesisServerContainer.py} (75%)
create mode 100644 docker/test/integration/features/consume_kafka.feature
delete mode 100644 docker/test/integration/features/kafka.feature
copy docker/test/integration/features/{attributes_to_json.feature =>
kinesis.feature} (56%)
copy docker/test/integration/features/{lua_script.feature => llamacpp.feature}
(52%)
create mode 100644 docker/test/integration/features/publish_kafka.feature
create mode 100644 docker/test/integration/features/steps/kafka_steps.py
copy docker/test/integration/minifi/processors/{DeleteS3Object.py =>
PutKinesisStream.py} (78%)
copy docker/test/integration/minifi/processors/{PutOPCProcessor.py =>
RunLlamaCppInference.py} (88%)
delete mode 100644 docker/test/integration/resources/kafka_broker/Dockerfile
delete mode 100644
docker/test/integration/resources/kafka_broker/conf/server.properties
create mode 100644 docker/test/integration/resources/kinesis-mock/Dockerfile
create mode 100644
docker/test/integration/resources/kinesis-mock/consumer/consumer.js
create mode 100644
docker/test/integration/resources/kinesis-mock/consumer/package.json
create mode 100644 docker/test/integration/resources/kinesis-mock/server.json
copy extensions/aws/processors/{S3Processor.cpp => AwsProcessor.cpp} (69%)
copy extensions/aws/processors/{S3Processor.h => AwsProcessor.h} (88%)
create mode 100644 extensions/aws/processors/PutKinesisStream.cpp
create mode 100644 extensions/aws/processors/PutKinesisStream.h
create mode 100644 extensions/aws/tests/PutKinesisStreamTests.cpp
copy minifi-api/include/minifi-cpp/core/ValidationResult.h =>
extensions/aws/utils/ProxyOptions.h (78%)
copy extensions/{mqtt => llamacpp}/CMakeLists.txt (62%)
create mode 100644 extensions/llamacpp/processors/DefaultLlamaContext.cpp
create mode 100644 extensions/llamacpp/processors/DefaultLlamaContext.h
copy extensions/{rocksdb-repos/database/ColumnHandle.cpp =>
llamacpp/processors/LlamaBackendInitializer.cpp} (69%)
copy libminifi/test/libtest/integration/CivetLibrary.h =>
extensions/llamacpp/processors/LlamaBackendInitializer.h (57%)
create mode 100644 extensions/llamacpp/processors/LlamaContext.h
create mode 100644 extensions/llamacpp/processors/RunLlamaCppInference.cpp
create mode 100644 extensions/llamacpp/processors/RunLlamaCppInference.h
copy extensions/{kafka => llamacpp}/tests/CMakeLists.txt (63%)
create mode 100644 extensions/llamacpp/tests/RunLlamaCppInferenceTests.cpp
create mode 100644 libminifi/include/core/ReferenceParser.h
create mode 100644
libminifi/src/core/controller/StandardControllerServiceProvider.cpp
copy libminifi/src/utils/{ParsingErrors.cpp => AttributeErrors.cpp} (77%)
create mode 100644 libminifi/test/integration/C2ControllerEnableFailureTest.cpp
copy libminifi/test/resources/{TestOnScheduleRetry.yml =>
TestC2InvalidController.yml} (67%)
copy libminifi/test/resources/{TestOnScheduleRetry.yml =>
TestC2ValidController.yml} (67%)
create mode 100644 libminifi/test/unit/AssetResolutionTests.cpp
create mode 100644 thirdparty/llamacpp/lu8_macro_fix.patch
copy utils/include/utils/{ParsingErrors.h => AttributeErrors.h} (63%)
