This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 7e7984b94ab61526b8b41e400744311f46a3ad63 Author: Ferenc Gerlits <[email protected]> AuthorDate: Mon Oct 2 13:57:13 2023 +0200 MINIFICPP-2221 Auto-generate CONTROLLERS.md Also: * simplify the auto-generation of PROCESSORS.md: don't create individual files for each processor * add missing controller services to CONTROLLERS.md * move some descriptions from CONTROLLERS.md to the Description of the controller services * fix some Visual Studio warnings and a bug in formatAllowedValues(). Closes #1665 Signed-off-by: Marton Szasz <[email protected]> --- CONTROLLERS.md | 297 +++++++++++++++++---- PROCESSORS.md | 244 ++++++++--------- .../aws/controllerservices/AWSCredentialsService.h | 4 +- .../AzureStorageCredentialsService.h | 3 +- .../GCPCredentialsControllerService.h | 3 +- libminifi/include/controllers/SSLContextService.h | 10 +- libminifi/include/core/PropertyDefinition.h | 4 +- libminifi/include/core/PropertyType.h | 16 +- minifi_main/AgentDocs.cpp | 234 ++++++++-------- minifi_main/AgentDocs.h | 9 +- minifi_main/MiNiFiMain.cpp | 28 +- 11 files changed, 525 insertions(+), 327 deletions(-) diff --git a/CONTROLLERS.md b/CONTROLLERS.md index 0939a8846..3468929de 100644 --- a/CONTROLLERS.md +++ b/CONTROLLERS.md @@ -1,46 +1,53 @@ <!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. --> -# Controller Services - ## Table of Contents - [AWSCredentialsService](#AWSCredentialsService) - [AzureStorageCredentialsService](#AzureStorageCredentialsService) -- [GCPCredentialsControllerService](#GCPCredentialsControllerService) - [ElasticsearchCredentialsControllerService](#ElasticsearchCredentialsControllerService) -- [KubernetesControllerService](#kubernetesControllerService) +- [ExecuteJavaControllerService](#ExecuteJavaControllerService) +- [GCPCredentialsControllerService](#GCPCredentialsControllerService) +- [JavaControllerService](#JavaControllerService) +- [KubernetesControllerService](#KubernetesControllerService) +- [LinuxPowerManagerService](#LinuxPowerManagerService) +- [NetworkPrioritizerService](#NetworkPrioritizerService) +- [ODBCService](#ODBCService) +- [PersistentMapStateStorage](#PersistentMapStateStorage) +- [RocksDbStateStorage](#RocksDbStateStorage) +- [SmbConnectionControllerService](#SmbConnectionControllerService) +- [SSLContextService](#SSLContextService) +- [UpdatePolicyControllerService](#UpdatePolicyControllerService) +- [VolatileMapStateStorage](#VolatileMapStateStorage) + ## AWSCredentialsService ### Description -Manages the Amazon Web Services (AWS) credentials for an AWS account. This allows for multiple -AWS credential services to be defined. This also allows for multiple AWS related processors to reference this single -controller service so that AWS credentials can be managed and controlled in a central location. +Manages the Amazon Web Services (AWS) credentials for an AWS account. This allows for multiple AWS credential services to be defined. This also allows for multiple AWS related processors to reference this single controller service so that AWS credentials can be managed and controlled in a central location. ### Properties -In the list below, the names of required properties appear in bold. Any other -properties (not in bold) are considered optional. +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. | Name | Default Value | Allowable Values | Description | |-----------------------------|---------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------------| -| **Use Default Credentials** | false | | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. | -| Access Key | | | Specifies the AWS Access Key | -| Secret Key | | | Specifies the AWS Secret Key | +| **Use Default Credentials** | false | true<br/>false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. | +| Access Key | | | Specifies the AWS Access Key. | +| Secret Key | | | Specifies the AWS Secret Key. | | Credentials File | | | Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey | @@ -48,13 +55,11 @@ properties (not in bold) are considered optional. ### Description -Manages the credentials for an Azure Storage account. This allows for multiple Azure Storage related processors to reference this single -controller service so that Azure storage credentials can be managed and controlled in a central location. +Manages the credentials for an Azure Storage account. This allows for multiple Azure Storage related processors to reference this single controller service so that Azure storage credentials can be managed and controlled in a central location. ### Properties -In the list below, the names of required properties appear in bold. Any other -properties (not in bold) are considered optional. +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. | Name | Default Value | Allowable Values | Description | |----------------------------------------|---------------|------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -63,59 +68,251 @@ properties (not in bold) are considered optional. | SAS Token | | | Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Managed Identity is not used. | | Common Storage Account Endpoint Suffix | | | Storage accounts in public Azure always use a common FQDN suffix. Override this endpoint suffix with a different suffix in certain circumstances (like Azure Stack or non-public Azure regions). | | Connection String | | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Managed Identity is not used. | -| **Use Managed Identity Credentials** | false | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties. | +| **Use Managed Identity Credentials** | false | true<br/>false | If true Managed Identity credentials will be used together with the Storage Account Name for authentication. | -## GCPCredentialsControllerService + +## ElasticsearchCredentialsControllerService ### Description -Manages the credentials for Google Cloud Platform. This allows for multiple Google Cloud Platform related processors to reference this single -controller service so that Google Cloud Platform credentials can be managed and controlled in a central location. +Elasticsearch/Opensearch Credentials Controller Service ### Properties -In the list below, the names of required properties appear in bold. Any other -properties (not in bold) are considered optional. +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. +| Name | Default Value | Allowable Values | Description | +|----------|---------------|------------------|----------------------------------------------------------------------------------| +| Username | | | The username for basic authentication<br/>**Supports Expression Language: true** | +| Password | | | The password for basic authentication<br/>**Supports Expression Language: true** | +| API Key | | | The API Key to use | -| Name | Default Value | Allowable Values | Description | -|---------------------------|----------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------| -| **Credentials Location** | Google Application Default Credentials | Google Application Default Credentials<br> Use Compute Engine Credentials<br>Service Account JSON File<br>Service Account JSON<br>Use Anonymous credentials | The location of the credentials. | -| Service Account JSON File | | | Path to a file containing a Service Account key file in JSON format. | -| Service Account JSON | | | The raw JSON containing a Service Account keyfile. | +## ExecuteJavaControllerService -## ElasticsearchCredentialsControllerService +### Description + +ExecuteJavaClass runs NiFi Controller services given a provided system path + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-----------------------------|---------------|------------------|-------------------------------------------------| +| **NiFi Controller Service** | | | Name of NiFi Controller Service to load and run | + + +## GCPCredentialsControllerService ### Description -Elasticsearch Credentials Controller Service +Manages the credentials for Google Cloud Platform. This allows for multiple Google Cloud Platform related processors to reference this single controller service so that Google Cloud Platform credentials can be managed and controlled in a central location. ### Properties -In the list below, the names of required properties appear in bold. Any other -properties (not in bold) are considered optional. +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. +| Name | Default Value | Allowable Values | Description | +|---------------------------|----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------| +| **Credentials Location** | Google Application Default Credentials | Google Application Default Credentials<br/>Use Compute Engine Credentials<br/>Service Account JSON File<br/>Service Account JSON<br/>Use Anonymous credentials | The location of the credentials. | +| Service Account JSON File | | | Path to a file containing a Service Account key file in JSON format. | +| Service Account JSON | | | The raw JSON containing a Service Account keyfile. | -| Name | Default Value | Allowable Values | Description | -|------------------------|---------------|----------------------------------|---------------------------------------| -| Username | | | The username for basic authentication | -| Password | | | The password for basic authentication | -| API Key | | | The API Key to use | + +## JavaControllerService + +### Description + +Allows specification of nars to be used within referenced processors. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------------------------------|---------------|------------------|-----------------------------------------------| +| **Nar Directory** | | | Directory containing the nars to deploy | +| **Nar Deployment Directory** | | | Directory in which nars will be deployed | +| **Nar Document Directory** | | | Directory in which documents will be deployed | ## KubernetesControllerService ### Description -Controller service that provides access to the Kubernetes API. +Controller service that provides access to the Kubernetes API ### Properties -In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. | Name | Default Value | Allowable Values | Description | |-----------------------|---------------|------------------|----------------------------------------------------------------------------------------------| | Namespace Filter | default | | Limit the output to pods in namespaces which match this regular expression | | Pod Name Filter | | | If present, limit the output to pods the name of which matches this regular expression | | Container Name Filter | | | If present, limit the output to containers the name of which matches this regular expression | + + +## LinuxPowerManagerService + +### Description + +Linux power management service that enables control of power usage in the agent through Linux power management information. Use name "ThreadPoolManager" to throttle battery consumption + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------------------------------|---------------------------------------|------------------|--------------------------------------------------------------------------------------------| +| **Battery Capacity Path** | /sys/class/power_supply/BAT0/capacity | | Path to the battery level | +| **Battery Status Path** | /sys/class/power_supply/BAT0/status | | Path to the battery status ( Discharging/Battery ) | +| **Battery Status Discharge** | Discharging | | Keyword to identify if battery is discharging | +| **Trigger Threshold** | 75 | | Battery threshold before which we consider a slow reduction. Should be a number from 1-100 | +| **Low Battery Threshold** | 50 | | Battery threshold before which we will aggressively reduce. Should be a number from 1-100 | +| **Wait Period** | 100 ms | | Decay between checking threshold and determining if a reduction is needed | + + +## NetworkPrioritizerService + +### Description + +Enables selection of networking interfaces on defined parameters to include output and payload size + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-----------------------|---------------|------------------|---------------------------------------------------------------------------------------| +| Network Controllers | | | Comma separated list of network controllers in order of priority for this prioritizer | +| **Max Throughput** | 1 MB | | Max throughput ( per second ) for these network controllers | +| **Max Payload** | 1 GB | | Maximum payload for these network controllers | +| **Verify Interfaces** | true | true<br/>false | Verify that interfaces are operational | +| Default Prioritizer | false | true<br/>false | Sets this controller service as the default prioritizer for all comms | + + +## ODBCService + +### Description + +Controller service that provides ODBC database connection + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-----------------------|---------------|------------------|----------------------------| +| **Connection String** | | | Database Connection String | + + +## PersistentMapStateStorage + +### Description + +A persistable state storage service implemented by a locked std::unordered_map<std::string, std::string> and persisted into a file + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|---------------------------|---------------|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| +| Always Persist | false | true<br/>false | Persist every change instead of persisting it periodically. | +| Auto Persistence Interval | 1 min | | The interval of the periodic task persisting all values. Only used if Always Persist is false. If set to 0 seconds, auto persistence will be disabled. | +| **File** | | | Path to a file to store state | + + +## RocksDbStateStorage + +### Description + +A state storage service implemented by RocksDB + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|---------------------------|---------------|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| +| Always Persist | false | true<br/>false | Persist every change instead of persisting it periodically. | +| Auto Persistence Interval | 1 min | | The interval of the periodic task persisting all values. Only used if Always Persist is false. If set to 0 seconds, auto persistence will be disabled. | +| **Directory** | | | Path to a directory for the database | + + +## SmbConnectionControllerService + +### Description + +SMB Connection Controller Service + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------|---------------|------------------|---------------------------------------------------------------------------------------------------------------------------------| +| **Hostname** | | | The network host to which files should be written. | +| **Share** | | | The network share to which files should be written. This is the "first folder" after the hostname: \\hostname\[share]\dir1\dir2 | +| Domain | | | The domain used for authentication. Optional, in most cases username and password is sufficient. | +| Username | | | The username used for authentication. If no username is set then anonymous authentication is attempted. | +| Password | | | The password used for authentication. Required if Username is set. | + + +## SSLContextService + +### Description + +Controller service that provides SSL/TLS capabilities to consuming interfaces + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|----------------------------|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------| +| Certificate Store Location | LocalMachine | CurrentUser<br/>LocalMachine<br/>CurrentService<br/>Services<br/>Users<br/>CurrentUserGroupPolicy<br/>LocalMachineGroupPolicy<br/>LocalMachineEnterprise | One of the Windows certificate store locations, eg. LocalMachine or CurrentUser (Windows only) | +| Server Cert Store | ROOT | | The name of the certificate store which contains the server certificate (Windows only) | +| Client Cert Store | MY | | The name of the certificate store which contains the client certificate (Windows only) | +| Client Cert CN | | | The CN that the client certificate is required to match; default: use the first available client certificate in the store (Windows only) | +| Client Cert Key Usage | Client Authentication | | Comma-separated list of enhanced key usage values that the client certificate is required to have (Windows only) | +| Client Certificate | | | Client Certificate | +| Private Key | | | Private Key file | +| Passphrase | | | Client passphrase. Either a file or unencrypted text | +| CA Certificate | | | CA certificate file | +| Use System Cert Store | false | true<br/>false | Whether to use the certificates in the OS's certificate store | + + +## UpdatePolicyControllerService + +### Description + +UpdatePolicyControllerService allows a flow specific policy on allowing or disallowing updates. Since the flow dictates the purpose of a device it will also be used to dictate updates to specific components. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-----------------------|---------------|------------------|-----------------------------------------------------------------------| +| Allow All Properties | false | true<br/>false | Allows all properties, which are also not disallowed, to be updated | +| Persist Updates | false | true<br/>false | Property that dictates whether updates should persist after a restart | +| Allowed Properties | | | Properties for which we will allow updates | +| Disallowed Properties | | | Properties for which we will not allow updates | + + +## VolatileMapStateStorage + +### Description + +A key-value service implemented by a locked std::unordered_map<std::string, std::string> + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-----------------|---------------|------------------|--------------------------------| +| Linked Services | | | Referenced Controller Services | diff --git a/PROCESSORS.md b/PROCESSORS.md index 6da9203df..940f0163f 100644 --- a/PROCESSORS.md +++ b/PROCESSORS.md @@ -15,91 +15,91 @@ limitations under the License. ## Table of Contents -- [AppendHostInfo](#appendhostinfo) -- [ApplyTemplate](#applytemplate) -- [AttributesToJSON](#attributestojson) -- [BinFiles](#binfiles) -- [CapturePacket](#capturepacket) -- [CaptureRTSPFrame](#capturertspframe) -- [CollectKubernetesPodMetrics](#collectkubernetespodmetrics) -- [CollectorInitiatedSubscription](#collectorinitiatedsubscription) -- [CompressContent](#compresscontent) -- [ConsumeJournald](#consumejournald) -- [ConsumeKafka](#consumekafka) -- [ConsumeMQTT](#consumemqtt) -- [ConsumeWindowsEventLog](#consumewindowseventlog) -- [DefragmentText](#defragmenttext) -- [DeleteAzureBlobStorage](#deleteazureblobstorage) -- [DeleteAzureDataLakeStorage](#deleteazuredatalakestorage) -- [DeleteGCSObject](#deletegcsobject) -- [DeleteS3Object](#deletes3object) -- [ExecuteJavaProcessor](#executejavaprocessor) -- [ExecuteProcess](#executeprocess) -- [ExecutePythonProcessor](#executepythonprocessor) -- [ExecuteScript](#executescript) -- [ExecuteSQL](#executesql) -- [ExtractText](#extracttext) -- [FetchAzureBlobStorage](#fetchazureblobstorage) -- [FetchAzureDataLakeStorage](#fetchazuredatalakestorage) -- [FetchFile](#fetchfile) -- [FetchGCSObject](#fetchgcsobject) -- [FetchOPCProcessor](#fetchopcprocessor) -- [FetchS3Object](#fetchs3object) -- [FetchSFTP](#fetchsftp) -- [FetchSmb](#fetchsmb) -- [FocusArchiveEntry](#focusarchiveentry) -- [GenerateFlowFile](#generateflowfile) -- [GetEnvironmentalSensors](#getenvironmentalsensors) -- [GetFile](#getfile) -- [GetGPS](#getgps) -- [GetMovementSensors](#getmovementsensors) -- [GetTCP](#gettcp) -- [GetUSBCamera](#getusbcamera) -- [HashContent](#hashcontent) -- [InvokeHTTP](#invokehttp) -- [ListAzureBlobStorage](#listazureblobstorage) -- [ListAzureDataLakeStorage](#listazuredatalakestorage) -- [ListenHTTP](#listenhttp) -- [ListenSyslog](#listensyslog) -- [ListenTCP](#listentcp) -- [ListenUDP](#listenudp) -- [ListFile](#listfile) -- [ListGCSBucket](#listgcsbucket) -- [ListS3](#lists3) -- [ListSFTP](#listsftp) -- [ListSmb](#listsmb) -- [LogAttribute](#logattribute) -- [ManipulateArchive](#manipulatearchive) -- [MergeContent](#mergecontent) -- [MotionDetector](#motiondetector) -- [PerformanceDataMonitor](#performancedatamonitor) -- [PostElasticsearch](#postelasticsearch) -- [ProcFsMonitor](#procfsmonitor) -- [PublishKafka](#publishkafka) -- [PublishMQTT](#publishmqtt) -- [PutAzureBlobStorage](#putazureblobstorage) -- [PutAzureDataLakeStorage](#putazuredatalakestorage) -- [PutFile](#putfile) -- [PutGCSObject](#putgcsobject) -- [PutOPCProcessor](#putopcprocessor) -- [PutS3Object](#puts3object) -- [PutSFTP](#putsftp) -- [PutSmb](#putsmb) -- [PutSplunkHTTP](#putsplunkhttp) -- [PutSQL](#putsql) -- [PutTCP](#puttcp) -- [PutUDP](#putudp) -- [QueryDatabaseTable](#querydatabasetable) -- [QuerySplunkIndexingStatus](#querysplunkindexingstatus) -- [ReplaceText](#replacetext) -- [RetryFlowFile](#retryflowfile) -- [RouteOnAttribute](#routeonattribute) -- [RouteText](#routetext) -- [SourceInitiatedSubscriptionListener](#sourceinitiatedsubscriptionlistener) -- [TailEventLog](#taileventlog) -- [TailFile](#tailfile) -- [UnfocusArchiveEntry](#unfocusarchiveentry) -- [UpdateAttribute](#updateattribute) +- [AppendHostInfo](#AppendHostInfo) +- [ApplyTemplate](#ApplyTemplate) +- [AttributesToJSON](#AttributesToJSON) +- [BinFiles](#BinFiles) +- [CapturePacket](#CapturePacket) +- [CaptureRTSPFrame](#CaptureRTSPFrame) +- [CollectKubernetesPodMetrics](#CollectKubernetesPodMetrics) +- [CollectorInitiatedSubscription](#CollectorInitiatedSubscription) +- [CompressContent](#CompressContent) +- [ConsumeJournald](#ConsumeJournald) +- [ConsumeKafka](#ConsumeKafka) +- [ConsumeMQTT](#ConsumeMQTT) +- [ConsumeWindowsEventLog](#ConsumeWindowsEventLog) +- [DefragmentText](#DefragmentText) +- [DeleteAzureBlobStorage](#DeleteAzureBlobStorage) +- [DeleteAzureDataLakeStorage](#DeleteAzureDataLakeStorage) +- [DeleteGCSObject](#DeleteGCSObject) +- [DeleteS3Object](#DeleteS3Object) +- [ExecuteJavaProcessor](#ExecuteJavaProcessor) +- [ExecuteProcess](#ExecuteProcess) +- [ExecutePythonProcessor](#ExecutePythonProcessor) +- [ExecuteScript](#ExecuteScript) +- [ExecuteSQL](#ExecuteSQL) +- [ExtractText](#ExtractText) +- [FetchAzureBlobStorage](#FetchAzureBlobStorage) +- [FetchAzureDataLakeStorage](#FetchAzureDataLakeStorage) +- [FetchFile](#FetchFile) +- [FetchGCSObject](#FetchGCSObject) +- [FetchOPCProcessor](#FetchOPCProcessor) +- [FetchS3Object](#FetchS3Object) +- [FetchSFTP](#FetchSFTP) +- [FetchSmb](#FetchSmb) +- [FocusArchiveEntry](#FocusArchiveEntry) +- [GenerateFlowFile](#GenerateFlowFile) +- [GetEnvironmentalSensors](#GetEnvironmentalSensors) +- [GetFile](#GetFile) +- [GetGPS](#GetGPS) +- [GetMovementSensors](#GetMovementSensors) +- [GetTCP](#GetTCP) +- [GetUSBCamera](#GetUSBCamera) +- [HashContent](#HashContent) +- [InvokeHTTP](#InvokeHTTP) +- [ListAzureBlobStorage](#ListAzureBlobStorage) +- [ListAzureDataLakeStorage](#ListAzureDataLakeStorage) +- [ListenHTTP](#ListenHTTP) +- [ListenSyslog](#ListenSyslog) +- [ListenTCP](#ListenTCP) +- [ListenUDP](#ListenUDP) +- [ListFile](#ListFile) +- [ListGCSBucket](#ListGCSBucket) +- [ListS3](#ListS3) +- [ListSFTP](#ListSFTP) +- [ListSmb](#ListSmb) +- [LogAttribute](#LogAttribute) +- [ManipulateArchive](#ManipulateArchive) +- [MergeContent](#MergeContent) +- [MotionDetector](#MotionDetector) +- [PerformanceDataMonitor](#PerformanceDataMonitor) +- [PostElasticsearch](#PostElasticsearch) +- [ProcFsMonitor](#ProcFsMonitor) +- [PublishKafka](#PublishKafka) +- [PublishMQTT](#PublishMQTT) +- [PutAzureBlobStorage](#PutAzureBlobStorage) +- [PutAzureDataLakeStorage](#PutAzureDataLakeStorage) +- [PutFile](#PutFile) +- [PutGCSObject](#PutGCSObject) +- [PutOPCProcessor](#PutOPCProcessor) +- [PutS3Object](#PutS3Object) +- [PutSFTP](#PutSFTP) +- [PutSmb](#PutSmb) +- [PutSplunkHTTP](#PutSplunkHTTP) +- [PutSQL](#PutSQL) +- [PutTCP](#PutTCP) +- [PutUDP](#PutUDP) +- [QueryDatabaseTable](#QueryDatabaseTable) +- [QuerySplunkIndexingStatus](#QuerySplunkIndexingStatus) +- [ReplaceText](#ReplaceText) +- [RetryFlowFile](#RetryFlowFile) +- [RouteOnAttribute](#RouteOnAttribute) +- [RouteText](#RouteText) +- [SourceInitiatedSubscriptionListener](#SourceInitiatedSubscriptionListener) +- [TailEventLog](#TailEventLog) +- [TailFile](#TailFile) +- [UnfocusArchiveEntry](#UnfocusArchiveEntry) +- [UpdateAttribute](#UpdateAttribute) ## AppendHostInfo @@ -860,14 +860,14 @@ Reads the contents of a file from disk and streams it into the contents of an in In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. -| Name | Default Value | Allowable Values | Description | -|--------------------------------------|---------------|-----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| File to Fetch | | | The fully-qualified filename of the file to fetch from the file system. If not defined the default ${absolute.path}/${filename} path is used.<br/>**Supports Expression Language: true** | -| **Completion Strategy** | None | None<br/>Move File<br/>Delete File | Specifies what to do with the original file on the file system once it has been pulled into MiNiFi | -| Move Destination Directory | | | The directory to move the original file to once it has been fetched from the file system. This property is ignored unless the Completion Strategy is set to "Move File". If the directory does not exist, it will be created.<br/>**Supports Expression Language: true** | -| **Move Conflict Strategy** | Rename | Rename<br/>Replace File<br/>Keep Existing<br/>Fail | If Completion Strategy is set to Move File and a file already exists in the destination directory with the same name, this property specifies how that naming conflict should be resolved | -| **Log level when file not found** | ERROR | TRACE<br/>DEBUG<br/>INFO<br/>WARN<br/>ERROR<br/>OFF | Log level to use in case the file does not exist when the processor is triggered | -| **Log level when permission denied** | ERROR | TRACE<br/>DEBUG<br/>INFO<br/>WARN<br/>ERROR<br/>OFF | Log level to use in case agent does not have sufficient permissions to read the file | +| Name | Default Value | Allowable Values | Description | +|--------------------------------------|---------------|------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| File to Fetch | | | The fully-qualified filename of the file to fetch from the file system. If not defined the default ${absolute.path}/${filename} path is used.<br/>**Supports Expression Language: true** | +| **Completion Strategy** | None | None<br/>Move File<br/>Delete File | Specifies what to do with the original file on the file system once it has been pulled into MiNiFi | +| Move Destination Directory | | | The directory to move the original file to once it has been fetched from the file system. This property is ignored unless the Completion Strategy is set to "Move File". If the directory does not exist, it will be created.<br/>**Supports Expression Language: true** | +| **Move Conflict Strategy** | Rename | Rename<br/>Replace File<br/>Keep Existing<br/>Fail | If Completion Strategy is set to Move File and a file already exists in the destination directory with the same name, this property specifies how that naming conflict should be resolved | +| **Log level when file not found** | ERROR | TRACE<br/>DEBUG<br/>INFO<br/>WARN<br/>ERROR<br/>CRITICAL<br/>OFF | Log level to use in case the file does not exist when the processor is triggered | +| **Log level when permission denied** | ERROR | TRACE<br/>DEBUG<br/>INFO<br/>WARN<br/>ERROR<br/>CRITICAL<br/>OFF | Log level to use in case agent does not have sufficient permissions to read the file | ### Relationships @@ -1042,7 +1042,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |---------------------------------------|---------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **SMB Connection Controller Service** | | | Specifies the SMB connection controller service to use for connecting to the SMB server. | +| **SMB Connection Controller Service** | | | Specifies the SMB connection controller service to use for connecting to the SMB server. If the SMB share is auto-mounted to a drive letter, its recommended to use FetchFile instead. | | Input Directory | | | The full path of the file to be retrieved from the remote server. If left empty, the path and filename attributes of the incoming flow file will be used.<br/>**Supports Expression Language: true** | ### Relationships @@ -1732,18 +1732,18 @@ Retrieves a listing of files from an SMB share. For each file that is listed, cr In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. -| Name | Default Value | Allowable Values | Description | -|---------------------------------------|---------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **SMB Connection Controller Service** | | | Specifies the SMB connection controller service to use for connecting to the SMB server. | -| Input Directory | | | The input directory to list the contents of | -| **Recurse Subdirectories** | true | | Indicates whether to list files from subdirectories of the directory | -| File Filter | | | Only files whose names match the given regular expression will be picked up | -| Path Filter | | | When Recurse Subdirectories is true, then only subdirectories whose path matches the given regular expression will be scanned | -| **Minimum File Age** | 5 sec | | The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored | -| Maximum File Age | | | The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored | -| Minimum File Size | | | The minimum size that a file must be in order to be pulled | -| Maximum File Size | | | The maximum size that a file can be in order to be pulled | -| **Ignore Hidden Files** | true | | Indicates whether or not hidden files should be ignored | +| Name | Default Value | Allowable Values | Description | +|---------------------------------------|---------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **SMB Connection Controller Service** | | | Specifies the SMB connection controller service to use for connecting to the SMB server. If the SMB share is auto-mounted to a drive letter, its recommended to use ListFile instead. | +| Input Directory | | | The input directory to list the contents of | +| **Recurse Subdirectories** | true | true<br/>false | Indicates whether to list files from subdirectories of the directory | +| File Filter | | | Only files whose names match the given regular expression will be picked up | +| Path Filter | | | When Recurse Subdirectories is true, then only subdirectories whose path matches the given regular expression will be scanned | +| **Minimum File Age** | 5 sec | | The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored | +| Maximum File Age | | | The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored | +| Minimum File Size | | | The minimum size that a file must be in order to be pulled | +| Maximum File Size | | | The maximum size that a file can be in order to be pulled | +| **Ignore Hidden Files** | true | true<br/>false | Indicates whether or not hidden files should be ignored | ### Relationships @@ -1753,15 +1753,15 @@ In the list below, the names of required properties appear in bold. Any other pr ### Output Attributes -| Attribute | Relationship | Description | -|------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| filename | success | The name of the file that was read from filesystem. | -| path | success | The path is set to the relative path of the file's directory on the remote filesystem compared to the Share root directory. For example, for a given remote location smb://HOSTNAME:PORT/SHARE/DIRECTORY, and a file is being listed from smb://HOSTNAME:PORT/SHARE/DIRECTORY/sub/folder/file then the path attribute will be set to "sub/folder". | -| serviceLocation | success | The SMB URL of the share. | -| lastModifiedTime | success | The timestamp of when the file's content changed in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'. | -| creationTime | success | The timestamp of when the file was created in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'. | -| lastAccessTime | success | The timestamp of when the file was accessed in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'. | -| size | success | The size of the file in bytes. | +| Attribute | Relationship | Description | +|------------------|--------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| filename | success | The name of the file that was read from filesystem. | +| path | success | The path is set to the relative path of the file's directory on the remote filesystem compared to the Share root directory. For example, for a given remote location smb://HOSTNAME:PORT/SHARE/DIRECTORY, and a file is being listed from smb://HOSTNAME:PORT/SHARE/DIRECTORY/sub/folder/file then the path attribute will be set to "DIRECTORY/sub/folder". | +| serviceLocation | success | The SMB URL of the share. | +| lastModifiedTime | success | The timestamp of when the file's content changed in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'. | +| creationTime | success | The timestamp of when the file was created in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'. | +| lastAccessTime | success | The timestamp of when the file was accessed in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'. | +| size | success | The size of the file in bytes. | ## LogAttribute @@ -2227,7 +2227,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Permissions | | | Sets the permissions on the output file to the value of this attribute. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems. | | Directory Permissions | | | Sets the permissions on the directories being created if 'Create Missing Directories' property is set. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems. | | Directory | . | | The output directory to which to put files<br/>**Supports Expression Language: true** | -| Conflict Resolution Strategy | fail | fail<br/>ignore<br/>replace | Indicates what should happen when a file with the same name already exists in the output directory | +| Conflict Resolution Strategy | fail | fail<br/>replace<br/>ignore | Indicates what should happen when a file with the same name already exists in the output directory | | **Create Missing Directories** | true | | If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure. | | Maximum File Count | -1 | | Specifies the maximum number of files that can exist in the output directory | @@ -2445,12 +2445,12 @@ Writes the contents of a FlowFile to an smb network location In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. -| Name | Default Value | Allowable Values | Description | -|---------------------------------------|---------------|-----------------------------|-----------------------------------------------------------------------------------------------------------------------| -| **SMB Connection Controller Service** | | | Specifies the SMB connection controller service to use for connecting to the SMB server. | -| Directory | . | | The output directory to which to put files<br/>**Supports Expression Language: true** | -| Conflict Resolution Strategy | fail | fail<br/>replace<br/>ignore | Indicates what should happen when a file with the same name already exists in the output directory | -| **Create Missing Directories** | true | | If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure. | +| Name | Default Value | Allowable Values | Description | +|---------------------------------------|---------------|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **SMB Connection Controller Service** | | | Specifies the SMB connection controller service to use for connecting to the SMB server. If the SMB share is auto-mounted to a drive letter, its recommended to use PutFile instead. | +| Directory | . | | The output directory to which to put files<br/>**Supports Expression Language: true** | +| Conflict Resolution Strategy | fail | fail<br/>replace<br/>ignore | Indicates what should happen when a file with the same name already exists in the output directory | +| **Create Missing Directories** | true | | If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure. | ### Relationships diff --git a/extensions/aws/controllerservices/AWSCredentialsService.h b/extensions/aws/controllerservices/AWSCredentialsService.h index 46049572f..5b2beaf22 100644 --- a/extensions/aws/controllerservices/AWSCredentialsService.h +++ b/extensions/aws/controllerservices/AWSCredentialsService.h @@ -47,7 +47,9 @@ class AWSCredentialsService : public core::controller::ControllerService { : ControllerService(name) { } - EXTENSIONAPI static constexpr const char* Description = "AWS Credentials Management Service"; + EXTENSIONAPI static constexpr const char* Description = "Manages the Amazon Web Services (AWS) credentials for an AWS account. This allows for multiple " + "AWS credential services to be defined. This also allows for multiple AWS related processors to reference this single " + "controller service so that AWS credentials can be managed and controlled in a central location."; EXTENSIONAPI static constexpr auto UseDefaultCredentials = core::PropertyDefinitionBuilder<>::createProperty("Use Default Credentials") .withDescription("If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc.") diff --git a/extensions/azure/controllerservices/AzureStorageCredentialsService.h b/extensions/azure/controllerservices/AzureStorageCredentialsService.h index 968be0aa9..4ab86fab1 100644 --- a/extensions/azure/controllerservices/AzureStorageCredentialsService.h +++ b/extensions/azure/controllerservices/AzureStorageCredentialsService.h @@ -34,7 +34,8 @@ namespace org::apache::nifi::minifi::azure::controllers { class AzureStorageCredentialsService : public core::controller::ControllerService { public: - EXTENSIONAPI static constexpr const char* Description = "Azure Storage Credentials Management Service"; + EXTENSIONAPI static constexpr const char* Description = "Manages the credentials for an Azure Storage account. This allows for multiple Azure Storage related processors to reference this single " + "controller service so that Azure storage credentials can be managed and controlled in a central location."; EXTENSIONAPI static constexpr auto StorageAccountName = core::PropertyDefinitionBuilder<>::createProperty("Storage Account Name") .withDescription("The storage account name.") diff --git a/extensions/gcp/controllerservices/GCPCredentialsControllerService.h b/extensions/gcp/controllerservices/GCPCredentialsControllerService.h index a4f217b31..838841525 100644 --- a/extensions/gcp/controllerservices/GCPCredentialsControllerService.h +++ b/extensions/gcp/controllerservices/GCPCredentialsControllerService.h @@ -64,7 +64,8 @@ namespace org::apache::nifi::minifi::extensions::gcp { class GCPCredentialsControllerService : public core::controller::ControllerService { public: - EXTENSIONAPI static constexpr const char* Description = "Google Cloud Platform Credentials Controller Service"; + EXTENSIONAPI static constexpr const char* Description = "Manages the credentials for Google Cloud Platform. This allows for multiple Google Cloud Platform related processors " + "to reference this single controller service so that Google Cloud Platform credentials can be managed and controlled in a central location."; EXTENSIONAPI static constexpr auto CredentialsLoc = core::PropertyDefinitionBuilder<magic_enum::enum_count<CredentialsLocation>()>::createProperty("Credentials Location") .withDescription("The location of the credentials.") diff --git a/libminifi/include/controllers/SSLContextService.h b/libminifi/include/controllers/SSLContextService.h index 1c2c74a41..c3523cd8d 100644 --- a/libminifi/include/controllers/SSLContextService.h +++ b/libminifi/include/controllers/SSLContextService.h @@ -177,27 +177,27 @@ class SSLContextService : public core::controller::ControllerService { #ifdef WIN32 MINIFIAPI static constexpr auto CertStoreLocation = core::PropertyDefinitionBuilder<utils::tls::WindowsCertStoreLocation::SIZE>::createProperty("Certificate Store Location") - .withDescription("One of the Windows certificate store locations, eg. LocalMachine or CurrentUser") + .withDescription("One of the Windows certificate store locations, eg. LocalMachine or CurrentUser (Windows only)") .withAllowedValues(utils::tls::WindowsCertStoreLocation::LOCATION_NAMES) .isRequired(false) .withDefaultValue(utils::tls::WindowsCertStoreLocation::DEFAULT_LOCATION) .build(); MINIFIAPI static constexpr auto ServerCertStore = core::PropertyDefinitionBuilder<>::createProperty("Server Cert Store") - .withDescription("The name of the certificate store which contains the server certificate") + .withDescription("The name of the certificate store which contains the server certificate (Windows only)") .isRequired(false) .withDefaultValue("ROOT") .build(); MINIFIAPI static constexpr auto ClientCertStore = core::PropertyDefinitionBuilder<>::createProperty("Client Cert Store") - .withDescription("The name of the certificate store which contains the client certificate") + .withDescription("The name of the certificate store which contains the client certificate (Windows only)") .isRequired(false) .withDefaultValue("MY") .build(); MINIFIAPI static constexpr auto ClientCertCN = core::PropertyDefinitionBuilder<>::createProperty("Client Cert CN") - .withDescription("The CN that the client certificate is required to match; default: use the first available client certificate in the store") + .withDescription("The CN that the client certificate is required to match; default: use the first available client certificate in the store (Windows only)") .isRequired(false) .build(); MINIFIAPI static constexpr auto ClientCertKeyUsage = core::PropertyDefinitionBuilder<>::createProperty("Client Cert Key Usage") - .withDescription("Comma-separated list of enhanced key usage values that the client certificate is required to have") + .withDescription("Comma-separated list of enhanced key usage values that the client certificate is required to have (Windows only)") .isRequired(false) .withDefaultValue("Client Authentication") .build(); diff --git a/libminifi/include/core/PropertyDefinition.h b/libminifi/include/core/PropertyDefinition.h index 1af98de75..12caa61e8 100644 --- a/libminifi/include/core/PropertyDefinition.h +++ b/libminifi/include/core/PropertyDefinition.h @@ -39,7 +39,7 @@ struct PropertyDefinition { std::array<std::string_view, NumDependentProperties> dependent_properties; std::array<std::pair<std::string_view, std::string_view>, NumExclusiveOfProperties> exclusive_of_properties; std::optional<std::string_view> default_value; - gsl::not_null<const PropertyType*> type = gsl::make_not_null(&StandardPropertyTypes::VALID_TYPE); + gsl::not_null<const PropertyType*> type{gsl::make_not_null(&StandardPropertyTypes::VALID_TYPE)}; bool supports_expression_language = false; }; @@ -53,7 +53,7 @@ struct PropertyReference { std::span<const std::string_view> dependent_properties; std::span<const std::pair<std::string_view, std::string_view>> exclusive_of_properties; std::optional<std::string_view> default_value; - gsl::not_null<const PropertyType*> type = gsl::make_not_null(&StandardPropertyTypes::VALID_TYPE); + gsl::not_null<const PropertyType*> type{gsl::make_not_null(&StandardPropertyTypes::VALID_TYPE)}; bool supports_expression_language = false; constexpr PropertyReference() = default; diff --git a/libminifi/include/core/PropertyType.h b/libminifi/include/core/PropertyType.h index 1a75132e9..880e35fdf 100644 --- a/libminifi/include/core/PropertyType.h +++ b/libminifi/include/core/PropertyType.h @@ -335,21 +335,21 @@ inline constexpr auto LISTEN_PORT_TYPE = ListenPortValidator{}; inline gsl::not_null<const PropertyValidator*> getValidator(const std::shared_ptr<minifi::state::response::Value>& input) { if (std::dynamic_pointer_cast<core::DataSizeValue>(input) != nullptr) { - return gsl::make_not_null(&DATA_SIZE_TYPE); + return gsl::make_not_null<const PropertyValidator*>(&DATA_SIZE_TYPE); } else if (std::dynamic_pointer_cast<core::TimePeriodValue>(input) != nullptr) { - return gsl::make_not_null(&TIME_PERIOD_TYPE); + return gsl::make_not_null<const PropertyValidator*>(&TIME_PERIOD_TYPE); } else if (std::dynamic_pointer_cast<minifi::state::response::BoolValue>(input) != nullptr) { - return gsl::make_not_null(&BOOLEAN_TYPE); + return gsl::make_not_null<const PropertyValidator*>(&BOOLEAN_TYPE); } else if (std::dynamic_pointer_cast<minifi::state::response::IntValue>(input) != nullptr) { - return gsl::make_not_null(&INTEGER_TYPE); + return gsl::make_not_null<const PropertyValidator*>(&INTEGER_TYPE); } else if (std::dynamic_pointer_cast<minifi::state::response::UInt32Value>(input) != nullptr) { - return gsl::make_not_null(&UNSIGNED_INT_TYPE);; + return gsl::make_not_null<const PropertyValidator*>(&UNSIGNED_INT_TYPE);; } else if (std::dynamic_pointer_cast<minifi::state::response::Int64Value>(input) != nullptr) { - return gsl::make_not_null(&LONG_TYPE); + return gsl::make_not_null<const PropertyValidator*>(&LONG_TYPE); } else if (std::dynamic_pointer_cast<minifi::state::response::UInt64Value>(input) != nullptr) { - return gsl::make_not_null(&UNSIGNED_LONG_TYPE); + return gsl::make_not_null<const PropertyValidator*>(&UNSIGNED_LONG_TYPE); } else { - return gsl::make_not_null(&VALID_TYPE); + return gsl::make_not_null<const PropertyValidator*>(&VALID_TYPE); } } } // namespace StandardPropertyTypes diff --git a/minifi_main/AgentDocs.cpp b/minifi_main/AgentDocs.cpp index bd1c6a8d9..3257d997c 100644 --- a/minifi_main/AgentDocs.cpp +++ b/minifi_main/AgentDocs.cpp @@ -17,16 +17,17 @@ #include "AgentDocs.h" -#include <map> #include <iostream> #include <string> #include <utility> #include <vector> +#include "range/v3/algorithm.hpp" #include "range/v3/action/transform.hpp" +#include "range/v3/algorithm/lexicographical_compare.hpp" +#include "range/v3/range/conversion.hpp" #include "range/v3/view/transform.hpp" #include "range/v3/view/join.hpp" -#include "range/v3/range/conversion.hpp" #include "agent/agent_docs.h" #include "agent/agent_version.h" @@ -51,7 +52,7 @@ std::string formatName(std::string_view name_view, bool is_required) { } std::string formatAllowedValues(const minifi::core::Property& property) { - if (&property.getValidator() == &minifi::core::StandardPropertyTypes::BOOLEAN_TYPE) { + if (property.getValidator().getValidatorName() == minifi::core::StandardPropertyTypes::BOOLEAN_TYPE.getValidatorName()) { return "true<br/>false"; } else { const auto allowed_values = property.getAllowedValues(); @@ -64,7 +65,7 @@ std::string formatAllowedValues(const minifi::core::Property& property) { std::string formatDescription(std::string_view description_view, bool supports_expression_language = false) { std::string description{description_view}; - org::apache::nifi::minifi::utils::StringUtils::replaceAll(description, "\n", "<br/>"); + minifi::utils::StringUtils::replaceAll(description, "\n", "<br/>"); return supports_expression_language ? description + "<br/>**Supports Expression Language: true**" : description; } @@ -72,122 +73,137 @@ std::string formatListOfRelationships(std::span<const minifi::core::Relationship return minifi::utils::StringUtils::join(", ", relationships, [](const auto& relationship) { return relationship.name; }); } -} // namespace +inline constexpr std::string_view APACHE_LICENSE = R"license(<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +-->)license"; + +void writeHeader(std::ostream& docs, const std::vector<std::pair<std::string, minifi::ClassDescription>>& class_descriptions) { + docs << APACHE_LICENSE; + + docs << "\n\n## Table of Contents\n\n"; + for (const auto& [name, documentation] : class_descriptions) { + docs << "- [" << name << "](#" << name << ")\n"; + } +} -namespace org::apache::nifi::minifi::docs { +void writeName(std::ostream& docs, std::string_view name) { + docs << "\n\n## " << name; +} + +void writeDescription(std::ostream& docs, const minifi::ClassDescription& documentation) { + docs << "\n\n### Description\n\n"; + docs << documentation.description_; +} -std::string AgentDocs::extractClassName(const std::string &processor) { - auto positionOfLastDot = processor.find_last_of('.'); - if (positionOfLastDot != std::string::npos) { - return processor.substr(positionOfLastDot + 1); +void writeProperties(std::ostream& docs, const minifi::ClassDescription& documentation) { + docs << "\n\n### Properties"; + docs << "\n\nIn the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. " + << "The table also indicates any default values, and whether a property supports the NiFi Expression Language."; + minifi::docs::Table properties{{"Name", "Default Value", "Allowable Values", "Description"}}; + for (const auto &property : documentation.class_properties_) { + properties.addRow({ + formatName(property.getName(), property.getRequired()), + property.getDefaultValue().to_string(), + formatAllowedValues(property), + formatDescription(property.getDescription(), property.supportsExpressionLanguage()) + }); } - return processor; + docs << "\n\n" << properties.toString(); } -void AgentDocs::generate(const std::filesystem::path& docsdir, std::ostream &genStream) { - std::map<std::string, ClassDescription> processorSet; - for (const auto &group : minifi::AgentBuild::getExtensions()) { - struct Components descriptions = build_description_.getClassDescriptions(group); - for (const auto& processor_description : descriptions.processors_) { - processorSet.insert(std::make_pair(extractClassName(processor_description.full_name_), processor_description)); - } +void writeDynamicProperties(std::ostream& docs, const minifi::ClassDescription& documentation) { + if (documentation.dynamic_properties_.empty()) { return; } + + docs << "\n### Dynamic Properties\n\n"; + minifi::docs::Table dynamic_properties{{"Name", "Value", "Description"}}; + for (const auto &dynamic_property : documentation.dynamic_properties_) { + dynamic_properties.addRow({ + formatName(dynamic_property.name, false), + std::string(dynamic_property.value), + formatDescription(dynamic_property.description, dynamic_property.supports_expression_language) + }); } - for (const auto &processor : processorSet) { - const auto& filename = docsdir / processor.first; - std::ofstream outfile(filename); - - outfile << "## " << processor.first << "\n\n"; - outfile << "### Description\n\n"; - outfile << processor.second.description_ << '\n'; - - outfile << "\n### Properties\n\n"; - outfile << "In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. " - << "The table also indicates any default values, and whether a property supports the NiFi Expression Language.\n\n"; - - Table properties{{"Name", "Default Value", "Allowable Values", "Description"}}; - for (const auto &prop : processor.second.class_properties_) { - properties.addRow({ - formatName(prop.getName(), prop.getRequired()), - prop.getDefaultValue().to_string(), - formatAllowedValues(prop), - formatDescription(prop.getDescription(), prop.supportsExpressionLanguage())}); - } - outfile << properties.toString() << '\n'; - - if (!processor.second.dynamic_properties_.empty()) { - outfile << "### Dynamic Properties\n\n"; - Table dynamic_properties{{"Name", "Value", "Description"}}; - for (const auto& dynamic_property : processor.second.dynamic_properties_) { - dynamic_properties.addRow({ - formatName(dynamic_property.name, false), - std::string(dynamic_property.value), - formatDescription(dynamic_property.description, dynamic_property.supports_expression_language) - }); - } - outfile << dynamic_properties.toString() << '\n'; - } + docs << dynamic_properties.toString(); +} - outfile << "### Relationships\n\n"; - Table relationships{{"Name", "Description"}}; - for (const auto &rel : processor.second.class_relationships_) { - relationships.addRow({rel.getName(), formatDescription(rel.getDescription())}); - } - outfile << relationships.toString() << '\n'; - - if (!processor.second.output_attributes_.empty()) { - outfile << "### Output Attributes\n\n"; - Table output_attributes{{"Attribute", "Relationship", "Description"}}; - for (const auto& output_attribute : processor.second.output_attributes_) { - output_attributes.addRow({ - std::string(output_attribute.name), - formatListOfRelationships(output_attribute.relationships), - formatDescription(output_attribute.description)}); - } - outfile << output_attributes.toString() << '\n'; - } +void writeRelationships(std::ostream& docs, const minifi::ClassDescription& documentation) { + docs << "\n### Relationships\n\n"; + minifi::docs::Table relationships{{"Name", "Description"}}; + for (const auto &rel : documentation.class_relationships_) { + relationships.addRow({rel.getName(), formatDescription(rel.getDescription())}); } + docs << relationships.toString(); +} + +void writeOutputAttributes(std::ostream& docs, const minifi::ClassDescription& documentation) { + if (documentation.output_attributes_.empty()) { return; } + + docs << "\n### Output Attributes"; + minifi::docs::Table output_attributes{{"Attribute", "Relationship", "Description"}}; + for (const auto &output_attribute : documentation.output_attributes_) { + output_attributes.addRow({ + std::string(output_attribute.name), + formatListOfRelationships(output_attribute.relationships), + formatDescription(output_attribute.description)}); + } + docs << "\n\n" << output_attributes.toString(); +} - std::map<std::string, std::filesystem::path> fileList; - auto fileFind = [&fileList](const std::filesystem::path& base_path, const std::filesystem::path& file) -> bool { - if (file.string().find(".extra") == std::string::npos) { - auto file_name = file.string(); - ranges::actions::transform(file_name, [](auto ch) { return ::tolower(static_cast<unsigned char>(ch)); }); - fileList.emplace(file_name, base_path / file); +std::string extractClassName(const std::string& full_class_name) { + return minifi::utils::StringUtils::split(full_class_name, ".").back(); +} + +std::string lowercaseFirst(const std::pair<std::string, minifi::ClassDescription>& key_value) { + return minifi::utils::StringUtils::toLower(key_value.first); +}; + +} // namespace + +namespace org::apache::nifi::minifi::docs { + +void AgentDocs::generate(const std::filesystem::path& docs_dir) { + std::vector<std::pair<std::string, minifi::ClassDescription>> controller_services; + std::vector<std::pair<std::string, minifi::ClassDescription>> processors; + for (const auto &group : minifi::AgentBuild::getExtensions()) { + struct Components descriptions = build_description_.getClassDescriptions(group); + for (const auto &controller_service_description : descriptions.controller_services_) { + controller_services.emplace_back(extractClassName(controller_service_description.full_name_), controller_service_description); + } + for (const auto &processor_description : descriptions.processors_) { + processors.emplace_back(extractClassName(processor_description.full_name_), processor_description); } - return true; - }; - utils::file::list_dir(docsdir, fileFind, core::logging::LoggerFactory<AgentDocs>::getLogger()); - - genStream << "<!--\n" - "Licensed to the Apache Software Foundation (ASF) under one or more\n" - "contributor license agreements. See the NOTICE file distributed with\n" - "this work for additional information regarding copyright ownership.\n" - "The ASF licenses this file to You under the Apache License, Version 2.0\n" - "(the \"License\"); you may not use this file except in compliance with\n" - "the License. You may obtain a copy of the License at\n" - " http://www.apache.org/licenses/LICENSE-2.0\n"; - "Unless required by applicable law or agreed to in writing, software\n" - "distributed under the License is distributed on an \"AS IS\" BASIS,\n" - "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" - "See the License for the specific language governing permissions and\n" - "limitations under the License.\n" - "-->\n\n"; - genStream << "## Table of Contents\n\n"; - - for (const auto& file : fileList) { - genStream << "- [" << file.second.filename().string() << "](#" << file.first << ")\n"; } - genStream << "\n\n"; - - for (const auto& file : fileList) { - std::ifstream filestream(file.second); - genStream << filestream.rdbuf() << '\n'; - auto extra_path = file.second; - extra_path += ".extra"; - std::ifstream filestreamExtra(extra_path); - if (filestreamExtra.good()) { - genStream << filestreamExtra.rdbuf() << '\n'; - } + ranges::sort(controller_services, std::less(), lowercaseFirst); + ranges::sort(processors, std::less(), lowercaseFirst); + + std::ofstream controllers_md(docs_dir / "CONTROLLERS.md"); + writeHeader(controllers_md, controller_services); + for (const auto& [name, documentation] : controller_services) { + writeName(controllers_md, name); + writeDescription(controllers_md, documentation); + writeProperties(controllers_md, documentation); + } + + std::ofstream processors_md(docs_dir / "PROCESSORS.md"); + writeHeader(processors_md, processors); + for (const auto& [name, documentation] : processors) { + writeName(processors_md, name); + writeDescription(processors_md, documentation); + writeProperties(processors_md, documentation); + writeDynamicProperties(processors_md, documentation); + writeRelationships(processors_md, documentation); + writeOutputAttributes(processors_md, documentation); } } diff --git a/minifi_main/AgentDocs.h b/minifi_main/AgentDocs.h index e2017a87c..276aef763 100644 --- a/minifi_main/AgentDocs.h +++ b/minifi_main/AgentDocs.h @@ -14,8 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MAIN_AGENTDOCS_H_ -#define MAIN_AGENTDOCS_H_ +#pragma once #include <iostream> #include "agent/build_description.h" @@ -24,12 +23,10 @@ namespace org::apache::nifi::minifi::docs { class AgentDocs { public: - void generate(const std::filesystem::path &docsdir, std::ostream &genStream); + void generate(const std::filesystem::path& docs_dir); + private: - [[nodiscard]] static std::string extractClassName(const std::string &processor); BuildDescription build_description_; }; } // namespace org::apache::nifi::minifi::docs - -#endif // MAIN_AGENTDOCS_H_ diff --git a/minifi_main/MiNiFiMain.cpp b/minifi_main/MiNiFiMain.cpp index 6e34b9360..9d1d9b02f 100644 --- a/minifi_main/MiNiFiMain.cpp +++ b/minifi_main/MiNiFiMain.cpp @@ -116,7 +116,7 @@ void sigHandler(int signal) { } } -void dumpDocs(const std::shared_ptr<minifi::Configure> &configuration, const std::string &dir, std::ostream &out) { +void dumpDocs(const std::shared_ptr<minifi::Configure> &configuration, const std::string &dir) { auto pythoncreator = core::ClassLoader::getDefaultClassLoader().instantiate("PythonCreator", "PythonCreator"); if (nullptr != pythoncreator) { pythoncreator->configure(configuration); @@ -124,7 +124,7 @@ void dumpDocs(const std::shared_ptr<minifi::Configure> &configuration, const std minifi::docs::AgentDocs docsCreator; - docsCreator.generate(dir, out); + docsCreator.generate(dir); } void writeJsonSchema(const std::shared_ptr<minifi::Configure> &configuration, std::ostream& out) { @@ -160,22 +160,7 @@ void dumpDocsIfRequested(const argparse::ArgumentParser& parser, const std::shar } std::cout << "Dumping docs to " << docs_params[0] << std::endl; - if (docs_params.size() > 1) { - auto path = std::filesystem::path(docs_params[1]); - if (std::filesystem::exists(path) && !std::filesystem::is_regular_file(path)) { - std::cerr << "PROCESSORS.md target path exists, but it is not a regular file: " << path << std::endl; - std::exit(1); - } - auto dir = path.parent_path(); - if (dir == docs_params[0]) { - std::cerr << "Target file should be out of the working directory: " << dir << std::endl; - std::exit(1); - } - std::ofstream outref(docs_params[1]); - dumpDocs(configure, docs_params[0], outref); - } else { - dumpDocs(configure, docs_params[0], std::cout); - } + dumpDocs(configure, docs_params[0]); std::exit(0); } @@ -209,10 +194,9 @@ int main(int argc, char **argv) { .metavar("KEY=VALUE") .help("Override a property read from minifi.properties file in key=value format"); argument_parser.add_argument("-d", "--docs") - .nargs(1, 2) - .metavar("PATH") - .help("Generate documentation in the directory specified in the first parameter. File path can be specified for the PROCESSORS.md file in the second parameter. " - "If no separate path is given for the PROCESSORS.md file, it will be printed to stdout."); + .nargs(1) + .metavar("DIRECTORY") + .help("Generate documentation in the specified directory"); argument_parser.add_argument("-s", "--schema") .metavar("PATH") .help("Generate JSON schema to the specified path");
