This is an automated email from the ASF dual-hosted git repository. lordgamez pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 38aec9608c0dec3d607206d7fde06736a9a4773a Author: Ferenc Gerlits <[email protected]> AuthorDate: Tue Jan 3 19:10:03 2023 +0100 MINIFICPP-2021 Auto-generate the remaining parts of PROCESSORS.md Signed-off-by: Gabor Gyimesi <[email protected]> This closes #1492 --- PROCESSORS.md | 253 +++++++++++---------- extensions/gcp/processors/DeleteGCSObject.h | 5 + extensions/gcp/processors/FetchGCSObject.h | 5 + .../processors/GCSProcessorStaticDefinitions.cpp | 57 +++++ extensions/gcp/processors/ListGCSBucket.h | 49 ++++ extensions/gcp/processors/PutGCSObject.h | 53 +++++ extensions/http-curl/processors/InvokeHTTP.cpp | 7 + extensions/http-curl/processors/InvokeHTTP.h | 13 ++ extensions/jni/jvm/NarClassLoader.h | 4 +- extensions/python/PythonCreator.h | 2 +- extensions/sql/processors/QueryDatabaseTable.h | 3 + .../processors/SQLProcessorStaticDefinitions.cpp | 4 + .../standard-processors/processors/ListFile.cpp | 16 ++ .../standard-processors/processors/ListFile.h | 21 ++ .../processors/ListenSyslog.cpp | 15 ++ .../standard-processors/processors/ListenSyslog.h | 35 +++ .../standard-processors/processors/ListenTCP.cpp | 3 + .../standard-processors/processors/ListenTCP.h | 4 + .../standard-processors/processors/ListenUDP.cpp | 3 + .../standard-processors/processors/ListenUDP.h | 4 + .../processors/RetryFlowFile.cpp | 20 +- .../standard-processors/processors/RetryFlowFile.h | 12 + .../standard-processors/processors/RouteText.cpp | 9 +- .../standard-processors/processors/RouteText.h | 6 + libminifi/include/agent/agent_docs.h | 23 +- libminifi/include/core/DynamicProperty.h | 58 +++++ .../include/core/OutputAttribute.h | 44 ++-- libminifi/include/core/Processor.h | 6 + .../include/core/state/nodes/AgentInformation.h | 4 +- libminifi/include/utils/StringUtils.h | 96 +++----- libminifi/src/agent/JsonSchema.cpp | 8 +- libminifi/src/agent/agent_docs.cpp | 24 +- libminifi/test/unit/StringUtilsTests.cpp | 36 ++- minifi_main/AgentDocs.cpp | 91 ++++++-- minifi_main/TableFormatter.cpp | 62 ++--- minifi_main/TableFormatter.h | 8 - 36 files changed, 733 insertions(+), 330 deletions(-) diff --git a/PROCESSORS.md b/PROCESSORS.md index 2910c6cd9..70de1ceed 100644 --- a/PROCESSORS.md +++ b/PROCESSORS.md @@ -593,11 +593,12 @@ In the list below, the names of required properties appear in bold. Any other pr ### Output Attributes -| Attribute | Relationship | Description | -|----------------------|--------------|---------------------------------------------------------| -| _gcs.status.message_ | failure | The status message received from google cloud. | -| _gcs.error.reason_ | failure | The description of the error occurred during operation. | -| _gcs.error.domain_ | failure | The domain of the error occurred during operation. | +| Attribute | Relationship | Description | +|--------------------|--------------|---------------------------------------------------------| +| gcs.status.message | failure | The status message received from google cloud. | +| gcs.error.reason | failure | The description of the error occurred during operation. | +| gcs.error.domain | failure | The domain of the error occurred during operation. | + ## DeleteS3Object @@ -903,11 +904,11 @@ In the list below, the names of required properties appear in bold. Any other pr ### Output Attributes -| Attribute | Relationship | Description | -|----------------------|--------------|---------------------------------------------------------| -| _gcs.status.message_ | failure | The status message received from google cloud. | -| _gcs.error.reason_ | failure | The description of the error occurred during operation. | -| _gcs.error.domain_ | failure | The domain of the error occurred during operation. | +| Attribute | Relationship | Description | +|--------------------|--------------|---------------------------------------------------------| +| gcs.status.message | failure | The status message received from google cloud. | +| gcs.error.reason | failure | The description of the error occurred during operation. | +| gcs.error.domain | failure | The domain of the error occurred during operation. | ## FetchOPCProcessor @@ -1252,12 +1253,12 @@ In the list below, the names of required properties appear in bold. Any other pr ### Output Attributes -| Attribute | Description | -|-----------------------------|----------------------------------------------------------------| -| _invokehttp.status.code_ | The status code that is returned | -| _invokehttp.status.message_ | The status message that is returned | -| _invokehttp.request.url_ | The original request URL | -| _invokehttp.tx.id_ | The transaction ID that is returned after reading the response | +| Attribute | Relationship | Description | +|---------------------------|------------------------------------|----------------------------------------------------------------| +| invokehttp.status.code | success, response, retry, no retry | The status code that is returned | +| invokehttp.status.message | success, response, retry, no retry | The status message that is returned | +| invokehttp.request.url | success, response, retry, no retry | The original request URL | +| invokehttp.tx.id | success, response, retry, no retry | The transaction ID that is returned after reading the response | ## ListAzureBlobStorage @@ -1376,23 +1377,23 @@ In the list below, the names of required properties appear in bold. Any other pr ### Output Attributes -| Attribute | Description | Requirements | -|--------------------------|--------------------------------------------------------------------|------------------------| -| _syslog.protocol_ | The protocol over which the Syslog message was received. | - | -| _syslog.port_ | The port over which the Syslog message was received. | - | -| _syslog.sender_ | The hostname of the Syslog server that sent the message. | - | -| _syslog.valid_ | An indicator of whether this message matched the expected formats. | Parsing enabled | -| _syslog.priority_ | The priority of the Syslog message. | Parsed RFC5424/RFC3164 | -| _syslog.severity_ | The severity of the Syslog message. | Parsed RFC5424/RFC3164 | -| _syslog.facility_ | The facility of the Syslog message. | Parsed RFC5424/RFC3164 | -| _syslog.timestamp_ | The timestamp of the Syslog message. | Parsed RFC5424/RFC3164 | -| _syslog.hostname_ | The hostname of the Syslog message. | Parsed RFC5424/RFC3164 | -| _syslog.msg_ | The free-form message of the Syslog message. | Parsed RFC5424/RFC3164 | -| _syslog.version_ | The version of the Syslog message. | Parsed RFC5424 | -| _syslog.app_name_ | The app name of the Syslog message. | Parsed RFC5424 | -| _syslog.proc_id_ | The proc id of the Syslog message. | Parsed RFC5424 | -| _syslog.msg_id_ | The message id of the Syslog message. | Parsed RFC5424 | -| _syslog.structured_data_ | The structured data of the Syslog message. | Parsed RFC5424 | +| Attribute | Relationship | Description | +|------------------------|--------------|---------------------------------------------------------------------------------------------------| +| syslog.protocol | | The protocol over which the Syslog message was received. | +| syslog.port | | The port over which the Syslog message was received. | +| syslog.sender | | The hostname of the Syslog server that sent the message. | +| syslog.valid | | An indicator of whether this message matched the expected formats. (requirement: parsing enabled) | +| syslog.priority | | The priority of the Syslog message. (requirement: parsed RFC5424/RFC3164) | +| syslog.severity | | The severity of the Syslog message. (requirement: parsed RFC5424/RFC3164) | +| syslog.facility | | The facility of the Syslog message. (requirement: parsed RFC5424/RFC3164) | +| syslog.timestamp | | The timestamp of the Syslog message. (requirement: parsed RFC5424/RFC3164) | +| syslog.hostname | | The hostname of the Syslog message. (requirement: parsed RFC5424/RFC3164) | +| syslog.msg | | The free-form message of the Syslog message. (requirement: parsed RFC5424/RFC3164) | +| syslog.version | | The version of the Syslog message. (requirement: parsed RFC5424) | +| syslog.app_name | | The app name of the Syslog message. (requirement: parsed RFC5424) | +| syslog.proc_id | | The proc id of the Syslog message. (requirement: parsed RFC5424) | +| syslog.msg_id | | The message id of the Syslog message. (requirement: parsed RFC5424) | +| syslog.structured_data | | The structured data of the Syslog message. (requirement: parsed RFC5424) | ## ListenTCP @@ -1421,12 +1422,10 @@ In the list below, the names of required properties appear in bold. Any other pr ### Output Attributes -| Attribute | Description | Requirements | -|--------------|----------------------------------------------|--------------| -| _tcp.port_ | The sending port the messages were received. | - | -| _tcp.sender_ | The sending host of the messages. | - | - - +| Attribute | Relationship | Description | +|------------|--------------|----------------------------------------------| +| tcp.port | | The sending port the messages were received. | +| tcp.sender | | The sending host of the messages. | ## ListenUDP @@ -1453,10 +1452,10 @@ In the list below, the names of required properties appear in bold. Any other pr ### Output Attributes -| Attribute | Description | Requirements | -|--------------|-----------------------------------------------|--------------| -| _udp.port_ | The sending port the messages were received. | - | -| _udp.sender_ | The sending host of the messages. | - | +| Attribute | Relationship | Description | +|------------|--------------|----------------------------------------------| +| udp.port | | The sending port the messages were received. | +| udp.sender | | The sending host of the messages. | ## ListFile @@ -1489,16 +1488,16 @@ In the list below, the names of required properties appear in bold. Any other pr ### Output Attributes -| Attribute | Relationship | Description | -|-------------------------|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| _filename_ | success | The name of the file that was read from filesystem. | -| _path_ | success | The path is set to the relative path of the file's directory on filesystem compared to the Input Directory property. For example, if Input Directory is set to /tmp, then files picked up from /tmp will have the path attribute set to "./". If the Recurse Subdirectories property is set to true and a file is picked up from /tmp/abc/1/2/3, then the path attribute will be set to "abc/1/2/3/". | -| _absolute.path_ | success | The absolute.path is set to the absolute path of the file's directory on filesystem. For example, if the Input Directory property is set to /tmp, then files picked up from /tmp will have the path attribute set to "/tmp/". If the Recurse Subdirectories property is set to true and a file is picked up from /tmp/abc/1/2/3, then the path attribute will be set to "/tmp/abc/1/2/3/". | -| _file.owner_ | success | The user that owns the file in filesystem | -| _file.group_ | success | The group that owns the file in filesystem | -| _file.size_ | success | The number of bytes in the file in filesystem | -| _file.permissions_ | success | The permissions for the file in filesystem. This is formatted as 3 characters for the owner, 3 for the group, and 3 for other users. For example rw-rw-r-- | -| _file.lastModifiedTime_ | success | The timestamp of when the file in filesystem was last modified as 'yyyy-MM-dd'T'HH:mm:ssZ' | +| Attribute | Relationship | Description | +|-----------------------|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| filename | success | The name of the file that was read from filesystem. | +| path | success | The path is set to the relative path of the file's directory on filesystem compared to the Input Directory property. For example, if Input Directory is set to /tmp, then files picked up from /tmp will have the path attribute set to "./". If the Recurse Subdirectories property is set to true and a file is picked up from /tmp/abc/1/2/3, then the path attribute will be set to "abc/1/2/3/". | +| absolute.path | success | The absolute.path is set to the absolute path of the file's directory on filesystem. For example, if the Input Directory property is set to /tmp, then files picked up from /tmp will have the path attribute set to "/tmp/". If the Recurse Subdirectories property is set to true and a file is picked up from /tmp/abc/1/2/3, then the path attribute will be set to "/tmp/abc/1/2/3/". | +| file.owner | success | The user that owns the file in filesystem | +| file.group | success | The group that owns the file in filesystem | +| file.size | success | The number of bytes in the file in filesystem | +| file.permissions | success | The permissions for the file in filesystem. This is formatted as 3 characters for the owner, 3 for the group, and 3 for other users. For example rw-rw-r-- | +| file.lastModifiedTime | success | The timestamp of when the file in filesystem was last modified as 'yyyy-MM-dd'T'HH:mm:ssZ' | ## ListGCSBucket @@ -1527,29 +1526,30 @@ In the list below, the names of required properties appear in bold. Any other pr ### Output Attributes -| Attribute | Relationship | Description | -|----------------------------|--------------|--------------------------------------------------------------------| -| _gcs.bucket_ | success | Bucket of the object. | -| _gcs.key, filename_ | success | Name of the object. | -| _gcs.size_ | success | Size of the object. | -| _gcs.crc32c_ | success | The CRC32C checksum of object's data, encoded in base64 | -| _gcs.md5_ | success | The MD5 hash of the object's data encoded in base64. | -| _gcs.owner.entity_ | success | The owner entity, in the form "user-emailAddress". | -| _gcs.owner.entity.id_ | success | The ID for the entity. | -| _gcs.content.encoding_ | success | The content encoding of the object. | -| _gcs.content.language_ | success | The content language of the object. | -| _gcs.content.disposition_ | success | The data content disposition of the object. | -| _gcs.media.link_ | success | The media download link to the object. | -| _gcs.self.link_ | success | The link to this object. | -| _gcs.etag_ | success | The HTTP 1.1 Entity tag for the object. | -| _gcs.generated.id_ | success | The service-generated ID for the object | -| _gcs.generation_ | success | The content generation of this object. Used for object versioning. | -| _gcs.metageneration_ | success | The metageneration of the object. | -| _gcs.create.time_ | success | Unix timestamp of the object's creation in milliseconds | -| _gcs.update.time_ | success | Unix timestamp of the object's last modification in milliseconds | -| _gcs.delete.time_ | success | Unix timestamp of the object's deletion in milliseconds | -| _gcs.encryption.algorithm_ | success | The algorithm used to encrypt the object. | -| _gcs.encryption.sha256_ | success | The SHA256 hash of the key used to encrypt the object | +| Attribute | Relationship | Description | +|--------------------------|--------------|--------------------------------------------------------------------| +| gcs.bucket | success | Bucket of the object. | +| gcs.key | success | Name of the object. | +| filename | success | Same as gcs.key | +| gcs.size | success | Size of the object. | +| gcs.crc32c | success | The CRC32C checksum of object's data, encoded in base64. | +| gcs.md5 | success | The MD5 hash of the object's data, encoded in base64. | +| gcs.owner.entity | success | The owner entity, in the form "user-emailAddress". | +| gcs.owner.entity.id | success | The ID for the entity. | +| gcs.content.encoding | success | The content encoding of the object. | +| gcs.content.language | success | The content language of the object. | +| gcs.content.disposition | success | The data content disposition of the object. | +| gcs.media.link | success | The media download link to the object. | +| gcs.self.link | success | The link to this object. | +| gcs.etag | success | The HTTP 1.1 Entity tag for the object. | +| gcs.generated.id | success | The service-generated ID for the object. | +| gcs.generation | success | The content generation of this object. Used for object versioning. | +| gcs.metageneration | success | The metageneration of the object. | +| gcs.create.time | success | Unix timestamp of the object's creation in milliseconds. | +| gcs.update.time | success | Unix timestamp of the object's last modification in milliseconds. | +| gcs.delete.time | success | Unix timestamp of the object's deletion in milliseconds. | +| gcs.encryption.algorithm | success | The algorithm used to encrypt the object. | +| gcs.encryption.sha256 | success | The SHA256 hash of the key used to encrypt the object. | ## ListS3 @@ -2150,32 +2150,32 @@ In the list below, the names of required properties appear in bold. Any other pr ### Output Attributes -| Attribute | Relationship | Description | -|----------------------------|--------------|--------------------------------------------------------------------| -| _gcs.status.message_ | failure | The status message received from google cloud. | -| _gcs.error.reason_ | failure | The description of the error occurred during upload. | -| _gcs.error.domain_ | failure | The domain of the error occurred during upload. | -| _gcs.bucket_ | success | Bucket of the object. | -| _gcs.key_ | success | Name of the object. | -| _gcs.size_ | success | Size of the object. | -| _gcs.crc32c_ | success | The CRC32C checksum of object's data, encoded in base64 | -| _gcs.md5_ | success | The MD5 hash of the object's data encoded in base64. | -| _gcs.owner.entity_ | success | The owner entity, in the form "user-emailAddress". | -| _gcs.owner.entity.id_ | success | The ID for the entity. | -| _gcs.content.encoding_ | success | The content encoding of the object. | -| _gcs.content.language_ | success | The content language of the object. | -| _gcs.content.disposition_ | success | The data content disposition of the object. | -| _gcs.media.link_ | success | The media download link to the object. | -| _gcs.self.link_ | success | The link to this object. | -| _gcs.etag_ | success | The HTTP 1.1 Entity tag for the object. | -| _gcs.generated.id_ | success | The service-generated ID for the object | -| _gcs.generation_ | success | The content generation of this object. Used for object versioning. | -| _gcs.metageneration_ | success | The metageneration of the object. | -| _gcs.create.time_ | success | Unix timestamp of the object's creation in milliseconds | -| _gcs.update.time_ | success | Unix timestamp of the object's last modification in milliseconds | -| _gcs.delete.time_ | success | Unix timestamp of the object's deletion in milliseconds | -| _gcs.encryption.algorithm_ | success | The algorithm used to encrypt the object. | -| _gcs.encryption.sha256_ | success | The SHA256 hash of the key used to encrypt the object | +| Attribute | Relationship | Description | +|--------------------------|--------------|--------------------------------------------------------------------| +| gcs.status.message | failure | The status message received from google cloud. | +| gcs.error.reason | failure | The description of the error occurred during upload. | +| gcs.error.domain | failure | The domain of the error occurred during upload. | +| gcs.bucket | success | Bucket of the object. | +| gcs.key | success | Name of the object. | +| gcs.size | success | Size of the object. | +| gcs.crc32c | success | The CRC32C checksum of object's data, encoded in base64. | +| gcs.md5 | success | The MD5 hash of the object's data, encoded in base64. | +| gcs.owner.entity | success | The owner entity, in the form "user-emailAddress". | +| gcs.owner.entity.id | success | The ID for the entity. | +| gcs.content.encoding | success | The content encoding of the object. | +| gcs.content.language | success | The content language of the object. | +| gcs.content.disposition | success | The data content disposition of the object. | +| gcs.media.link | success | The media download link to the object. | +| gcs.self.link | success | The link to this object. | +| gcs.etag | success | The HTTP 1.1 Entity tag for the object. | +| gcs.generated.id | success | The service-generated ID for the object. | +| gcs.generation | success | The content generation of this object. Used for object versioning. | +| gcs.metageneration | success | The metageneration of the object. | +| gcs.create.time | success | Unix timestamp of the object's creation in milliseconds. | +| gcs.update.time | success | Unix timestamp of the object's last modification in milliseconds. | +| gcs.delete.time | success | Unix timestamp of the object's deletion in milliseconds. | +| gcs.encryption.algorithm | success | The algorithm used to encrypt the object. | +| gcs.encryption.sha256 | success | The SHA256 hash of the key used to encrypt the object. | ## PutOPCProcessor @@ -2434,6 +2434,7 @@ In the list below, the names of required properties appear in bold. Any other pr ### Description Fetches all rows of a table, whose values in the specified Maximum-value Columns are larger than the previously-seen maxima. If that property is not provided, all rows are returned. The rows are grouped according to the value of Max Rows Per Flow File property and formatted as JSON. + ### Properties In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. @@ -2448,18 +2449,18 @@ In the list below, the names of required properties appear in bold. Any other pr | Maximum-value Columns | | | A comma-separated list of column names. The processor will keep track of the maximum value for each column that has been returned since the processor started running. Using multiple columns implies an order to the column list, and each column's values are expected to increase more slowly than the previous columns' values. Thus, using multiple columns implies a hierarchical structure of columns, which is usually used fo [...] | Where Clause | | | A custom clause to be added in the WHERE condition when building SQL queries.<br/>**Supports Expression Language: true** [...] +### Dynamic Properties + +| Name | Value | Description | +|-------------------------------------|------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| initial.maxvalue.<max_value_column> | Initial maximum value for the specified column | Specifies an initial max value for max value column(s). Properties should be added in the format `initial.maxvalue.<max_value_column>`. This value is only used the first time the table is accessed (when a Maximum Value Column is specified).<br/>**Supports Expression Language: true** | + ### Relationships | Name | Description | |---------|----------------------------------------------------------| | success | Successfully created FlowFile from SQL query result set. | -### Dynamic Properties: - -| Name | Value | Description | -|-------------------------------------|------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| initial.maxvalue.<max_value_column> | Initial maximum value for the specified column | Specifies an initial max value for max value column(s). Properties should be added in the format `initial.maxvalue.<max_value_column>`. This value is only used the first time the table is accessed (when a Maximum Value Column is specified).<br/>**Supports Expression Language: true** | - ## QuerySplunkIndexingStatus @@ -2558,6 +2559,12 @@ In the list below, the names of required properties appear in bold. Any other pr | **Fail on Non-numerical Overwrite** | false | | If the FlowFile already has the attribute defined in 'Retry Attribute' that is *not* a number, fail the FlowFile instead of resetting that value to '1' | | **Reuse Mode** | Fail on Reuse | Fail on Reuse<br/>Reset Reuse<br/>Warn on Reuse | Defines how the Processor behaves if the retry FlowFile has a different retry UUID than the instance that received the FlowFile. This generally means that the attribute was not reset after being successfully retried by a previous instance of this processor. | +### Dynamic Properties + +| Name | Value | Description | +|---------------------------------|--------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Exceeded FlowFile Attribute Key | The value of the attribute added to the FlowFile | One or more dynamic properties can be used to add attributes to FlowFiles passed to the 'retries_exceeded' relationship.<br/>**Supports Expression Language: true** | + ### Relationships | Name | Description | @@ -2566,18 +2573,12 @@ In the list below, the names of required properties appear in bold. Any other pr | retries_exceeded | Input FlowFile has exceeded the configured maximum retry count, do not pass this relationship back to the input Processor to terminate the limited feedback loop. | | failure | The processor is configured such that a non-numerical value on 'Retry Attribute' results in a failure instead of resetting that value to '1'. This will immediately terminate the limited feedback loop. Might also include when 'Maximum Retries' contains attribute expression language that does not resolve to an Integer. | -### Dynamic Properties: - -| Name | Value | Description | -|---------------------------------|--------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Exceeded FlowFile Attribute Key | The value of the attribute added to the FlowFile | One or more dynamic properties can be used to add attributes to FlowFiles passed to the 'retries_exceeded' relationship.<br/>**Supports Expression Language: true** | - -### Writes Attributes: +### Output Attributes -| Name | Description | -|-----------------------|--------------------------------------------------------------------------------------------------| -| Retry Attribute | User defined retry attribute is updated with the current retry count | -| Retry Attribute .uuid | User defined retry attribute with .uuid that determines what processor retried the FlowFile last | +| Attribute | Relationship | Description | +|-----------------------|--------------|-------------------------------------------------------------------------------------------------------------------------| +| Retry Attribute | | User defined retry attribute is updated with the current retry count | +| Retry Attribute .uuid | | User defined retry attribute with .uuid suffix is updated with the UUID of the processor that retried the FlowFile last | ## RouteOnAttribute @@ -2626,9 +2627,9 @@ In the list below, the names of required properties appear in bold. Any other pr ### Dynamic Properties -| Name | Value | Description | -|-------------------|------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Relationship Name | value to match against | Routes data that matches the value specified in the Dynamic Property Value to the Relationship specified in the Dynamic Property Key.<br>**Supports Expression Language: true** | +| Name | Value | Description | +|-------------------|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Relationship Name | value to match against | Routes data that matches the value specified in the Dynamic Property Value to the Relationship specified in the Dynamic Property Key.<br/>**Supports Expression Language: true** | ### Relationships @@ -2638,11 +2639,11 @@ In the list below, the names of required properties appear in bold. Any other pr | unmatched | Segments that do not satisfy the required user-defined rules will be routed to this Relationship | | matched | Segments that satisfy the required user-defined rules will be routed to this Relationship | -### Writes Attributes +### Output Attributes -| Name | Description | -|-----------------|----------------------------------------------------------------------------------------------------------------------------------------------------------| -| RouteText.Group | The value captured by all capturing groups in the 'Grouping Regular Expression' property. If this property is not set, this attribute will not be added. | +| Attribute | Relationship | Description | +|-----------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------| +| RouteText.Group | | The value captured by all capturing groups in the 'Grouping Regular Expression' property. If this property is not set, this attribute will not be added. | ## SourceInitiatedSubscriptionListener diff --git a/extensions/gcp/processors/DeleteGCSObject.h b/extensions/gcp/processors/DeleteGCSObject.h index 024d17c10..66238b449 100644 --- a/extensions/gcp/processors/DeleteGCSObject.h +++ b/extensions/gcp/processors/DeleteGCSObject.h @@ -53,6 +53,11 @@ class DeleteGCSObject : public GCSProcessor { EXTENSIONAPI static const core::Relationship Failure; static auto relationships() { return std::array{Success, Failure}; } + EXTENSIONAPI static const core::OutputAttribute Message; + EXTENSIONAPI static const core::OutputAttribute Reason; + EXTENSIONAPI static const core::OutputAttribute Domain; + static auto outputAttributes() { return std::array{Message, Reason, Domain}; } + EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false; EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false; EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_REQUIRED; diff --git a/extensions/gcp/processors/FetchGCSObject.h b/extensions/gcp/processors/FetchGCSObject.h index 13845012c..6616a667d 100644 --- a/extensions/gcp/processors/FetchGCSObject.h +++ b/extensions/gcp/processors/FetchGCSObject.h @@ -54,6 +54,11 @@ class FetchGCSObject : public GCSProcessor { EXTENSIONAPI static const core::Relationship Failure; static auto relationships() { return std::array{Success, Failure}; } + EXTENSIONAPI static const core::OutputAttribute Message; + EXTENSIONAPI static const core::OutputAttribute Reason; + EXTENSIONAPI static const core::OutputAttribute Domain; + static auto outputAttributes() { return std::array{Message, Reason, Domain}; } + EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false; EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false; EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_REQUIRED; diff --git a/extensions/gcp/processors/GCSProcessorStaticDefinitions.cpp b/extensions/gcp/processors/GCSProcessorStaticDefinitions.cpp index 87d58e3e0..60240fd5b 100644 --- a/extensions/gcp/processors/GCSProcessorStaticDefinitions.cpp +++ b/extensions/gcp/processors/GCSProcessorStaticDefinitions.cpp @@ -17,6 +17,7 @@ #include "DeleteGCSObject.h" #include "FetchGCSObject.h" +#include "../GCPAttributes.h" #include "GCSProcessor.h" #include "ListGCSBucket.h" #include "PutGCSObject.h" @@ -86,6 +87,10 @@ const core::Property DeleteGCSObject::EncryptionKey( const core::Relationship DeleteGCSObject::Success("success", "FlowFiles are routed to this relationship after a successful Google Cloud Storage operation."); const core::Relationship DeleteGCSObject::Failure("failure", "FlowFiles are routed to this relationship if the Google Cloud Storage operation fails."); +const core::OutputAttribute DeleteGCSObject::Message{GCS_STATUS_MESSAGE, { Failure }, "The status message received from google cloud."}; +const core::OutputAttribute DeleteGCSObject::Reason{GCS_ERROR_REASON, { Failure }, "The description of the error occurred during operation."}; +const core::OutputAttribute DeleteGCSObject::Domain{GCS_ERROR_DOMAIN, { Failure }, "The domain of the error occurred during operation."}; + REGISTER_RESOURCE(DeleteGCSObject, Processor); @@ -121,6 +126,10 @@ const core::Property FetchGCSObject::EncryptionKey( const core::Relationship FetchGCSObject::Success("success", "FlowFiles are routed to this relationship after a successful Google Cloud Storage operation."); const core::Relationship FetchGCSObject::Failure("failure", "FlowFiles are routed to this relationship if the Google Cloud Storage operation fails."); +const core::OutputAttribute FetchGCSObject::Message{GCS_STATUS_MESSAGE, { Failure }, "The status message received from google cloud."}; +const core::OutputAttribute FetchGCSObject::Reason{GCS_ERROR_REASON, { Failure }, "The description of the error occurred during operation."}; +const core::OutputAttribute FetchGCSObject::Domain{GCS_ERROR_DOMAIN, { Failure }, "The domain of the error occurred during operation."}; + REGISTER_RESOURCE(FetchGCSObject, Processor); @@ -141,6 +150,29 @@ const core::Property ListGCSBucket::ListAllVersions( const core::Relationship ListGCSBucket::Success("success", "FlowFiles are routed to this relationship after a successful Google Cloud Storage operation."); +const core::OutputAttribute ListGCSBucket::BucketOutputAttribute{GCS_BUCKET_ATTR, { Success }, "Bucket of the object."}; +const core::OutputAttribute ListGCSBucket::Key{GCS_OBJECT_NAME_ATTR, { Success }, "Name of the object."}; +const core::OutputAttribute ListGCSBucket::Filename{"filename", { Success }, std::string{"Same as "} + GCS_OBJECT_NAME_ATTR}; // NOLINT +const core::OutputAttribute ListGCSBucket::Size{GCS_SIZE_ATTR, { Success }, "Size of the object."}; +const core::OutputAttribute ListGCSBucket::Crc32c{GCS_CRC32C_ATTR, { Success }, "The CRC32C checksum of object's data, encoded in base64."}; +const core::OutputAttribute ListGCSBucket::Md5{GCS_MD5_ATTR, { Success }, "The MD5 hash of the object's data, encoded in base64."}; +const core::OutputAttribute ListGCSBucket::OwnerEntity{GCS_OWNER_ENTITY_ATTR, { Success }, "The owner entity, in the form \"user-emailAddress\"."}; +const core::OutputAttribute ListGCSBucket::OwnerEntityId{GCS_OWNER_ENTITY_ID_ATTR, { Success }, "The ID for the entity."}; +const core::OutputAttribute ListGCSBucket::ContentEncoding{GCS_CONTENT_ENCODING_ATTR, { Success }, "The content encoding of the object."}; +const core::OutputAttribute ListGCSBucket::ContentLanguage{GCS_CONTENT_LANGUAGE_ATTR, { Success }, "The content language of the object."}; +const core::OutputAttribute ListGCSBucket::ContentDisposition{GCS_CONTENT_DISPOSITION_ATTR, { Success }, "The data content disposition of the object."}; +const core::OutputAttribute ListGCSBucket::MediaLink{GCS_MEDIA_LINK_ATTR, { Success }, "The media download link to the object."}; +const core::OutputAttribute ListGCSBucket::SelfLink{GCS_SELF_LINK_ATTR, { Success }, "The link to this object."}; +const core::OutputAttribute ListGCSBucket::Etag{GCS_ETAG_ATTR, { Success }, "The HTTP 1.1 Entity tag for the object."}; +const core::OutputAttribute ListGCSBucket::GeneratedId{GCS_GENERATED_ID, { Success }, "The service-generated ID for the object."}; +const core::OutputAttribute ListGCSBucket::Generation{GCS_GENERATION, { Success }, "The content generation of this object. Used for object versioning."}; +const core::OutputAttribute ListGCSBucket::Metageneration{GCS_META_GENERATION, { Success }, "The metageneration of the object."}; +const core::OutputAttribute ListGCSBucket::CreateTime{GCS_CREATE_TIME_ATTR, { Success }, "Unix timestamp of the object's creation in milliseconds."}; +const core::OutputAttribute ListGCSBucket::UpdateTime{GCS_UPDATE_TIME_ATTR, { Success }, "Unix timestamp of the object's last modification in milliseconds."}; +const core::OutputAttribute ListGCSBucket::DeleteTime{GCS_DELETE_TIME_ATTR, { Success }, "Unix timestamp of the object's deletion in milliseconds."}; +const core::OutputAttribute ListGCSBucket::EncryptionAlgorithm{GCS_ENCRYPTION_ALGORITHM_ATTR, { Success }, "The algorithm used to encrypt the object."}; +const core::OutputAttribute ListGCSBucket::EncryptionSha256{GCS_ENCRYPTION_SHA256_ATTR, { Success }, "The SHA256 hash of the key used to encrypt the object."}; + REGISTER_RESOURCE(ListGCSBucket, Processor); @@ -205,6 +237,31 @@ const core::Property PutGCSObject::OverwriteObject( const core::Relationship PutGCSObject::Success("success", "Files that have been successfully written to Google Cloud Storage are transferred to this relationship"); const core::Relationship PutGCSObject::Failure("failure", "Files that could not be written to Google Cloud Storage for some reason are transferred to this relationship"); +const core::OutputAttribute PutGCSObject::Message{GCS_STATUS_MESSAGE, { Failure }, "The status message received from google cloud."}; +const core::OutputAttribute PutGCSObject::Reason{GCS_ERROR_REASON, { Failure }, "The description of the error occurred during upload."}; +const core::OutputAttribute PutGCSObject::Domain{GCS_ERROR_DOMAIN, { Failure }, "The domain of the error occurred during upload."}; +const core::OutputAttribute PutGCSObject::BucketOutputAttribute{GCS_BUCKET_ATTR, { Success }, "Bucket of the object."}; +const core::OutputAttribute PutGCSObject::KeyOutputAttribute{GCS_OBJECT_NAME_ATTR, { Success }, "Name of the object."}; +const core::OutputAttribute PutGCSObject::Size{GCS_SIZE_ATTR, { Success }, "Size of the object."}; +const core::OutputAttribute PutGCSObject::Crc32c{GCS_CRC32C_ATTR, { Success }, "The CRC32C checksum of object's data, encoded in base64."}; +const core::OutputAttribute PutGCSObject::Md5{GCS_MD5_ATTR, { Success }, "The MD5 hash of the object's data, encoded in base64."}; +const core::OutputAttribute PutGCSObject::OwnerEntity{GCS_OWNER_ENTITY_ATTR, { Success }, "The owner entity, in the form \"user-emailAddress\"."}; +const core::OutputAttribute PutGCSObject::OwnerEntityId{GCS_OWNER_ENTITY_ID_ATTR, { Success }, "The ID for the entity."}; +const core::OutputAttribute PutGCSObject::ContentEncoding{GCS_CONTENT_ENCODING_ATTR, { Success }, "The content encoding of the object."}; +const core::OutputAttribute PutGCSObject::ContentLanguage{GCS_CONTENT_LANGUAGE_ATTR, { Success }, "The content language of the object."}; +const core::OutputAttribute PutGCSObject::ContentDisposition{GCS_CONTENT_DISPOSITION_ATTR, { Success }, "The data content disposition of the object."}; +const core::OutputAttribute PutGCSObject::MediaLink{GCS_MEDIA_LINK_ATTR, { Success }, "The media download link to the object."}; +const core::OutputAttribute PutGCSObject::SelfLink{GCS_SELF_LINK_ATTR, { Success }, "The link to this object."}; +const core::OutputAttribute PutGCSObject::Etag{GCS_ETAG_ATTR, { Success }, "The HTTP 1.1 Entity tag for the object."}; +const core::OutputAttribute PutGCSObject::GeneratedId{GCS_GENERATED_ID, { Success }, "The service-generated ID for the object."}; +const core::OutputAttribute PutGCSObject::Generation{GCS_GENERATION, { Success }, "The content generation of this object. Used for object versioning."}; +const core::OutputAttribute PutGCSObject::Metageneration{GCS_META_GENERATION, { Success }, "The metageneration of the object."}; +const core::OutputAttribute PutGCSObject::CreateTime{GCS_CREATE_TIME_ATTR, { Success }, "Unix timestamp of the object's creation in milliseconds."}; +const core::OutputAttribute PutGCSObject::UpdateTime{GCS_UPDATE_TIME_ATTR, { Success }, "Unix timestamp of the object's last modification in milliseconds."}; +const core::OutputAttribute PutGCSObject::DeleteTime{GCS_DELETE_TIME_ATTR, { Success }, "Unix timestamp of the object's deletion in milliseconds."}; +const core::OutputAttribute PutGCSObject::EncryptionAlgorithm{GCS_ENCRYPTION_ALGORITHM_ATTR, { Success }, "The algorithm used to encrypt the object."}; +const core::OutputAttribute PutGCSObject::EncryptionSha256{GCS_ENCRYPTION_SHA256_ATTR, { Success }, "The SHA256 hash of the key used to encrypt the object."}; + REGISTER_RESOURCE(PutGCSObject, Processor); } // namespace org::apache::nifi::minifi::extensions::gcp diff --git a/extensions/gcp/processors/ListGCSBucket.h b/extensions/gcp/processors/ListGCSBucket.h index df85c4b0f..50716c5d1 100644 --- a/extensions/gcp/processors/ListGCSBucket.h +++ b/extensions/gcp/processors/ListGCSBucket.h @@ -49,6 +49,55 @@ class ListGCSBucket : public GCSProcessor { EXTENSIONAPI static const core::Relationship Success; static auto relationships() { return std::array{Success}; } + EXTENSIONAPI static const core::OutputAttribute BucketOutputAttribute; + EXTENSIONAPI static const core::OutputAttribute Key; + EXTENSIONAPI static const core::OutputAttribute Filename; + EXTENSIONAPI static const core::OutputAttribute Size; + EXTENSIONAPI static const core::OutputAttribute Crc32c; + EXTENSIONAPI static const core::OutputAttribute Md5; + EXTENSIONAPI static const core::OutputAttribute OwnerEntity; + EXTENSIONAPI static const core::OutputAttribute OwnerEntityId; + EXTENSIONAPI static const core::OutputAttribute ContentEncoding; + EXTENSIONAPI static const core::OutputAttribute ContentLanguage; + EXTENSIONAPI static const core::OutputAttribute ContentDisposition; + EXTENSIONAPI static const core::OutputAttribute MediaLink; + EXTENSIONAPI static const core::OutputAttribute SelfLink; + EXTENSIONAPI static const core::OutputAttribute Etag; + EXTENSIONAPI static const core::OutputAttribute GeneratedId; + EXTENSIONAPI static const core::OutputAttribute Generation; + EXTENSIONAPI static const core::OutputAttribute Metageneration; + EXTENSIONAPI static const core::OutputAttribute CreateTime; + EXTENSIONAPI static const core::OutputAttribute UpdateTime; + EXTENSIONAPI static const core::OutputAttribute DeleteTime; + EXTENSIONAPI static const core::OutputAttribute EncryptionAlgorithm; + EXTENSIONAPI static const core::OutputAttribute EncryptionSha256; + static auto outputAttributes() { + return std::array{ + BucketOutputAttribute, + Key, + Filename, + Size, + Crc32c, + Md5, + OwnerEntity, + OwnerEntityId, + ContentEncoding, + ContentLanguage, + ContentDisposition, + MediaLink, + SelfLink, + Etag, + GeneratedId, + Generation, + Metageneration, + CreateTime, + UpdateTime, + DeleteTime, + EncryptionAlgorithm, + EncryptionSha256 + }; + } + EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false; EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false; EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_FORBIDDEN; diff --git a/extensions/gcp/processors/PutGCSObject.h b/extensions/gcp/processors/PutGCSObject.h index b96e849a1..9bd98bfd7 100644 --- a/extensions/gcp/processors/PutGCSObject.h +++ b/extensions/gcp/processors/PutGCSObject.h @@ -72,6 +72,59 @@ class PutGCSObject : public GCSProcessor { EXTENSIONAPI static const core::Relationship Failure; static auto relationships() { return std::array{Success, Failure}; } + EXTENSIONAPI static const core::OutputAttribute Message; + EXTENSIONAPI static const core::OutputAttribute Reason; + EXTENSIONAPI static const core::OutputAttribute Domain; + EXTENSIONAPI static const core::OutputAttribute BucketOutputAttribute; + EXTENSIONAPI static const core::OutputAttribute KeyOutputAttribute; + EXTENSIONAPI static const core::OutputAttribute Size; + EXTENSIONAPI static const core::OutputAttribute Crc32c; + EXTENSIONAPI static const core::OutputAttribute Md5; + EXTENSIONAPI static const core::OutputAttribute OwnerEntity; + EXTENSIONAPI static const core::OutputAttribute OwnerEntityId; + EXTENSIONAPI static const core::OutputAttribute ContentEncoding; + EXTENSIONAPI static const core::OutputAttribute ContentLanguage; + EXTENSIONAPI static const core::OutputAttribute ContentDisposition; + EXTENSIONAPI static const core::OutputAttribute MediaLink; + EXTENSIONAPI static const core::OutputAttribute SelfLink; + EXTENSIONAPI static const core::OutputAttribute Etag; + EXTENSIONAPI static const core::OutputAttribute GeneratedId; + EXTENSIONAPI static const core::OutputAttribute Generation; + EXTENSIONAPI static const core::OutputAttribute Metageneration; + EXTENSIONAPI static const core::OutputAttribute CreateTime; + EXTENSIONAPI static const core::OutputAttribute UpdateTime; + EXTENSIONAPI static const core::OutputAttribute DeleteTime; + EXTENSIONAPI static const core::OutputAttribute EncryptionAlgorithm; + EXTENSIONAPI static const core::OutputAttribute EncryptionSha256; + static auto outputAttributes() { + return std::array{ + Message, + Reason, + Domain, + BucketOutputAttribute, + KeyOutputAttribute, + Size, + Crc32c, + Md5, + OwnerEntity, + OwnerEntityId, + ContentEncoding, + ContentLanguage, + ContentDisposition, + MediaLink, + SelfLink, + Etag, + GeneratedId, + Generation, + Metageneration, + CreateTime, + UpdateTime, + DeleteTime, + EncryptionAlgorithm, + EncryptionSha256 + }; + } + EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false; EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false; EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_REQUIRED; diff --git a/extensions/http-curl/processors/InvokeHTTP.cpp b/extensions/http-curl/processors/InvokeHTTP.cpp index 9e803eec1..a7cbe5a2f 100644 --- a/extensions/http-curl/processors/InvokeHTTP.cpp +++ b/extensions/http-curl/processors/InvokeHTTP.cpp @@ -149,6 +149,13 @@ const core::Relationship InvokeHTTP::RelFailure("failure", "The original FlowFile will be routed on any type of connection failure, " "timeout or general exception. It will have new attributes detailing the request."); + +const core::OutputAttribute InvokeHTTP::StatusCode{STATUS_CODE, { Success, RelResponse, RelRetry, RelNoRetry }, "The status code that is returned"}; +const core::OutputAttribute InvokeHTTP::StatusMessage{STATUS_MESSAGE, { Success, RelResponse, RelRetry, RelNoRetry }, "The status message that is returned"}; +const core::OutputAttribute InvokeHTTP::RequestUrl{REQUEST_URL, { Success, RelResponse, RelRetry, RelNoRetry }, "The original request URL"}; +const core::OutputAttribute InvokeHTTP::TxId{TRANSACTION_ID, { Success, RelResponse, RelRetry, RelNoRetry }, "The transaction ID that is returned after reading the response"}; + + void InvokeHTTP::initialize() { logger_->log_trace("Initializing InvokeHTTP"); setSupportedProperties(properties()); diff --git a/extensions/http-curl/processors/InvokeHTTP.h b/extensions/http-curl/processors/InvokeHTTP.h index 39cb55a6e..c28b784fa 100644 --- a/extensions/http-curl/processors/InvokeHTTP.h +++ b/extensions/http-curl/processors/InvokeHTTP.h @@ -120,6 +120,19 @@ class InvokeHTTP : public core::Processor { }; } + EXTENSIONAPI static const core::OutputAttribute StatusCode; + EXTENSIONAPI static const core::OutputAttribute StatusMessage; + EXTENSIONAPI static const core::OutputAttribute RequestUrl; + EXTENSIONAPI static const core::OutputAttribute TxId; + static auto outputAttributes() { + return std::array{ + StatusCode, + StatusMessage, + RequestUrl, + TxId + }; + } + EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false; EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false; EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_ALLOWED; diff --git a/extensions/jni/jvm/NarClassLoader.h b/extensions/jni/jvm/NarClassLoader.h index 96a80d2c4..8bfc0b477 100644 --- a/extensions/jni/jvm/NarClassLoader.h +++ b/extensions/jni/jvm/NarClassLoader.h @@ -361,8 +361,8 @@ class NarClassLoader { } auto classDescription = getStringMethod("getDescription", jni_component_clazz, env, component); - description.dynamic_relationships_ = getBoolmethod("getDynamicRelationshipsSupported", jni_component_clazz, env, component); - description.dynamic_properties_ = getBoolmethod("getDynamicPropertiesSupported", jni_component_clazz, env, component); + description.supports_dynamic_relationships_ = getBoolmethod("getDynamicRelationshipsSupported", jni_component_clazz, env, component); + description.supports_dynamic_properties_ = getBoolmethod("getDynamicPropertiesSupported", jni_component_clazz, env, component); description.description_ = classDescription; diff --git a/extensions/python/PythonCreator.h b/extensions/python/PythonCreator.h index eb5c59c67..5a60f9704 100644 --- a/extensions/python/PythonCreator.h +++ b/extensions/python/PythonCreator.h @@ -101,7 +101,7 @@ class PythonCreator : public minifi::core::CoreComponent { .description_ = processor->getDescription(), .class_properties_ = processor->getPythonProperties(), .class_relationships_ = processor->getSupportedRelationships(), - .dynamic_properties_ = processor->getPythonSupportDynamicProperties(), + .supports_dynamic_properties_ = processor->getPythonSupportDynamicProperties(), .inputRequirement_ = toString(processor->getInputRequirement()), .isSingleThreaded_ = processor->isSingleThreaded()}; diff --git a/extensions/sql/processors/QueryDatabaseTable.h b/extensions/sql/processors/QueryDatabaseTable.h index b3a64b404..9c7ce98f6 100644 --- a/extensions/sql/processors/QueryDatabaseTable.h +++ b/extensions/sql/processors/QueryDatabaseTable.h @@ -65,6 +65,9 @@ class QueryDatabaseTable: public SQLProcessor, public FlowFileSource { static auto relationships() { return std::array{Success}; } EXTENSIONAPI static constexpr bool SupportsDynamicProperties = true; + EXTENSIONAPI static const core::DynamicProperty InitialMaxValue; + static auto dynamicProperties() { return std::array{InitialMaxValue}; } + EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false; EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_FORBIDDEN; EXTENSIONAPI static constexpr bool IsSingleThreaded = true; diff --git a/extensions/sql/processors/SQLProcessorStaticDefinitions.cpp b/extensions/sql/processors/SQLProcessorStaticDefinitions.cpp index 9375082be..fe9b0fd78 100644 --- a/extensions/sql/processors/SQLProcessorStaticDefinitions.cpp +++ b/extensions/sql/processors/SQLProcessorStaticDefinitions.cpp @@ -93,6 +93,10 @@ const core::Property QueryDatabaseTable::WhereClause( const core::Relationship QueryDatabaseTable::Success("success", "Successfully created FlowFile from SQL query result set."); +const core::DynamicProperty QueryDatabaseTable::InitialMaxValue("initial.maxvalue.<max_value_column>", "Initial maximum value for the specified column", + "Specifies an initial max value for max value column(s). Properties should be added in the format `initial.maxvalue.<max_value_column>`. " + "This value is only used the first time the table is accessed (when a Maximum Value Column is specified).", true); + REGISTER_RESOURCE(QueryDatabaseTable, Processor); diff --git a/extensions/standard-processors/processors/ListFile.cpp b/extensions/standard-processors/processors/ListFile.cpp index 0365a6e9a..e84be9bf0 100644 --- a/extensions/standard-processors/processors/ListFile.cpp +++ b/extensions/standard-processors/processors/ListFile.cpp @@ -81,6 +81,22 @@ const core::Property ListFile::IgnoreHiddenFiles( const core::Relationship ListFile::Success("success", "All FlowFiles that are received are routed to success"); +const core::OutputAttribute ListFile::Filename{"filename", { Success }, "The name of the file that was read from filesystem."}; +const core::OutputAttribute ListFile::Path{"path", { Success }, + "The path is set to the relative path of the file's directory on filesystem compared to the Input Directory property. " + "For example, if Input Directory is set to /tmp, then files picked up from /tmp will have the path attribute set to \"./\". " + "If the Recurse Subdirectories property is set to true and a file is picked up from /tmp/abc/1/2/3, then the path attribute will be set to \"abc/1/2/3/\"."}; +const core::OutputAttribute ListFile::AbsolutePath{"absolute.path", { Success }, + "The absolute.path is set to the absolute path of the file's directory on filesystem. " + "For example, if the Input Directory property is set to /tmp, then files picked up from /tmp will have the path attribute set to \"/tmp/\". " + "If the Recurse Subdirectories property is set to true and a file is picked up from /tmp/abc/1/2/3, then the path attribute will be set to \"/tmp/abc/1/2/3/\"."}; +const core::OutputAttribute ListFile::FileOwner{"file.owner", { Success }, "The user that owns the file in filesystem"}; +const core::OutputAttribute ListFile::FileGroup{"file.group", { Success }, "The group that owns the file in filesystem"}; +const core::OutputAttribute ListFile::FileSize{"file.size", { Success }, "The number of bytes in the file in filesystem"}; +const core::OutputAttribute ListFile::FilePermissions{"file.permissions", { Success }, + "The permissions for the file in filesystem. This is formatted as 3 characters for the owner, 3 for the group, and 3 for other users. For example rw-rw-r--"}; +const core::OutputAttribute ListFile::FileLastModifiedTime{"file.lastModifiedTime", { Success }, "The timestamp of when the file in filesystem was last modified as 'yyyy-MM-dd'T'HH:mm:ssZ'"}; + void ListFile::initialize() { setSupportedProperties(properties()); setSupportedRelationships(relationships()); diff --git a/extensions/standard-processors/processors/ListFile.h b/extensions/standard-processors/processors/ListFile.h index 970f28cf0..6f64f86d3 100644 --- a/extensions/standard-processors/processors/ListFile.h +++ b/extensions/standard-processors/processors/ListFile.h @@ -67,6 +67,27 @@ class ListFile : public core::Processor { EXTENSIONAPI static const core::Relationship Success; static auto relationships() { return std::array{Success}; } + EXTENSIONAPI static const core::OutputAttribute Filename; + EXTENSIONAPI static const core::OutputAttribute Path; + EXTENSIONAPI static const core::OutputAttribute AbsolutePath; + EXTENSIONAPI static const core::OutputAttribute FileOwner; + EXTENSIONAPI static const core::OutputAttribute FileGroup; + EXTENSIONAPI static const core::OutputAttribute FileSize; + EXTENSIONAPI static const core::OutputAttribute FilePermissions; + EXTENSIONAPI static const core::OutputAttribute FileLastModifiedTime; + static auto outputAttributes() { + return std::array{ + Filename, + Path, + AbsolutePath, + FileOwner, + FileGroup, + FileSize, + FilePermissions, + FileLastModifiedTime + }; + } + EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false; EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false; EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_FORBIDDEN; diff --git a/extensions/standard-processors/processors/ListenSyslog.cpp b/extensions/standard-processors/processors/ListenSyslog.cpp index ecd6b7477..003646862 100644 --- a/extensions/standard-processors/processors/ListenSyslog.cpp +++ b/extensions/standard-processors/processors/ListenSyslog.cpp @@ -76,6 +76,21 @@ const core::Relationship ListenSyslog::Success("success", "Incoming messages tha "When Parse Messages is set to false, all incoming message will be sent to this relationship."); const core::Relationship ListenSyslog::Invalid("invalid", "Incoming messages that do not match the expected format when parsing will be sent to this relationship."); +const core::OutputAttribute ListenSyslog::Protocol{"syslog.protocol", {}, "The protocol over which the Syslog message was received."}; +const core::OutputAttribute ListenSyslog::PortOutputAttribute{"syslog.port", {}, "The port over which the Syslog message was received."}; +const core::OutputAttribute ListenSyslog::Sender{"syslog.sender", {}, "The hostname of the Syslog server that sent the message."}; +const core::OutputAttribute ListenSyslog::Valid{"syslog.valid", {}, "An indicator of whether this message matched the expected formats. (requirement: parsing enabled)"}; +const core::OutputAttribute ListenSyslog::Priority{"syslog.priority", {}, "The priority of the Syslog message. (requirement: parsed RFC5424/RFC3164)"}; +const core::OutputAttribute ListenSyslog::Severity{"syslog.severity", {}, "The severity of the Syslog message. (requirement: parsed RFC5424/RFC3164)"}; +const core::OutputAttribute ListenSyslog::Facility{"syslog.facility", {}, "The facility of the Syslog message. (requirement: parsed RFC5424/RFC3164)"}; +const core::OutputAttribute ListenSyslog::Timestamp{"syslog.timestamp", {}, "The timestamp of the Syslog message. (requirement: parsed RFC5424/RFC3164)"}; +const core::OutputAttribute ListenSyslog::Hostname{"syslog.hostname", {}, "The hostname of the Syslog message. (requirement: parsed RFC5424/RFC3164)"}; +const core::OutputAttribute ListenSyslog::Msg{"syslog.msg", {}, "The free-form message of the Syslog message. (requirement: parsed RFC5424/RFC3164)"}; +const core::OutputAttribute ListenSyslog::Version{"syslog.version", {}, "The version of the Syslog message. (requirement: parsed RFC5424)"}; +const core::OutputAttribute ListenSyslog::AppName{"syslog.app_name", {}, "The app name of the Syslog message. (requirement: parsed RFC5424)"}; +const core::OutputAttribute ListenSyslog::ProcId{"syslog.proc_id", {}, "The proc id of the Syslog message. (requirement: parsed RFC5424)"}; +const core::OutputAttribute ListenSyslog::MsgId{"syslog.msg_id", {}, "The message id of the Syslog message. (requirement: parsed RFC5424)"}; +const core::OutputAttribute ListenSyslog::StructuredData{"syslog.structured_data", {}, "The structured data of the Syslog message. (requirement: parsed RFC5424)"}; const std::regex ListenSyslog::rfc5424_pattern_( R"(^<(?:(\d|\d{2}|1[1-8]\d|19[01]))>)" // priority diff --git a/extensions/standard-processors/processors/ListenSyslog.h b/extensions/standard-processors/processors/ListenSyslog.h index d26031332..48f84a4e7 100644 --- a/extensions/standard-processors/processors/ListenSyslog.h +++ b/extensions/standard-processors/processors/ListenSyslog.h @@ -63,6 +63,41 @@ class ListenSyslog : public NetworkListenerProcessor { EXTENSIONAPI static const core::Relationship Invalid; static auto relationships() { return std::array{Success, Invalid}; } + EXTENSIONAPI static const core::OutputAttribute Protocol; + EXTENSIONAPI static const core::OutputAttribute PortOutputAttribute; + EXTENSIONAPI static const core::OutputAttribute Sender; + EXTENSIONAPI static const core::OutputAttribute Valid; + EXTENSIONAPI static const core::OutputAttribute Priority; + EXTENSIONAPI static const core::OutputAttribute Severity; + EXTENSIONAPI static const core::OutputAttribute Facility; + EXTENSIONAPI static const core::OutputAttribute Timestamp; + EXTENSIONAPI static const core::OutputAttribute Hostname; + EXTENSIONAPI static const core::OutputAttribute Msg; + EXTENSIONAPI static const core::OutputAttribute Version; + EXTENSIONAPI static const core::OutputAttribute AppName; + EXTENSIONAPI static const core::OutputAttribute ProcId; + EXTENSIONAPI static const core::OutputAttribute MsgId; + EXTENSIONAPI static const core::OutputAttribute StructuredData; + static auto outputAttributes() { + return std::array{ + Protocol, + PortOutputAttribute, + Sender, + Valid, + Priority, + Severity, + Facility, + Timestamp, + Hostname, + Msg, + Version, + AppName, + ProcId, + MsgId, + StructuredData + }; + } + void initialize() override; void onSchedule(const std::shared_ptr<core::ProcessContext>& context, const std::shared_ptr<core::ProcessSessionFactory>& sessionFactory) override; diff --git a/extensions/standard-processors/processors/ListenTCP.cpp b/extensions/standard-processors/processors/ListenTCP.cpp index 2b600677e..05b14de6c 100644 --- a/extensions/standard-processors/processors/ListenTCP.cpp +++ b/extensions/standard-processors/processors/ListenTCP.cpp @@ -61,6 +61,9 @@ const core::Property ListenTCP::ClientAuth( const core::Relationship ListenTCP::Success("success", "Messages received successfully will be sent out this relationship."); +const core::OutputAttribute ListenTCP::PortOutputAttribute{"tcp.port", {}, "The sending port the messages were received."}; +const core::OutputAttribute ListenTCP::Sender{"tcp.sender", {}, "The sending host of the messages."}; + void ListenTCP::initialize() { setSupportedProperties(properties()); setSupportedRelationships(relationships()); diff --git a/extensions/standard-processors/processors/ListenTCP.h b/extensions/standard-processors/processors/ListenTCP.h index 5cdcfdf22..75ff908f4 100644 --- a/extensions/standard-processors/processors/ListenTCP.h +++ b/extensions/standard-processors/processors/ListenTCP.h @@ -53,6 +53,10 @@ class ListenTCP : public NetworkListenerProcessor { EXTENSIONAPI static const core::Relationship Success; static auto relationships() { return std::array{Success}; } + EXTENSIONAPI static const core::OutputAttribute PortOutputAttribute; + EXTENSIONAPI static const core::OutputAttribute Sender; + static auto outputAttributes() { return std::array{PortOutputAttribute, Sender}; } + void initialize() override; void onSchedule(const std::shared_ptr<core::ProcessContext>& context, const std::shared_ptr<core::ProcessSessionFactory>& sessionFactory) override; diff --git a/extensions/standard-processors/processors/ListenUDP.cpp b/extensions/standard-processors/processors/ListenUDP.cpp index 819919e91..70dd5ce61 100644 --- a/extensions/standard-processors/processors/ListenUDP.cpp +++ b/extensions/standard-processors/processors/ListenUDP.cpp @@ -48,6 +48,9 @@ const core::Property ListenUDP::MaxBatchSize( const core::Relationship ListenUDP::Success("success", "Messages received successfully will be sent out this relationship."); +const core::OutputAttribute ListenUDP::PortOutputAttribute{"udp.port", {}, "The sending port the messages were received."}; +const core::OutputAttribute ListenUDP::Sender{"udp.sender", {}, "The sending host of the messages."}; + void ListenUDP::initialize() { setSupportedProperties(properties()); setSupportedRelationships(relationships()); diff --git a/extensions/standard-processors/processors/ListenUDP.h b/extensions/standard-processors/processors/ListenUDP.h index 647386ac8..b18dba132 100644 --- a/extensions/standard-processors/processors/ListenUDP.h +++ b/extensions/standard-processors/processors/ListenUDP.h @@ -46,6 +46,10 @@ class ListenUDP : public NetworkListenerProcessor { EXTENSIONAPI static const core::Relationship Success; static auto relationships() { return std::array{Success}; } + EXTENSIONAPI static const core::OutputAttribute PortOutputAttribute; + EXTENSIONAPI static const core::OutputAttribute Sender; + static auto outputAttributes() { return std::array{PortOutputAttribute, Sender}; } + void initialize() override; void onSchedule(const std::shared_ptr<core::ProcessContext>& context, const std::shared_ptr<core::ProcessSessionFactory>& sessionFactory) override; diff --git a/extensions/standard-processors/processors/RetryFlowFile.cpp b/extensions/standard-processors/processors/RetryFlowFile.cpp index b5690bd23..02ef23adb 100644 --- a/extensions/standard-processors/processors/RetryFlowFile.cpp +++ b/extensions/standard-processors/processors/RetryFlowFile.cpp @@ -21,11 +21,7 @@ #include "core/PropertyValidation.h" #include "core/Resource.h" -namespace org { -namespace apache { -namespace nifi { -namespace minifi { -namespace processors { +namespace org::apache::nifi::minifi::processors { const core::Property RetryFlowFile::RetryAttribute(core::PropertyBuilder::createProperty("Retry Attribute") ->withDescription( @@ -75,6 +71,14 @@ const core::Relationship RetryFlowFile::Failure("failure", "that value to '1'. This will immediately terminate the limited feedback loop. Might also include when 'Maximum Retries' contains " " attribute expression language that does not resolve to an Integer."); +const core::OutputAttribute RetryFlowFile::RetryOutputAttribute("Retry Attribute", {}, + "User defined retry attribute is updated with the current retry count"); +const core::OutputAttribute RetryFlowFile::RetryWithUuidOutputAttribute("Retry Attribute .uuid", {}, + "User defined retry attribute with .uuid suffix is updated with the UUID of the processor that retried the FlowFile last"); + +const core::DynamicProperty RetryFlowFile::RetriesExceededAttribute("Exceeded FlowFile Attribute Key", "The value of the attribute added to the FlowFile", + "One or more dynamic properties can be used to add attributes to FlowFiles passed to the 'retries_exceeded' relationship.", true); + void RetryFlowFile::initialize() { setSupportedProperties(properties()); setSupportedRelationships(relationships()); @@ -174,8 +178,4 @@ void RetryFlowFile::setRetriesExceededAttributesOnFlowFile(core::ProcessContext* REGISTER_RESOURCE(RetryFlowFile, Processor); -} /* namespace processors */ -} /* namespace minifi */ -} /* namespace nifi */ -} /* namespace apache */ -} /* namespace org */ +} // namespace org::apache::nifi::minifi::processors diff --git a/extensions/standard-processors/processors/RetryFlowFile.h b/extensions/standard-processors/processors/RetryFlowFile.h index 66fa50e10..2ef556552 100644 --- a/extensions/standard-processors/processors/RetryFlowFile.h +++ b/extensions/standard-processors/processors/RetryFlowFile.h @@ -75,7 +75,19 @@ class RetryFlowFile : public core::Processor { }; } + EXTENSIONAPI static const core::OutputAttribute RetryOutputAttribute; + EXTENSIONAPI static const core::OutputAttribute RetryWithUuidOutputAttribute; + static auto outputAttributes() { + return std::array{ + RetryOutputAttribute, + RetryWithUuidOutputAttribute + }; + } + EXTENSIONAPI static constexpr bool SupportsDynamicProperties = true; + EXTENSIONAPI static const core::DynamicProperty RetriesExceededAttribute; + static auto dynamicProperties() { return std::array{RetriesExceededAttribute}; } + EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false; EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_REQUIRED; EXTENSIONAPI static constexpr bool IsSingleThreaded = false; diff --git a/extensions/standard-processors/processors/RouteText.cpp b/extensions/standard-processors/processors/RouteText.cpp index a68fd42b7..ec2f3afe0 100644 --- a/extensions/standard-processors/processors/RouteText.cpp +++ b/extensions/standard-processors/processors/RouteText.cpp @@ -17,10 +17,10 @@ #include "RouteText.h" +#include <algorithm> #include <map> #include <vector> #include <utility> -#include <algorithm> #include "core/ProcessSession.h" #include "core/PropertyBuilder.h" @@ -31,7 +31,6 @@ #include "range/v3/range/conversion.hpp" #include "range/v3/view/tail.hpp" #include "range/v3/view/join.hpp" -#include "range/v3/view/cache1.hpp" #include "utils/ProcessorConfigUtils.h" #include "utils/OptionalUtils.h" #include "utils/Searcher.h" @@ -104,6 +103,12 @@ const core::Relationship RouteText::Unmatched("unmatched", "Segments that do not const core::Relationship RouteText::Matched("matched", "Segments that satisfy the required user-defined rules will be routed to this Relationship"); +const core::OutputAttribute RouteText::Group("RouteText.Group", {}, + "The value captured by all capturing groups in the 'Grouping Regular Expression' property. If this property is not set, this attribute will not be added."); + +const core::DynamicProperty RouteText::RelationshipToRouteTo("Relationship Name", "value to match against", + "Routes data that matches the value specified in the Dynamic Property Value to the Relationship specified in the Dynamic Property Key.", true); + RouteText::RouteText(std::string name, const utils::Identifier& uuid) : core::Processor(std::move(name), uuid), logger_(core::logging::LoggerFactory<RouteText>::getLogger(uuid)) {} diff --git a/extensions/standard-processors/processors/RouteText.h b/extensions/standard-processors/processors/RouteText.h index 9fd0a396c..3d1e1a0d5 100644 --- a/extensions/standard-processors/processors/RouteText.h +++ b/extensions/standard-processors/processors/RouteText.h @@ -67,7 +67,13 @@ class RouteText : public core::Processor { }; } + EXTENSIONAPI static const core::OutputAttribute Group; + static auto outputAttributes() { return std::array{Group}; } + EXTENSIONAPI static constexpr bool SupportsDynamicProperties = true; + EXTENSIONAPI static const core::DynamicProperty RelationshipToRouteTo; + static auto dynamicProperties() { return std::array{RelationshipToRouteTo}; } + EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = true; EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_REQUIRED; EXTENSIONAPI static constexpr bool IsSingleThreaded = false; diff --git a/libminifi/include/agent/agent_docs.h b/libminifi/include/agent/agent_docs.h index cb51b0ed8..bf73bdb7c 100644 --- a/libminifi/include/agent/agent_docs.h +++ b/libminifi/include/agent/agent_docs.h @@ -1,4 +1,5 @@ -/*** Licensed to the Apache Software Foundation (ASF) under one or more +/** + * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 @@ -21,6 +22,8 @@ #include <vector> #include "core/Annotation.h" +#include "core/DynamicProperty.h" +#include "core/OutputAttribute.h" #include "core/Property.h" #include "core/Relationship.h" #include "utils/Export.h" @@ -38,9 +41,11 @@ struct ClassDescription { std::string full_name_{}; std::string description_{}; std::vector<core::Property> class_properties_{}; + std::vector<core::DynamicProperty> dynamic_properties_{}; std::vector<core::Relationship> class_relationships_{}; - bool dynamic_properties_ = false; - bool dynamic_relationships_ = false; + std::vector<core::OutputAttribute> output_attributes_{}; + bool supports_dynamic_properties_ = false; + bool supports_dynamic_relationships_ = false; std::string inputRequirement_{}; bool isSingleThreaded_ = false; }; @@ -77,8 +82,6 @@ class AgentDocs { return class_mappings_; } - static bool getDescription(const std::string &feature, std::string &value); - template<typename Class, ResourceType Type> static void createClassDescription(const std::string& group, const std::string& name) { Components& components = class_mappings_[group]; @@ -90,9 +93,11 @@ class AgentDocs { .full_name_ = detail::classNameWithDots<Class>(), .description_ = Class::Description, .class_properties_ = detail::toVector(Class::properties()), + .dynamic_properties_ = detail::toVector(Class::dynamicProperties()), .class_relationships_ = detail::toVector(Class::relationships()), - .dynamic_properties_ = Class::SupportsDynamicProperties, - .dynamic_relationships_ = Class::SupportsDynamicRelationships, + .output_attributes_ = detail::toVector(Class::outputAttributes()), + .supports_dynamic_properties_ = Class::SupportsDynamicProperties, + .supports_dynamic_relationships_ = Class::SupportsDynamicRelationships, .inputRequirement_ = toString(Class::InputRequirement), .isSingleThreaded_ = Class::IsSingleThreaded }); @@ -103,7 +108,7 @@ class AgentDocs { .full_name_ = detail::classNameWithDots<Class>(), .description_ = Class::Description, .class_properties_ = detail::toVector(Class::properties()), - .dynamic_properties_ = Class::SupportsDynamicProperties, + .supports_dynamic_properties_ = Class::SupportsDynamicProperties, }); } else if constexpr (Type == ResourceType::InternalResource) { components.other_components_.push_back(ClassDescription{ @@ -111,7 +116,7 @@ class AgentDocs { .short_name_ = name, .full_name_ = detail::classNameWithDots<Class>(), .class_properties_ = detail::toVector(Class::properties()), - .dynamic_properties_ = Class::SupportsDynamicProperties, + .supports_dynamic_properties_ = Class::SupportsDynamicProperties, }); } else if constexpr (Type == ResourceType::DescriptionOnly) { components.other_components_.push_back(ClassDescription{ diff --git a/libminifi/include/core/DynamicProperty.h b/libminifi/include/core/DynamicProperty.h new file mode 100644 index 000000000..90d90c9bf --- /dev/null +++ b/libminifi/include/core/DynamicProperty.h @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include <string> +#include <utility> + +namespace org::apache::nifi::minifi::core { + +class DynamicProperty { + public: + DynamicProperty() = default; // required by VS 2019 to create an empty array; not required by VS 2022 + + DynamicProperty(std::string name, std::string value, std::string description, bool supports_expression_language) + : name_(std::move(name)), + value_(std::move(value)), + description_(std::move(description)), + supports_expression_language_(supports_expression_language) { + } + + [[nodiscard]] std::string getName() const { + return name_; + } + + [[nodiscard]] std::string getValue() const { + return value_; + } + + [[nodiscard]] std::string getDescription() const { + return description_; + } + + [[nodiscard]] bool supportsExpressionLanguage() const { + return supports_expression_language_; + } + + private: + std::string name_; + std::string value_; + std::string description_; + bool supports_expression_language_ = false; +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/minifi_main/TableFormatter.h b/libminifi/include/core/OutputAttribute.h similarity index 50% copy from minifi_main/TableFormatter.h copy to libminifi/include/core/OutputAttribute.h index 38ef054ca..7b3458a5c 100644 --- a/minifi_main/TableFormatter.h +++ b/libminifi/include/core/OutputAttribute.h @@ -14,34 +14,42 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - #pragma once #include <string> #include <utility> #include <vector> -#include "core/PropertyValue.h" +#include "Relationship.h" -namespace org::apache::nifi::minifi::docs { +namespace org::apache::nifi::minifi::core { -class Table { +class OutputAttribute { public: - explicit Table(std::vector<std::string> header) : header_{std::move(header)} {} - void addRow(std::vector<std::string> row); - [[nodiscard]] std::string toString() const; + OutputAttribute() = default; // required by VS 2019 to create an empty array; not required by VS 2022 - private: - [[nodiscard]] std::vector<size_t> findWidths() const; + OutputAttribute(std::string name, std::vector<Relationship> relationships, std::string description) + : name_(std::move(name)), + relationships_(std::move(relationships)), + description_(std::move(description)) { + } - std::vector<std::string> header_; - std::vector<std::vector<std::string>> rows_; -}; + [[nodiscard]] std::string getName() const { + return name_; + } + + [[nodiscard]] std::vector<Relationship> getRelationships() const { + return relationships_; + } -std::string formatName(const std::string& name, bool is_required); -std::string formatAllowableValues(const std::vector<org::apache::nifi::minifi::core::PropertyValue>& values); -std::string formatDescription(std::string description, bool supports_expression_language = false); -std::string formatSeparator(const std::vector<size_t>& widths); -std::string formatRow(const std::vector<std::string>& items, const std::vector<size_t>& widths); + [[nodiscard]] std::string getDescription() const { + return description_; + } + + private: + std::string name_; + std::vector<Relationship> relationships_; + std::string description_; +}; -} // namespace org::apache::nifi::minifi::docs +} // namespace org::apache::nifi::minifi::core diff --git a/libminifi/include/core/Processor.h b/libminifi/include/core/Processor.h index 637d13598..824b5eb03 100644 --- a/libminifi/include/core/Processor.h +++ b/libminifi/include/core/Processor.h @@ -35,11 +35,13 @@ #include "Connectable.h" #include "Core.h" #include "core/Annotation.h" +#include "DynamicProperty.h" #include "Scheduling.h" #include "utils/TimeUtil.h" #include "core/state/nodes/MetricsBase.h" #include "ProcessorMetrics.h" #include "utils/gsl.h" +#include "OutputAttribute.h" #define ADD_GET_PROCESSOR_NAME \ std::string getProcessorType() const override { \ @@ -225,6 +227,10 @@ class Processor : public Connectable, public ConfigurableComponent, public state return metrics_; } + static std::array<DynamicProperty, 0> dynamicProperties() { return {}; } + + static std::array<OutputAttribute, 0> outputAttributes() { return {}; } + protected: virtual void notifyStop() { } diff --git a/libminifi/include/core/state/nodes/AgentInformation.h b/libminifi/include/core/state/nodes/AgentInformation.h index 5e699c6a3..4579fff31 100644 --- a/libminifi/include/core/state/nodes/AgentInformation.h +++ b/libminifi/include/core/state/nodes/AgentInformation.h @@ -237,11 +237,11 @@ class ComponentManifest : public DeviceInformation { SerializedResponseNode dyn_prop; dyn_prop.name = "supportsDynamicProperties"; - dyn_prop.value = group.dynamic_properties_; + dyn_prop.value = group.supports_dynamic_properties_; SerializedResponseNode dyn_relat; dyn_relat.name = "supportsDynamicRelationships"; - dyn_relat.value = group.dynamic_relationships_; + dyn_relat.value = group.supports_dynamic_relationships_; // only for processors if (!group.class_relationships_.empty()) { diff --git a/libminifi/include/utils/StringUtils.h b/libminifi/include/utils/StringUtils.h index 2b9b6d581..6d96fc232 100644 --- a/libminifi/include/utils/StringUtils.h +++ b/libminifi/include/utils/StringUtils.h @@ -14,8 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef LIBMINIFI_INCLUDE_UTILS_STRINGUTILS_H_ -#define LIBMINIFI_INCLUDE_UTILS_STRINGUTILS_H_ +#pragma once #include <algorithm> #include <cstring> @@ -46,8 +45,11 @@ constexpr std::strong_ordering operator<=>(const std::string& lhs, const std::st } #endif -namespace org::apache::nifi::minifi { -namespace utils { +#include "range/v3/view/transform.hpp" +#include "range/v3/view/join.hpp" +#include "range/v3/range/conversion.hpp" + +namespace org::apache::nifi::minifi::utils { template<class Char> struct string_traits; @@ -271,67 +273,40 @@ class StringUtils { } /** - * Concatenates strings stored in an arbitrary container using the provided separator. - * @tparam TChar char type of the string (char or wchar_t) - * @tparam U arbitrary container which has string or wstring value type - * @param separator that is inserted between each elements. Type should match the type of strings in container. - * @param container that contains the strings to be concatenated - * @return the result string + * Concatenates elements stored in a container to a string, using the separator. + * @param separator that is inserted between the elements (no separator at the end) + * @param container that contains the elements to be joined + * @param projection function object which can convert an element to a string + * @return the elements of the container (transformed by the projection) joined using the separator */ - template<class TChar, class U, typename std::enable_if<std::is_same<typename U::value_type, std::basic_string<TChar>>::value>::type* = nullptr> - static std::basic_string<TChar> join(const std::basic_string<TChar>& separator, const U& container) { - typedef typename U::const_iterator ITtype; - ITtype it = container.cbegin(); - std::basic_stringstream<TChar> sstream; - while (it != container.cend()) { - sstream << (*it); - ++it; - if (it != container.cend()) { - sstream << separator; - } - } - return sstream.str(); + template<typename Separator, typename Container, typename Projection> + static auto join(Separator&& separator, Container&& container, Projection&& projection) { + const auto separator_view = [&separator] { + if constexpr (std::is_convertible_v<Separator, std::string_view>) { return std::string_view{separator}; } + else if constexpr (std::is_convertible_v<Separator, std::wstring_view>) { return std::wstring_view{separator}; } + }(); + + return container + | ranges::views::transform(projection) + | ranges::views::join(separator_view) + | ranges::to<std::basic_string>(); } - /** - * Just a wrapper for the above function to be able to create separator from const char* or const wchar_t* - */ - template<class TChar, class U, typename std::enable_if<std::is_same<typename U::value_type, std::basic_string<TChar>>::value>::type* = nullptr> - static std::basic_string<TChar> join(const TChar* separator, const U& container) { - return join(std::basic_string<TChar>(separator), container); + template<typename Separator, typename Container> + requires(std::is_arithmetic_v<typename std::remove_reference_t<Container>::value_type> && std::is_convertible_v<Separator, std::string_view>) + static auto join(Separator&& separator, Container&& container) { + return join(separator, container, [](auto number) { return std::to_string(number); }); } - /** - * Concatenates string representation of integrals stored in an arbitrary container using the provided separator. - * @tparam TChar char type of the string (char or wchar_t) - * @tparam U arbitrary container which has any integral value type - * @param separator that is inserted between each elements. Type of this determines the result type. (wstring separator -> wstring) - * @param container that contains the integrals to be concatenated - * @return the result string - */ - template<class TChar, class U, typename std::enable_if<std::is_integral<typename U::value_type>::value>::type* = nullptr, - typename std::enable_if<!std::is_same<U, std::basic_string<TChar>>::value>::type* = nullptr> - static std::basic_string<TChar> join(const std::basic_string<TChar>& separator, const U& container) { - typedef typename U::const_iterator ITtype; - ITtype it = container.cbegin(); - std::basic_stringstream<TChar> sstream; - while (it != container.cend()) { - sstream << string_traits<TChar>::convert_to_string(*it); - ++it; - if (it != container.cend()) { - sstream << separator; - } - } - return sstream.str(); + template<typename Separator, typename Container> + requires(std::is_arithmetic_v<typename std::remove_reference_t<Container>::value_type> && std::is_convertible_v<Separator, std::wstring_view>) + static auto join(Separator&& separator, Container&& container) { + return join(separator, container, [](auto number) { return std::to_wstring(number); }); } - /** - * Just a wrapper for the above function to be able to create separator from const char* or const wchar_t* - */ - template<class TChar, class U, typename std::enable_if<std::is_integral<typename U::value_type>::value>::type* = nullptr, - typename std::enable_if<!std::is_same<U, std::basic_string<TChar>>::value>::type* = nullptr> - static std::basic_string<TChar> join(const TChar* separator, const U& container) { - return join(std::basic_string<TChar>(separator), container); + template<typename Separator, typename Container> + static auto join(Separator&& separator, Container&& container) { + return join(separator, container, [](auto x) { return x; }); // std::identity is not supported by AppleClang 13 } /** @@ -508,7 +483,4 @@ class StringUtils { private: }; -} // namespace utils -} // namespace org::apache::nifi::minifi - -#endif // LIBMINIFI_INCLUDE_UTILS_STRINGUTILS_H_ +} // namespace org::apache::nifi::minifi::utils diff --git a/libminifi/src/agent/JsonSchema.cpp b/libminifi/src/agent/JsonSchema.cpp index 857525e4e..af4cde860 100644 --- a/libminifi/src/agent/JsonSchema.cpp +++ b/libminifi/src/agent/JsonSchema.cpp @@ -398,18 +398,18 @@ std::string generateJsonSchema() { { std::stringstream rel_schema; rel_schema << R"({"anyOf": [)"; - if (proc.dynamic_relationships_) { + if (proc.supports_dynamic_relationships_) { rel_schema << R"({"type": "string"})"; } for (size_t rel_idx = 0; rel_idx < proc.class_relationships_.size(); ++rel_idx) { - if (rel_idx != 0 || proc.dynamic_relationships_) rel_schema << ", "; + if (rel_idx != 0 || proc.supports_dynamic_relationships_) rel_schema << ", "; rel_schema << R"({"const": ")" << escape(proc.class_relationships_[rel_idx].getName()) << "\"}"; } rel_schema << "]}"; relationships[proc.short_name_] = std::move(rel_schema).str(); } - writeProperties(proc.class_properties_, proc.dynamic_properties_, schema); + writeProperties(proc.class_properties_, proc.supports_dynamic_properties_, schema); schema << "}"; // "properties" schema << "}"; // "then" @@ -428,7 +428,7 @@ std::string generateJsonSchema() { << R"("required": ["Properties"],)" << R"("properties": {)"; - writeProperties(service.class_properties_, service.dynamic_properties_, schema); + writeProperties(service.class_properties_, service.supports_dynamic_properties_, schema); schema << "}"; // "properties" schema << "}"; // "then" diff --git a/libminifi/src/agent/agent_docs.cpp b/libminifi/src/agent/agent_docs.cpp index 0d8bba847..0b4091bad 100644 --- a/libminifi/src/agent/agent_docs.cpp +++ b/libminifi/src/agent/agent_docs.cpp @@ -17,28 +17,8 @@ #include "agent/agent_docs.h" -namespace org { -namespace apache { -namespace nifi { -namespace minifi { +namespace org::apache::nifi::minifi { std::map<std::string, Components> AgentDocs::class_mappings_; -bool AgentDocs::getDescription(const std::string &feature, std::string &value) { - for (const auto& [module_name, module_classes] : class_mappings_) { - for (const auto& class_descriptions : {module_classes.processors_, module_classes.controller_services_, module_classes.other_components_}) { - for (const auto& class_description : class_descriptions) { - if (class_description.full_name_ == feature || class_description.short_name_ == feature) { - value = class_description.description_; - return true; - } - } - } - } - return false; -} - -} // namespace minifi -} // namespace nifi -} // namespace apache -} // namespace org +} // namespace org::apache::nifi::minifi diff --git a/libminifi/test/unit/StringUtilsTests.cpp b/libminifi/test/unit/StringUtilsTests.cpp index 18fb1274e..1f12032f6 100644 --- a/libminifi/test/unit/StringUtilsTests.cpp +++ b/libminifi/test/unit/StringUtilsTests.cpp @@ -16,7 +16,6 @@ * limitations under the License. */ -#include <algorithm> #include <cstdint> #include <list> #include <optional> @@ -32,7 +31,6 @@ using org::apache::nifi::minifi::utils::StringUtils; using utils::as_string; - // NOLINTBEGIN(readability-container-size-empty) TEST_CASE("StringUtils::chomp works correctly", "[StringUtils][chomp]") { @@ -157,20 +155,41 @@ TEST_CASE("StringUtils::replaceEnvironmentVariables works correctly", "[replaceE TEST_CASE("TestStringUtils::testJoin", "[test string join]") { std::set<std::string> strings = {"3", "2", "1"}; - REQUIRE(StringUtils::join(",", strings) == "1,2,3"); + CHECK(StringUtils::join(",", strings) == "1,2,3"); std::wstring sep = L"é"; std::vector<std::wstring> wstrings = {L"1", L"2"}; - REQUIRE(StringUtils::join(sep, wstrings) == L"1é2"); + CHECK(StringUtils::join(sep, wstrings) == L"1é2"); std::list<uint64_t> ulist = {1, 2}; - REQUIRE(StringUtils::join(sep, ulist) == L"1é2"); + CHECK(StringUtils::join(sep, ulist) == L"1é2"); - REQUIRE(StringUtils::join(">", ulist) == "1>2"); + CHECK(StringUtils::join(">", ulist) == "1>2"); - REQUIRE(StringUtils::join("", ulist) == "12"); + CHECK(StringUtils::join("", ulist) == "12"); - REQUIRE(StringUtils::join("this separator wont appear", std::vector<std::string>()) == ""); + CHECK(StringUtils::join("this separator wont appear", std::vector<std::string>()) == ""); +} + +TEST_CASE("Test the join function with a projection", "[join][projection]") { + std::vector<std::string> fruits = { "APPLE", "OrAnGe" }; + CHECK(StringUtils::join(", ", fruits, [](auto fruit) { return StringUtils::toLower(fruit); }) == "apple, orange"); + + std::set numbers_set = { 3, 2, 1 }; + CHECK(StringUtils::join(", ", numbers_set, [](auto x) { return std::to_string(x); }) == "1, 2, 3"); + + std::wstring sep = L"é"; + std::vector numbers_vector = { 1, 2 }; + CHECK(StringUtils::join(sep, numbers_vector, [](auto x) { return std::to_wstring(x); }) == L"1é2"); + + std::list<uint64_t> numbers_list = { 1, 2, 3 }; + CHECK(StringUtils::join(", ", numbers_list, [](auto x) { return std::to_string(x * x); }) == "1, 4, 9"); + + CHECK(StringUtils::join(std::string_view{"-"}, numbers_list, [](auto x) { return std::to_string(5 * x); }) == "5-10-15"); + + CHECK(StringUtils::join(std::string{}, numbers_list, [](auto x) { return '<' + std::to_string(x) + '>'; }) == "<1><2><3>"); + + CHECK(StringUtils::join("this separator wont appear", std::vector<int>{}, [](int) { return std::string_view{"foo"}; }) == ""); } TEST_CASE("TestStringUtils::trim", "[test trim]") { @@ -536,4 +555,5 @@ TEST_CASE("StringUtils::matchesSequence works correctly", "[matchesSequence]") { REQUIRE(StringUtils::matchesSequence("xxxabcxxxabcxxxdefxxx", {"abc", "abc", "def"})); REQUIRE(!StringUtils::matchesSequence("xxxabcxxxdefxxx", {"abc", "abc", "def"})); } + // NOLINTEND(readability-container-size-empty) diff --git a/minifi_main/AgentDocs.cpp b/minifi_main/AgentDocs.cpp index 649601a7a..c8b342827 100644 --- a/minifi_main/AgentDocs.cpp +++ b/minifi_main/AgentDocs.cpp @@ -19,15 +19,53 @@ #include <map> #include <iostream> -#include <set> #include <string> +#include <utility> +#include <vector> + +#include "range/v3/action/transform.hpp" +#include "range/v3/view/transform.hpp" +#include "range/v3/view/join.hpp" +#include "range/v3/range/conversion.hpp" #include "agent/agent_docs.h" #include "agent/agent_version.h" #include "core/Core.h" -#include "range/v3/action/transform.hpp" +#include "core/PropertyValue.h" +#include "core/Relationship.h" #include "TableFormatter.h" #include "utils/file/FileUtils.h" +#include "utils/StringUtils.h" + +namespace { + +namespace minifi = org::apache::nifi::minifi; + +std::string formatName(const std::string& name, bool is_required) { + if (is_required) { + return "**" + name + "**"; + } else { + return name; + } +} + +std::string formatAllowableValues(const std::vector<minifi::core::PropertyValue>& values) { + return values + | ranges::views::transform([](const auto& value) { return value.to_string(); }) + | ranges::views::join(std::string_view{"<br/>"}) + | ranges::to<std::string>(); +} + +std::string formatDescription(std::string description, bool supports_expression_language = false) { + org::apache::nifi::minifi::utils::StringUtils::replaceAll(description, "\n", "<br/>"); + return supports_expression_language ? description + "<br/>**Supports Expression Language: true**" : description; +} + +std::string formatListOfRelationships(const std::vector<minifi::core::Relationship>& relationships) { + return minifi::utils::StringUtils::join(", ", relationships, [](const auto& relationship) { return relationship.getName(); }); +} + +} // namespace namespace org::apache::nifi::minifi::docs { @@ -43,27 +81,17 @@ void AgentDocs::generate(const std::filesystem::path& docsdir, std::ostream &gen std::map<std::string, ClassDescription> processorSet; for (const auto &group : minifi::AgentBuild::getExtensions()) { struct Components descriptions = build_description_.getClassDescriptions(group); - for (const auto &processorName : descriptions.processors_) { - processorSet.insert(std::make_pair(extractClassName(processorName.full_name_), processorName)); + for (const auto& processor_description : descriptions.processors_) { + processorSet.insert(std::make_pair(extractClassName(processor_description.full_name_), processor_description)); } } for (const auto &processor : processorSet) { const auto& filename = docsdir / processor.first; std::ofstream outfile(filename); - { - std::string description; - bool foundDescription = minifi::AgentDocs::getDescription(processor.first, description); - if (!foundDescription) { - foundDescription = minifi::AgentDocs::getDescription(processor.second.full_name_, description); - } - - outfile << "## " << processor.first << "\n\n"; - if (foundDescription) { - outfile << "### Description\n\n"; - outfile << description << '\n'; - } - } + outfile << "## " << processor.first << "\n\n"; + outfile << "### Description\n\n"; + outfile << processor.second.description_ << '\n'; outfile << "\n### Properties\n\n"; outfile << "In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. " @@ -77,14 +105,39 @@ void AgentDocs::generate(const std::filesystem::path& docsdir, std::ostream &gen formatAllowableValues(prop.getAllowedValues()), formatDescription(prop.getDescription(), prop.supportsExpressionLanguage())}); } - outfile << properties.toString(); + outfile << properties.toString() << '\n'; + + if (!processor.second.dynamic_properties_.empty()) { + outfile << "### Dynamic Properties\n\n"; + Table dynamic_properties{{"Name", "Value", "Description"}}; + for (const auto& dynamic_property : processor.second.dynamic_properties_) { + dynamic_properties.addRow({ + formatName(dynamic_property.getName(), false), + dynamic_property.getValue(), + formatDescription(dynamic_property.getDescription(), dynamic_property.supportsExpressionLanguage()) + }); + } + outfile << dynamic_properties.toString() << '\n'; + } - outfile << "\n### Relationships\n\n"; + outfile << "### Relationships\n\n"; Table relationships{{"Name", "Description"}}; for (const auto &rel : processor.second.class_relationships_) { relationships.addRow({rel.getName(), formatDescription(rel.getDescription())}); } outfile << relationships.toString() << '\n'; + + if (!processor.second.output_attributes_.empty()) { + outfile << "### Output Attributes\n\n"; + Table output_attributes{{"Attribute", "Relationship", "Description"}}; + for (const auto& output_attribute : processor.second.output_attributes_) { + output_attributes.addRow({ + output_attribute.getName(), + formatListOfRelationships(output_attribute.getRelationships()), + formatDescription(output_attribute.getDescription())}); + } + outfile << output_attributes.toString() << '\n'; + } } std::map<std::string, std::filesystem::path> fileList; diff --git a/minifi_main/TableFormatter.cpp b/minifi_main/TableFormatter.cpp index f8dcd1817..200ba2794 100644 --- a/minifi_main/TableFormatter.cpp +++ b/minifi_main/TableFormatter.cpp @@ -17,10 +17,35 @@ #include "TableFormatter.h" +#include <algorithm> + #include "range/v3/view/transform.hpp" #include "range/v3/view/join.hpp" #include "range/v3/range/conversion.hpp" +#include "utils/gsl.h" + +namespace { + +std::string formatRow(const std::vector<std::string>& items, const std::vector<size_t>& widths) { + std::string result; + for (size_t i = 0; i < items.size(); ++i) { + size_t padding = widths[i] - items[i].length() + 1; + result.append("| ").append(items[i]).append(std::string(padding, ' ')); + } + return result + "|\n"; +} + +std::string formatSeparator(const std::vector<size_t>& widths) { + std::string result = widths + | ranges::views::transform([](size_t width) { return std::string(width + 2, '-'); }) + | ranges::views::join('|') + | ranges::to<std::string>(); + return '|' + result + "|\n"; +} + +} // namespace + namespace org::apache::nifi::minifi::docs { void Table::addRow(std::vector<std::string> row) { @@ -50,41 +75,4 @@ std::string Table::toString() const { return result; } -std::string formatName(const std::string& name, bool is_required) { - if (is_required) { - return "**" + name + "**"; - } else { - return name; - } -} - -std::string formatAllowableValues(const std::vector<org::apache::nifi::minifi::core::PropertyValue>& values) { - return values - | ranges::views::transform([](const auto& value) { return value.to_string(); }) - | ranges::views::join(std::string_view{"<br/>"}) - | ranges::to<std::string>(); -} - -std::string formatDescription(std::string description, bool supports_expression_language) { - org::apache::nifi::minifi::utils::StringUtils::replaceAll(description, "\n", "<br/>"); - return supports_expression_language ? description + "<br/>**Supports Expression Language: true**" : description; -} - -std::string formatSeparator(const std::vector<size_t>& widths) { - std::string result = widths - | ranges::views::transform([](size_t width) { return std::string(width + 2, '-'); }) - | ranges::views::join('|') - | ranges::to<std::string>(); - return '|' + result + "|\n"; -} - -std::string formatRow(const std::vector<std::string>& items, const std::vector<size_t>& widths) { - std::string result; - for (size_t i = 0; i < items.size(); ++i) { - size_t padding = widths[i] - items[i].length() + 1; - result.append("| ").append(items[i]).append(std::string(padding, ' ')); - } - return result + "|\n"; -} - } // namespace org::apache::nifi::minifi::docs diff --git a/minifi_main/TableFormatter.h b/minifi_main/TableFormatter.h index 38ef054ca..d00210967 100644 --- a/minifi_main/TableFormatter.h +++ b/minifi_main/TableFormatter.h @@ -21,8 +21,6 @@ #include <utility> #include <vector> -#include "core/PropertyValue.h" - namespace org::apache::nifi::minifi::docs { class Table { @@ -38,10 +36,4 @@ class Table { std::vector<std::vector<std::string>> rows_; }; -std::string formatName(const std::string& name, bool is_required); -std::string formatAllowableValues(const std::vector<org::apache::nifi::minifi::core::PropertyValue>& values); -std::string formatDescription(std::string description, bool supports_expression_language = false); -std::string formatSeparator(const std::vector<size_t>& widths); -std::string formatRow(const std::vector<std::string>& items, const std::vector<size_t>& widths); - } // namespace org::apache::nifi::minifi::docs
