This is an automated email from the ASF dual-hosted git repository. markap14 pushed a commit to branch NIFI-15258 in repository https://gitbox.apache.org/repos/asf/nifi.git
commit e1f0916fe7f23d47036f013a2ead0ace43641e87 Author: Matt Gilman <[email protected]> AuthorDate: Thu Dec 18 16:51:09 2025 -0500 NIFI-15361: Allowing configuration step documentation to be returned … (#10667) * NIFI-15361: Allowing configuration step documentation to be returned independent of any connector instances. --- .../connectors/kafkas3/KafkaConnectionStep.java | 32 ----------- .../org.apache.nifi.components.connector.Connector | 1 + .../Kafka_Connection.md | 27 +++++++++ .../Kafka_Topics.md | 34 +++++++++++ .../S3_Configuration.md | 65 ++++++++++++++++++++++ .../api/dto/ConfigurationStepConfigurationDTO.java | 14 ++--- .../web/api/entity/StepDocumentationEntity.java | 35 ++++++++++++ .../nifi/manifest/RuntimeManifestService.java | 16 +++++- .../manifest/StandardRuntimeManifestService.java | 45 +++++++++++++++ .../StandardRuntimeManifestServiceTest.java | 48 ++++++++++++++++ .../org.example.TestConnector/Another_Test_Step.md | 4 ++ .../org.example.TestConnector/Test_Step.md | 8 +++ .../org/apache/nifi/web/NiFiServiceFacade.java | 12 ++++ .../apache/nifi/web/StandardNiFiServiceFacade.java | 5 ++ .../java/org/apache/nifi/web/api/FlowResource.java | 45 +++++++++++++++ .../org/apache/nifi/web/api/dto/DtoFactory.java | 34 ++++++++++- .../configuration/WebApplicationConfiguration.java | 1 + .../nifi/web/controller/ControllerFacade.java | 16 ++++++ .../web/dao/impl/StandardConnectorDAOTest.java | 23 +++++++- 19 files changed, 420 insertions(+), 45 deletions(-) diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaConnectionStep.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaConnectionStep.java index 22ae253a98..793b1e9be0 100644 --- a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaConnectionStep.java +++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaConnectionStep.java @@ -29,37 +29,6 @@ import java.util.List; public class KafkaConnectionStep { public static final String STEP_NAME = "Kafka Connection"; - // Temporary documentation to verify the inline documentation feature in the UI. - // In practice, this content should probably be loaded from an external file. - private static final String KAFKA_CONNECTION_DOCUMENTATION = """ - # Kafka Connection Configuration - - This step configures the connection to your Apache Kafka cluster. - - ## Kafka Server Settings - - Enter the bootstrap servers for your Kafka cluster. You can specify multiple brokers - as a comma-separated list (e.g., `broker1:9092,broker2:9092,broker3:9092`). - - ### Security Configuration - - Select the appropriate security protocol based on your Kafka cluster configuration: - - | Protocol | Description | - |----------|-------------| - | PLAINTEXT | No encryption or authentication | - | SSL | TLS encryption without SASL authentication | - | SASL_PLAINTEXT | SASL authentication without encryption | - | SASL_SSL | Both SASL authentication and TLS encryption (recommended) | - - If using SASL authentication, provide your username and password credentials. - - ## Schema Registry (Optional) - - If your Kafka topics use Avro, Protobuf, or JSON Schema, configure the Schema Registry - URL to enable schema-based serialization and deserialization. - """; - public static final ConnectorPropertyDescriptor KAFKA_BROKERS = new ConnectorPropertyDescriptor.Builder() .name("Kafka Brokers") .description("A comma-separated list of Kafka brokers to connect to.") @@ -158,7 +127,6 @@ public class KafkaConnectionStep { public static final ConfigurationStep KAFKA_CONNECTION_STEP = new ConfigurationStep.Builder() .name(STEP_NAME) .description("Configure Kafka connection settings") - .documentation(KAFKA_CONNECTION_DOCUMENTATION) .propertyGroups(List.of( KAFKA_SERVER_GROUP, SCHEMA_REGISTRY_GROUP diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector index de5fe10a48..d5b0fe6db6 100644 --- a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector +++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector @@ -14,3 +14,4 @@ # limitations under the License. org.apache.nifi.connectors.kafkas3.KafkaToS3 + diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/Kafka_Connection.md b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/Kafka_Connection.md new file mode 100644 index 0000000000..1621b37061 --- /dev/null +++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/Kafka_Connection.md @@ -0,0 +1,27 @@ +# Kafka Connection Configuration + +This step configures the connection to your Apache Kafka cluster. + +## Kafka Server Settings + +Enter the bootstrap servers for your Kafka cluster. You can specify multiple brokers +as a comma-separated list (e.g., `broker1:9092,broker2:9092,broker3:9092`). + +### Security Configuration + +Select the appropriate security protocol based on your Kafka cluster configuration: + +| Protocol | Description | +|----------|-------------| +| PLAINTEXT | No encryption or authentication | +| SSL | TLS encryption without SASL authentication | +| SASL_PLAINTEXT | SASL authentication without encryption | +| SASL_SSL | Both SASL authentication and TLS encryption (recommended) | + +If using SASL authentication, provide your username and password credentials. + +## Schema Registry (Optional) + +If your Kafka topics use Avro, Protobuf, or JSON Schema, configure the Schema Registry +URL to enable schema-based serialization and deserialization. + diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/Kafka_Topics.md b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/Kafka_Topics.md new file mode 100644 index 0000000000..2e2739676c --- /dev/null +++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/Kafka_Topics.md @@ -0,0 +1,34 @@ +# Kafka Topics Configuration + +This step configures which Kafka topics to consume from and how to consume them. + +## Topic Names + +Select one or more Kafka topics to consume from. The connector will automatically +fetch the list of available topics from your Kafka cluster. + +## Consumer Group ID + +Specify a unique consumer group ID for this connector. Kafka uses consumer groups +to track message offsets and ensure each message is processed only once within a group. + +**Best Practice:** Use a descriptive name that identifies the purpose of this connector, +such as `kafka-to-s3-production` or `analytics-pipeline-consumer`. + +## Offset Reset + +Controls the behavior when no prior offset exists or the current offset is invalid: + +| Value | Description | +|-------|-------------| +| earliest | Start reading from the oldest available message | +| latest | Start reading from the newest messages only | +| none | Fail if no prior offset exists | + +## Kafka Data Format + +Specify the format of messages in your Kafka topics: + +- **Avro**: Messages are encoded using Apache Avro (requires Schema Registry) +- **JSON**: Messages are plain JSON objects + diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/S3_Configuration.md b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/S3_Configuration.md new file mode 100644 index 0000000000..fcd6092d43 --- /dev/null +++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/S3_Configuration.md @@ -0,0 +1,65 @@ +# S3 Configuration + +This step configures the connection to Amazon S3 or an S3-compatible storage system. + +## S3 Destination Configuration + +### S3 Bucket + +The name of the S3 bucket where data will be written. Ensure the bucket exists +and your credentials have write permissions. + +### S3 Prefix + +An optional prefix (folder path) to prepend to all object keys. For example, +setting this to `kafka-data/` will result in objects like: +`kafka-data/2024/01/15/data-001.json` + +### S3 Region + +The AWS region where your S3 bucket is located (e.g., `us-east-1`, `eu-west-1`). + +### S3 Data Format + +The format to use when writing objects to S3: + +- **JSON**: Write data as JSON objects (human-readable) +- **Avro**: Write data in Apache Avro format (compact, schema-embedded) + +### S3 Endpoint Override URL + +Leave this blank for standard AWS S3. Use this field only when connecting to +S3-compatible storage systems like MinIO, LocalStack, or Ceph. + +## Merge Configuration + +### Target Object Size + +The connector merges multiple Kafka messages together before writing to S3 to +reduce the number of objects created. Specify the target size for merged objects +(e.g., `256 MB`, `1 GB`). + +### Merge Latency + +The maximum time to wait while collecting messages before writing to S3. Even if +the target size hasn't been reached, data will be written after this duration to +ensure timely data availability. + +## S3 Credentials + +### Authentication Strategy + +Choose how to authenticate with AWS: + +| Strategy | Description | +|----------|-------------| +| Access Key ID and Secret Key | Use explicit AWS access credentials | +| Default AWS Credentials | Use the default credential chain (environment variables, IAM roles, etc.) | + +### S3 Access Key ID / Secret Access Key + +When using explicit credentials, provide your AWS access key ID and secret access key. + +**Security Note:** Store sensitive credentials using NiFi's sensitive property +protection or parameter contexts with secret providers. + diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ConfigurationStepConfigurationDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ConfigurationStepConfigurationDTO.java index 94b0c7ff71..4b58f9df2c 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ConfigurationStepConfigurationDTO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ConfigurationStepConfigurationDTO.java @@ -29,7 +29,7 @@ public class ConfigurationStepConfigurationDTO { private String configurationStepName; private String configurationStepDescription; - private String configurationStepDocumentation; + private boolean documented; private List<PropertyGroupConfigurationDTO> propertyGroupConfigurations; /** @@ -57,15 +57,15 @@ public class ConfigurationStepConfigurationDTO { } /** - * @return the configuration step documentation in markdown + * @return whether this step has extended documentation available */ - @Schema(description = "Extended documentation or help text for the configuration step.") - public String getConfigurationStepDocumentation() { - return configurationStepDocumentation; + @Schema(description = "Whether extended documentation is available for this configuration step.") + public boolean isDocumented() { + return documented; } - public void setConfigurationStepDocumentation(final String configurationStepDocumentation) { - this.configurationStepDocumentation = configurationStepDocumentation; + public void setDocumented(final boolean documented) { + this.documented = documented; } /** diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/StepDocumentationEntity.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/StepDocumentationEntity.java new file mode 100644 index 0000000000..e01da34a1d --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/StepDocumentationEntity.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.entity; + +import jakarta.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "stepDocumentationEntity") +public class StepDocumentationEntity extends Entity { + + private String stepDocumentation; + + public String getStepDocumentation() { + return stepDocumentation; + } + + public void setStepDocumentation(final String stepDocumentation) { + this.stepDocumentation = stepDocumentation; + } + +} + diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/manifest/RuntimeManifestService.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/manifest/RuntimeManifestService.java index 3acbfda5b4..60c9463cbd 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/manifest/RuntimeManifestService.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/manifest/RuntimeManifestService.java @@ -43,12 +43,24 @@ public interface RuntimeManifestService { RuntimeManifest getManifestForBundle(String group, String artifact, String version); /** - * Returns a mapping of additionalDetails for the speicfied bundle. + * Returns a mapping of additionalDetails for the specified bundle. * * @param group The bundle group * @param artifact The bundle artifact * @param version The bundle version - * @return The additionaDetails mapping + * @return The additionalDetails mapping */ Map<String, File> discoverAdditionalDetails(String group, String artifact, String version); + + /** + * Returns a mapping of step documentation files for the specified Connector type. + * The keys are the step names and the values are the corresponding documentation files. + * + * @param group The bundle group + * @param artifact The bundle artifact + * @param version The bundle version + * @param connectorType The fully qualified class name of the Connector + * @return The step documentation mapping + */ + Map<String, File> discoverStepDocumentation(String group, String artifact, String version, String connectorType); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java index 1a7f3ef214..197951ccb0 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java @@ -401,6 +401,51 @@ public class StandardRuntimeManifestService implements RuntimeManifestService { return additionalDetailsMap; } + @Override + public Map<String, File> discoverStepDocumentation(final String group, final String artifact, final String version, final String connectorType) { + final BundleCoordinate bundleCoordinate = new BundleCoordinate(group, artifact, version); + final Bundle bundle = extensionManager.getBundle(bundleCoordinate); + + if (bundle == null) { + throw new ResourceNotFoundException("Unable to find bundle [" + bundleCoordinate + "]"); + } + + return discoverStepDocumentation(bundle.getBundleDetails(), connectorType); + } + + private Map<String, File> discoverStepDocumentation(final BundleDetails bundleDetails, final String connectorType) { + final Map<String, File> stepDocsMap = new LinkedHashMap<>(); + + final File stepDocsDir = new File(bundleDetails.getWorkingDirectory(), "META-INF/docs/step-documentation/" + connectorType); + if (!stepDocsDir.exists()) { + LOGGER.debug("No step-documentation directory found for [{}] under [{}]", connectorType, bundleDetails.getWorkingDirectory().getAbsolutePath()); + return stepDocsMap; + } + + final File[] stepDocFiles = stepDocsDir.listFiles(); + if (stepDocFiles == null) { + return stepDocsMap; + } + + for (final File stepDocFile : stepDocFiles) { + if (!stepDocFile.isFile() || !stepDocFile.getName().endsWith(".md")) { + LOGGER.debug("Skipping [{}], not a markdown file...", stepDocFile.getAbsolutePath()); + continue; + } + + final String fileName = stepDocFile.getName().substring(0, stepDocFile.getName().length() - 3); + final String stepName = fileNameToStepName(fileName); + stepDocsMap.put(stepName, stepDocFile); + LOGGER.debug("Discovered step documentation for step [{}] at [{}]", stepName, stepDocFile.getAbsolutePath()); + } + + return stepDocsMap; + } + + private String fileNameToStepName(final String fileName) { + return fileName.replace("_", " "); + } + private Map<String, String> loadAdditionalDetails(final BundleDetails bundleDetails) { final Map<String, File> additionalDetailsMap = discoverAdditionalDetails(bundleDetails); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/manifest/StandardRuntimeManifestServiceTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/manifest/StandardRuntimeManifestServiceTest.java index fae3db4f85..fe9b3d93ef 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/manifest/StandardRuntimeManifestServiceTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/manifest/StandardRuntimeManifestServiceTest.java @@ -31,6 +31,7 @@ import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.PythonBundle; import org.apache.nifi.processor.Processor; import org.apache.nifi.python.PythonProcessorDetails; +import org.apache.nifi.web.ResourceNotFoundException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -38,11 +39,14 @@ import java.io.File; import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import static java.util.Collections.emptySet; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -163,6 +167,50 @@ public class StandardRuntimeManifestServiceTest { assertEquals(0, controllerServiceDefinitions.size()); } + @Test + public void testDiscoverStepDocumentation() { + when(extensionManager.getBundle(testComponentsBundle.getBundleDetails().getCoordinate())).thenReturn(testComponentsBundle); + + final BundleCoordinate coordinate = testComponentsBundle.getBundleDetails().getCoordinate(); + final Map<String, File> stepDocs = runtimeManifestService.discoverStepDocumentation( + coordinate.getGroup(), coordinate.getId(), coordinate.getVersion(), "org.example.TestConnector"); + + assertNotNull(stepDocs); + assertEquals(2, stepDocs.size()); + assertTrue(stepDocs.containsKey("Test Step")); + assertTrue(stepDocs.containsKey("Another Test Step")); + + final File testStepFile = stepDocs.get("Test Step"); + assertNotNull(testStepFile); + assertTrue(testStepFile.exists()); + assertEquals("Test_Step.md", testStepFile.getName()); + + final File anotherTestStepFile = stepDocs.get("Another Test Step"); + assertNotNull(anotherTestStepFile); + assertTrue(anotherTestStepFile.exists()); + assertEquals("Another_Test_Step.md", anotherTestStepFile.getName()); + } + + @Test + public void testDiscoverStepDocumentationWithNonExistentConnector() { + when(extensionManager.getBundle(testComponentsBundle.getBundleDetails().getCoordinate())).thenReturn(testComponentsBundle); + + final BundleCoordinate coordinate = testComponentsBundle.getBundleDetails().getCoordinate(); + final Map<String, File> stepDocs = runtimeManifestService.discoverStepDocumentation( + coordinate.getGroup(), coordinate.getId(), coordinate.getVersion(), "org.example.NonExistentConnector"); + + assertNotNull(stepDocs); + assertTrue(stepDocs.isEmpty()); + } + + @Test + public void testDiscoverStepDocumentationWithNonExistentBundle() { + when(extensionManager.getBundle(new BundleCoordinate("org.example", "nonexistent", "1.0.0"))).thenReturn(null); + + assertThrows(ResourceNotFoundException.class, () -> + runtimeManifestService.discoverStepDocumentation("org.example", "nonexistent", "1.0.0", "org.example.TestConnector")); + } + /** * Override getFrameworkBundle to provide a mocked Bundle. */ diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/TestRuntimeManifest/nifi-test-components-nar/META-INF/docs/step-documentation/org.example.TestConnector/Another_Test_Step.md b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/TestRuntimeManifest/nifi-test-components-nar/META-INF/docs/step-documentation/org.example.TestConnector/Another_Test_Step.md new file mode 100644 index 0000000000..4673395a0c --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/TestRuntimeManifest/nifi-test-components-nar/META-INF/docs/step-documentation/org.example.TestConnector/Another_Test_Step.md @@ -0,0 +1,4 @@ +# Another Test Step + +Documentation for another step. + diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/TestRuntimeManifest/nifi-test-components-nar/META-INF/docs/step-documentation/org.example.TestConnector/Test_Step.md b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/TestRuntimeManifest/nifi-test-components-nar/META-INF/docs/step-documentation/org.example.TestConnector/Test_Step.md new file mode 100644 index 0000000000..22a6c822b8 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/TestRuntimeManifest/nifi-test-components-nar/META-INF/docs/step-documentation/org.example.TestConnector/Test_Step.md @@ -0,0 +1,8 @@ +# Test Step Documentation + +This is test documentation for the Test Step. + +## Configuration + +Configure the test step properties as needed. + diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java index b10cf18a45..14472b78bd 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java @@ -654,6 +654,18 @@ public interface NiFiServiceFacade { */ String getAdditionalDetails(String group, String artifact, String version, String type); + /** + * Return the step documentation for the specified Connector configuration step. + * + * @param group The bundle group + * @param artifact The bundle artifact + * @param version The bundle version + * @param connectorType The fully qualified class name of the Connector + * @param stepName The name of the configuration step + * @return The step documentation markdown content + */ + String getStepDocumentation(String group, String artifact, String version, String connectorType, String stepName); + /** * Returns the list of parameter provider types. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java index f9309f931d..9130a348f7 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java @@ -4374,6 +4374,11 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { return controllerFacade.getAdditionalDetails(group, artifact, version, type); } + @Override + public String getStepDocumentation(final String group, final String artifact, final String version, final String connectorType, final String stepName) { + return controllerFacade.getStepDocumentation(group, artifact, version, connectorType, stepName); + } + @Override public Set<DocumentedTypeDTO> getParameterProviderTypes(final String bundleGroup, final String bundleArtifact, final String type) { return controllerFacade.getParameterProviderTypes(bundleGroup, bundleArtifact, type); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowResource.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowResource.java index 643ff8dc61..ff48508762 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowResource.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowResource.java @@ -106,6 +106,7 @@ import org.apache.nifi.web.api.entity.AboutEntity; import org.apache.nifi.web.api.entity.ActionEntity; import org.apache.nifi.web.api.entity.ActivateControllerServicesEntity; import org.apache.nifi.web.api.entity.AdditionalDetailsEntity; +import org.apache.nifi.web.api.entity.StepDocumentationEntity; import org.apache.nifi.web.api.entity.BannerEntity; import org.apache.nifi.web.api.entity.BulletinBoardEntity; import org.apache.nifi.web.api.entity.ClearBulletinsForGroupRequestEntity; @@ -2341,6 +2342,50 @@ public class FlowResource extends ApplicationResource { return generateOkResponse(entity).build(); } + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("step-documentation/{group}/{artifact}/{version}/{connectorType}/{stepName}") + @Operation( + summary = "Retrieves the step documentation for the specified Connector configuration step.", + description = NON_GUARANTEED_ENDPOINT, + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = StepDocumentationEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The step documentation for the coordinates could not be located.") + }, + security = { + @SecurityRequirement(name = "Read - /flow") + } + ) + public Response getStepDocumentation( + @Parameter(description = "The bundle group", required = true) + @PathParam("group") final String group, + @Parameter(description = "The bundle artifact", required = true) + @PathParam("artifact") final String artifact, + @Parameter(description = "The bundle version", required = true) + @PathParam("version") final String version, + @Parameter(description = "The fully qualified Connector type", required = true) + @PathParam("connectorType") final String connectorType, + @Parameter(description = "The configuration step name", required = true) + @PathParam("stepName") final String stepName + ) throws InterruptedException { + + authorizeFlow(); + + if (isReplicateRequest()) { + return replicate(HttpMethod.GET); + } + + final String stepDocumentation = serviceFacade.getStepDocumentation(group, artifact, version, connectorType, stepName); + final StepDocumentationEntity entity = new StepDocumentationEntity(); + entity.setStepDocumentation(stepDocumentation); + + return generateOkResponse(entity).build(); + } + /** * Retrieves the types of parameter providers that this NiFi supports. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java index 33680eb043..72686f650c 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java @@ -145,6 +145,7 @@ import org.apache.nifi.groups.RemoteProcessGroup; import org.apache.nifi.groups.RemoteProcessGroupCounts; import org.apache.nifi.history.History; import org.apache.nifi.nar.ExtensionDefinition; +import org.apache.nifi.manifest.RuntimeManifestService; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.NarClassLoadersHolder; import org.apache.nifi.nar.NarManifest; @@ -318,6 +319,7 @@ public final class DtoFactory { private Authorizer authorizer; private ExtensionManager extensionManager; private ConnectorAssetRepository connectorAssetRepository; + private RuntimeManifestService runtimeManifestService; public ControllerConfigurationDTO createControllerConfigurationDto(final ControllerFacade controllerFacade) { final ControllerConfigurationDTO dto = new ControllerConfigurationDTO(); @@ -5218,6 +5220,10 @@ public final class DtoFactory { this.connectorAssetRepository = connectorAssetRepository; } + public void setRuntimeManifestService(RuntimeManifestService runtimeManifestService) { + this.runtimeManifestService = runtimeManifestService; + } + private ProcessingPerformanceStatusDTO createProcessingPerformanceStatusDTO(final ProcessingPerformanceStatus performanceStatus) { final ProcessingPerformanceStatusDTO performanceStatusDTO = new ProcessingPerformanceStatusDTO(); @@ -5272,16 +5278,38 @@ public final class DtoFactory { return null; } + final BundleCoordinate bundleCoordinate = connector.getBundleCoordinate(); + final String connectorType = connector.getCanonicalClassName(); + + final Set<String> stepsWithDocumentation = discoverStepsWithDocumentation(bundleCoordinate, connectorType); + final ConnectorConfiguration configuration = flowContext.getConfigurationContext().toConnectorConfiguration(); final ConnectorConfigurationDTO dto = new ConnectorConfigurationDTO(); final List<ConfigurationStepConfigurationDTO> configurationStepDtos = configurationSteps.stream() - .map(step -> createConfigurationStepConfigurationDtoFromStep(step, configuration)) + .map(step -> createConfigurationStepConfigurationDtoFromStep(step, configuration, stepsWithDocumentation)) .collect(Collectors.toList()); dto.setConfigurationStepConfigurations(configurationStepDtos); return dto; } - private ConfigurationStepConfigurationDTO createConfigurationStepConfigurationDtoFromStep(final ConfigurationStep step, final ConnectorConfiguration configuration) { + private Set<String> discoverStepsWithDocumentation(final BundleCoordinate bundleCoordinate, final String connectorType) { + if (runtimeManifestService == null) { + return Collections.emptySet(); + } + + try { + final Map<String, File> stepDocs = runtimeManifestService.discoverStepDocumentation( + bundleCoordinate.getGroup(), bundleCoordinate.getId(), bundleCoordinate.getVersion(), connectorType); + return stepDocs.keySet(); + } catch (final Exception e) { + logger.debug("Unable to discover step documentation for connector [{}]: {}", connectorType, e.getMessage()); + return Collections.emptySet(); + } + } + + private ConfigurationStepConfigurationDTO createConfigurationStepConfigurationDtoFromStep(final ConfigurationStep step, + final ConnectorConfiguration configuration, + final Set<String> stepsWithDocumentation) { if (step == null) { return null; } @@ -5289,7 +5317,7 @@ public final class DtoFactory { final ConfigurationStepConfigurationDTO dto = new ConfigurationStepConfigurationDTO(); dto.setConfigurationStepName(step.getName()); dto.setConfigurationStepDescription(step.getDescription()); - dto.setConfigurationStepDocumentation(step.getDocumentation()); + dto.setDocumented(stepsWithDocumentation.contains(step.getName())); // Get the current configuration values for this step from the flat StepConfiguration final StepConfiguration stepConfig = configuration.getNamedStepConfigurations().stream() diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/configuration/WebApplicationConfiguration.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/configuration/WebApplicationConfiguration.java index f51fb0c0c2..0659d8ef72 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/configuration/WebApplicationConfiguration.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/configuration/WebApplicationConfiguration.java @@ -179,6 +179,7 @@ public class WebApplicationConfiguration { dtoFactory.setEntityFactory(entityFactory()); dtoFactory.setExtensionManager(extensionManager); dtoFactory.setConnectorAssetRepository(flowController.getConnectorRepository().getAssetRepository()); + dtoFactory.setRuntimeManifestService(runtimeManifestService); return dtoFactory; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java index 63611f6a1f..95d6f7f9e8 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java @@ -695,6 +695,22 @@ public class ControllerFacade implements Authorizable { } } + public String getStepDocumentation(final String group, final String artifact, final String version, final String connectorType, final String stepName) { + final Map<String, File> stepDocsMap = runtimeManifestService.discoverStepDocumentation(group, artifact, version, connectorType); + final File stepDocFile = stepDocsMap.get(stepName); + + if (stepDocFile == null) { + throw new ResourceNotFoundException("Unable to find step documentation for step [%s] in connector [%s]".formatted(stepName, connectorType)); + } + + try (final Stream<String> stepDocLines = Files.lines(stepDocFile.toPath())) { + return stepDocLines.collect(Collectors.joining("\n")); + } catch (final IOException e) { + throw new RuntimeException("Unable to load step documentation content for " + + stepDocFile.getAbsolutePath() + " due to: " + e.getMessage(), e); + } + } + /** * Gets the ParameterProvider types that this controller supports. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardConnectorDAOTest.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardConnectorDAOTest.java index 420c650620..f3c480efd5 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardConnectorDAOTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardConnectorDAOTest.java @@ -18,10 +18,13 @@ package org.apache.nifi.web.dao.impl; import org.apache.nifi.components.AllowableValue; import org.apache.nifi.components.connector.ConnectorAssetRepository; +import org.apache.nifi.components.connector.ConnectorConfiguration; import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.components.connector.ConnectorRepository; import org.apache.nifi.components.connector.ConnectorUpdateContext; import org.apache.nifi.components.connector.FlowUpdateException; +import org.apache.nifi.components.connector.FrameworkFlowContext; +import org.apache.nifi.components.connector.MutableConnectorConfigurationContext; import org.apache.nifi.controller.FlowController; import org.apache.nifi.web.NiFiCoreException; import org.apache.nifi.web.ResourceNotFoundException; @@ -31,6 +34,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -61,6 +65,15 @@ class StandardConnectorDAOTest { @Mock private ConnectorAssetRepository connectorAssetRepository; + @Mock + private FrameworkFlowContext frameworkFlowContext; + + @Mock + private MutableConnectorConfigurationContext configurationContext; + + @Mock + private ConnectorConfiguration connectorConfiguration; + private static final String CONNECTOR_ID = "test-connector-id"; private static final String STEP_NAME = "test-step"; private static final String PROPERTY_NAME = "test-property"; @@ -71,12 +84,18 @@ class StandardConnectorDAOTest { connectorDAO.setFlowController(flowController); when(flowController.getConnectorRepository()).thenReturn(connectorRepository); - when(connectorRepository.getAssetRepository()).thenReturn(connectorAssetRepository); } @Test void testApplyConnectorUpdate() throws Exception { when(connectorRepository.getConnector(CONNECTOR_ID)).thenReturn(connectorNode); + when(connectorRepository.getAssetRepository()).thenReturn(connectorAssetRepository); + when(connectorNode.getActiveFlowContext()).thenReturn(frameworkFlowContext); + when(frameworkFlowContext.getConfigurationContext()).thenReturn(configurationContext); + when(configurationContext.toConnectorConfiguration()).thenReturn(connectorConfiguration); + when(connectorConfiguration.getNamedStepConfigurations()).thenReturn(Collections.emptySet()); + when(connectorNode.getIdentifier()).thenReturn(CONNECTOR_ID); + when(connectorAssetRepository.getAssets(CONNECTOR_ID)).thenReturn(Collections.emptyList()); connectorDAO.applyConnectorUpdate(CONNECTOR_ID, connectorUpdateContext); @@ -220,6 +239,8 @@ class StandardConnectorDAOTest { @Test void testDeleteConnectorRemovesConnectorAndAssets() { + when(connectorRepository.getAssetRepository()).thenReturn(connectorAssetRepository); + connectorDAO.deleteConnector(CONNECTOR_ID); verify(connectorRepository).removeConnector(CONNECTOR_ID);
