This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch NIFI-15258
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/NIFI-15258 by this push:
new 60efb80c1d NIFI-15361: Allowing configuration step documentation to be
returned … (#10667)
60efb80c1d is described below
commit 60efb80c1df76d6c1d9a100cb0342cb7a213b666
Author: Matt Gilman <[email protected]>
AuthorDate: Thu Dec 18 16:51:09 2025 -0500
NIFI-15361: Allowing configuration step documentation to be returned …
(#10667)
* NIFI-15361: Allowing configuration step documentation to be returned
independent of any connector instances.
---
.../connectors/kafkas3/KafkaConnectionStep.java | 32 -----------
.../org.apache.nifi.components.connector.Connector | 1 +
.../Kafka_Connection.md | 27 +++++++++
.../Kafka_Topics.md | 34 +++++++++++
.../S3_Configuration.md | 65 ++++++++++++++++++++++
.../api/dto/ConfigurationStepConfigurationDTO.java | 14 ++---
.../web/api/entity/StepDocumentationEntity.java | 35 ++++++++++++
.../nifi/manifest/RuntimeManifestService.java | 16 +++++-
.../manifest/StandardRuntimeManifestService.java | 45 +++++++++++++++
.../StandardRuntimeManifestServiceTest.java | 48 ++++++++++++++++
.../org.example.TestConnector/Another_Test_Step.md | 4 ++
.../org.example.TestConnector/Test_Step.md | 8 +++
.../org/apache/nifi/web/NiFiServiceFacade.java | 12 ++++
.../apache/nifi/web/StandardNiFiServiceFacade.java | 5 ++
.../java/org/apache/nifi/web/api/FlowResource.java | 45 +++++++++++++++
.../org/apache/nifi/web/api/dto/DtoFactory.java | 34 ++++++++++-
.../configuration/WebApplicationConfiguration.java | 1 +
.../nifi/web/controller/ControllerFacade.java | 16 ++++++
.../web/dao/impl/StandardConnectorDAOTest.java | 23 +++++++-
19 files changed, 420 insertions(+), 45 deletions(-)
diff --git
a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaConnectionStep.java
b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaConnectionStep.java
index 22ae253a98..793b1e9be0 100644
---
a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaConnectionStep.java
+++
b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaConnectionStep.java
@@ -29,37 +29,6 @@ import java.util.List;
public class KafkaConnectionStep {
public static final String STEP_NAME = "Kafka Connection";
- // Temporary documentation to verify the inline documentation feature in
the UI.
- // In practice, this content should probably be loaded from an external
file.
- private static final String KAFKA_CONNECTION_DOCUMENTATION = """
- # Kafka Connection Configuration
-
- This step configures the connection to your Apache Kafka cluster.
-
- ## Kafka Server Settings
-
- Enter the bootstrap servers for your Kafka cluster. You can specify
multiple brokers
- as a comma-separated list (e.g.,
`broker1:9092,broker2:9092,broker3:9092`).
-
- ### Security Configuration
-
- Select the appropriate security protocol based on your Kafka cluster
configuration:
-
- | Protocol | Description |
- |----------|-------------|
- | PLAINTEXT | No encryption or authentication |
- | SSL | TLS encryption without SASL authentication |
- | SASL_PLAINTEXT | SASL authentication without encryption |
- | SASL_SSL | Both SASL authentication and TLS encryption (recommended)
|
-
- If using SASL authentication, provide your username and password
credentials.
-
- ## Schema Registry (Optional)
-
- If your Kafka topics use Avro, Protobuf, or JSON Schema, configure the
Schema Registry
- URL to enable schema-based serialization and deserialization.
- """;
-
public static final ConnectorPropertyDescriptor KAFKA_BROKERS = new
ConnectorPropertyDescriptor.Builder()
.name("Kafka Brokers")
.description("A comma-separated list of Kafka brokers to connect to.")
@@ -158,7 +127,6 @@ public class KafkaConnectionStep {
public static final ConfigurationStep KAFKA_CONNECTION_STEP = new
ConfigurationStep.Builder()
.name(STEP_NAME)
.description("Configure Kafka connection settings")
- .documentation(KAFKA_CONNECTION_DOCUMENTATION)
.propertyGroups(List.of(
KAFKA_SERVER_GROUP,
SCHEMA_REGISTRY_GROUP
diff --git
a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector
b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector
index de5fe10a48..d5b0fe6db6 100644
---
a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector
+++
b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector
@@ -14,3 +14,4 @@
# limitations under the License.
org.apache.nifi.connectors.kafkas3.KafkaToS3
+
diff --git
a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/Kafka_Connection.md
b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/Kafka_Connection.md
new file mode 100644
index 0000000000..1621b37061
--- /dev/null
+++
b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/Kafka_Connection.md
@@ -0,0 +1,27 @@
+# Kafka Connection Configuration
+
+This step configures the connection to your Apache Kafka cluster.
+
+## Kafka Server Settings
+
+Enter the bootstrap servers for your Kafka cluster. You can specify multiple
brokers
+as a comma-separated list (e.g., `broker1:9092,broker2:9092,broker3:9092`).
+
+### Security Configuration
+
+Select the appropriate security protocol based on your Kafka cluster
configuration:
+
+| Protocol | Description |
+|----------|-------------|
+| PLAINTEXT | No encryption or authentication |
+| SSL | TLS encryption without SASL authentication |
+| SASL_PLAINTEXT | SASL authentication without encryption |
+| SASL_SSL | Both SASL authentication and TLS encryption (recommended) |
+
+If using SASL authentication, provide your username and password credentials.
+
+## Schema Registry (Optional)
+
+If your Kafka topics use Avro, Protobuf, or JSON Schema, configure the Schema
Registry
+URL to enable schema-based serialization and deserialization.
+
diff --git
a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/Kafka_Topics.md
b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/Kafka_Topics.md
new file mode 100644
index 0000000000..2e2739676c
--- /dev/null
+++
b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/Kafka_Topics.md
@@ -0,0 +1,34 @@
+# Kafka Topics Configuration
+
+This step configures which Kafka topics to consume from and how to consume
them.
+
+## Topic Names
+
+Select one or more Kafka topics to consume from. The connector will
automatically
+fetch the list of available topics from your Kafka cluster.
+
+## Consumer Group ID
+
+Specify a unique consumer group ID for this connector. Kafka uses consumer
groups
+to track message offsets and ensure each message is processed only once within
a group.
+
+**Best Practice:** Use a descriptive name that identifies the purpose of this
connector,
+such as `kafka-to-s3-production` or `analytics-pipeline-consumer`.
+
+## Offset Reset
+
+Controls the behavior when no prior offset exists or the current offset is
invalid:
+
+| Value | Description |
+|-------|-------------|
+| earliest | Start reading from the oldest available message |
+| latest | Start reading from the newest messages only |
+| none | Fail if no prior offset exists |
+
+## Kafka Data Format
+
+Specify the format of messages in your Kafka topics:
+
+- **Avro**: Messages are encoded using Apache Avro (requires Schema Registry)
+- **JSON**: Messages are plain JSON objects
+
diff --git
a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/S3_Configuration.md
b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/S3_Configuration.md
new file mode 100644
index 0000000000..fcd6092d43
--- /dev/null
+++
b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/src/main/resources/META-INF/docs/step-documentation/org.apache.nifi.connectors.kafkas3.KafkaToS3/S3_Configuration.md
@@ -0,0 +1,65 @@
+# S3 Configuration
+
+This step configures the connection to Amazon S3 or an S3-compatible storage
system.
+
+## S3 Destination Configuration
+
+### S3 Bucket
+
+The name of the S3 bucket where data will be written. Ensure the bucket exists
+and your credentials have write permissions.
+
+### S3 Prefix
+
+An optional prefix (folder path) to prepend to all object keys. For example,
+setting this to `kafka-data/` will result in objects like:
+`kafka-data/2024/01/15/data-001.json`
+
+### S3 Region
+
+The AWS region where your S3 bucket is located (e.g., `us-east-1`,
`eu-west-1`).
+
+### S3 Data Format
+
+The format to use when writing objects to S3:
+
+- **JSON**: Write data as JSON objects (human-readable)
+- **Avro**: Write data in Apache Avro format (compact, schema-embedded)
+
+### S3 Endpoint Override URL
+
+Leave this blank for standard AWS S3. Use this field only when connecting to
+S3-compatible storage systems like MinIO, LocalStack, or Ceph.
+
+## Merge Configuration
+
+### Target Object Size
+
+The connector merges multiple Kafka messages together before writing to S3 to
+reduce the number of objects created. Specify the target size for merged
objects
+(e.g., `256 MB`, `1 GB`).
+
+### Merge Latency
+
+The maximum time to wait while collecting messages before writing to S3. Even
if
+the target size hasn't been reached, data will be written after this duration
to
+ensure timely data availability.
+
+## S3 Credentials
+
+### Authentication Strategy
+
+Choose how to authenticate with AWS:
+
+| Strategy | Description |
+|----------|-------------|
+| Access Key ID and Secret Key | Use explicit AWS access credentials |
+| Default AWS Credentials | Use the default credential chain (environment
variables, IAM roles, etc.) |
+
+### S3 Access Key ID / Secret Access Key
+
+When using explicit credentials, provide your AWS access key ID and secret
access key.
+
+**Security Note:** Store sensitive credentials using NiFi's sensitive property
+protection or parameter contexts with secret providers.
+
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ConfigurationStepConfigurationDTO.java
b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ConfigurationStepConfigurationDTO.java
index 94b0c7ff71..4b58f9df2c 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ConfigurationStepConfigurationDTO.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ConfigurationStepConfigurationDTO.java
@@ -29,7 +29,7 @@ public class ConfigurationStepConfigurationDTO {
private String configurationStepName;
private String configurationStepDescription;
- private String configurationStepDocumentation;
+ private boolean documented;
private List<PropertyGroupConfigurationDTO> propertyGroupConfigurations;
/**
@@ -57,15 +57,15 @@ public class ConfigurationStepConfigurationDTO {
}
/**
- * @return the configuration step documentation in markdown
+ * @return whether this step has extended documentation available
*/
- @Schema(description = "Extended documentation or help text for the
configuration step.")
- public String getConfigurationStepDocumentation() {
- return configurationStepDocumentation;
+ @Schema(description = "Whether extended documentation is available for
this configuration step.")
+ public boolean isDocumented() {
+ return documented;
}
- public void setConfigurationStepDocumentation(final String
configurationStepDocumentation) {
- this.configurationStepDocumentation = configurationStepDocumentation;
+ public void setDocumented(final boolean documented) {
+ this.documented = documented;
}
/**
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/StepDocumentationEntity.java
b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/StepDocumentationEntity.java
new file mode 100644
index 0000000000..e01da34a1d
--- /dev/null
+++
b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/StepDocumentationEntity.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.entity;
+
+import jakarta.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement(name = "stepDocumentationEntity")
+public class StepDocumentationEntity extends Entity {
+
+ private String stepDocumentation;
+
+ public String getStepDocumentation() {
+ return stepDocumentation;
+ }
+
+ public void setStepDocumentation(final String stepDocumentation) {
+ this.stepDocumentation = stepDocumentation;
+ }
+
+}
+
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/manifest/RuntimeManifestService.java
b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/manifest/RuntimeManifestService.java
index d0963e44fc..afd2c3c40f 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/manifest/RuntimeManifestService.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/manifest/RuntimeManifestService.java
@@ -43,12 +43,24 @@ public interface RuntimeManifestService {
RuntimeManifest getManifestForBundle(String group, String artifact, String
version);
/**
- * Returns a mapping of additionalDetails for the speicfied bundle.
+ * Returns a mapping of additionalDetails for the specified bundle.
*
* @param group The bundle group
* @param artifact The bundle artifact
* @param version The bundle version
- * @return The additionaDetails mapping
+ * @return The additionalDetails mapping
*/
Map<String, File> discoverAdditionalDetails(String group, String artifact,
String version);
+
+ /**
+ * Returns a mapping of step documentation files for the specified
Connector type.
+ * The keys are the step names and the values are the corresponding
documentation files.
+ *
+ * @param group The bundle group
+ * @param artifact The bundle artifact
+ * @param version The bundle version
+ * @param connectorType The fully qualified class name of the Connector
+ * @return The step documentation mapping
+ */
+ Map<String, File> discoverStepDocumentation(String group, String artifact,
String version, String connectorType);
}
\ No newline at end of file
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java
index 1a7f3ef214..197951ccb0 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java
@@ -401,6 +401,51 @@ public class StandardRuntimeManifestService implements
RuntimeManifestService {
return additionalDetailsMap;
}
+ @Override
+ public Map<String, File> discoverStepDocumentation(final String group,
final String artifact, final String version, final String connectorType) {
+ final BundleCoordinate bundleCoordinate = new BundleCoordinate(group,
artifact, version);
+ final Bundle bundle = extensionManager.getBundle(bundleCoordinate);
+
+ if (bundle == null) {
+ throw new ResourceNotFoundException("Unable to find bundle [" +
bundleCoordinate + "]");
+ }
+
+ return discoverStepDocumentation(bundle.getBundleDetails(),
connectorType);
+ }
+
+ private Map<String, File> discoverStepDocumentation(final BundleDetails
bundleDetails, final String connectorType) {
+ final Map<String, File> stepDocsMap = new LinkedHashMap<>();
+
+ final File stepDocsDir = new File(bundleDetails.getWorkingDirectory(),
"META-INF/docs/step-documentation/" + connectorType);
+ if (!stepDocsDir.exists()) {
+ LOGGER.debug("No step-documentation directory found for [{}] under
[{}]", connectorType, bundleDetails.getWorkingDirectory().getAbsolutePath());
+ return stepDocsMap;
+ }
+
+ final File[] stepDocFiles = stepDocsDir.listFiles();
+ if (stepDocFiles == null) {
+ return stepDocsMap;
+ }
+
+ for (final File stepDocFile : stepDocFiles) {
+ if (!stepDocFile.isFile() ||
!stepDocFile.getName().endsWith(".md")) {
+ LOGGER.debug("Skipping [{}], not a markdown file...",
stepDocFile.getAbsolutePath());
+ continue;
+ }
+
+ final String fileName = stepDocFile.getName().substring(0,
stepDocFile.getName().length() - 3);
+ final String stepName = fileNameToStepName(fileName);
+ stepDocsMap.put(stepName, stepDocFile);
+ LOGGER.debug("Discovered step documentation for step [{}] at
[{}]", stepName, stepDocFile.getAbsolutePath());
+ }
+
+ return stepDocsMap;
+ }
+
+ private String fileNameToStepName(final String fileName) {
+ return fileName.replace("_", " ");
+ }
+
private Map<String, String> loadAdditionalDetails(final BundleDetails
bundleDetails) {
final Map<String, File> additionalDetailsMap =
discoverAdditionalDetails(bundleDetails);
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/manifest/StandardRuntimeManifestServiceTest.java
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/manifest/StandardRuntimeManifestServiceTest.java
index fae3db4f85..fe9b3d93ef 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/manifest/StandardRuntimeManifestServiceTest.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/manifest/StandardRuntimeManifestServiceTest.java
@@ -31,6 +31,7 @@ import org.apache.nifi.nar.ExtensionManager;
import org.apache.nifi.nar.PythonBundle;
import org.apache.nifi.processor.Processor;
import org.apache.nifi.python.PythonProcessorDetails;
+import org.apache.nifi.web.ResourceNotFoundException;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -38,11 +39,14 @@ import java.io.File;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import static java.util.Collections.emptySet;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -163,6 +167,50 @@ public class StandardRuntimeManifestServiceTest {
assertEquals(0, controllerServiceDefinitions.size());
}
+ @Test
+ public void testDiscoverStepDocumentation() {
+
when(extensionManager.getBundle(testComponentsBundle.getBundleDetails().getCoordinate())).thenReturn(testComponentsBundle);
+
+ final BundleCoordinate coordinate =
testComponentsBundle.getBundleDetails().getCoordinate();
+ final Map<String, File> stepDocs =
runtimeManifestService.discoverStepDocumentation(
+ coordinate.getGroup(), coordinate.getId(),
coordinate.getVersion(), "org.example.TestConnector");
+
+ assertNotNull(stepDocs);
+ assertEquals(2, stepDocs.size());
+ assertTrue(stepDocs.containsKey("Test Step"));
+ assertTrue(stepDocs.containsKey("Another Test Step"));
+
+ final File testStepFile = stepDocs.get("Test Step");
+ assertNotNull(testStepFile);
+ assertTrue(testStepFile.exists());
+ assertEquals("Test_Step.md", testStepFile.getName());
+
+ final File anotherTestStepFile = stepDocs.get("Another Test Step");
+ assertNotNull(anotherTestStepFile);
+ assertTrue(anotherTestStepFile.exists());
+ assertEquals("Another_Test_Step.md", anotherTestStepFile.getName());
+ }
+
+ @Test
+ public void testDiscoverStepDocumentationWithNonExistentConnector() {
+
when(extensionManager.getBundle(testComponentsBundle.getBundleDetails().getCoordinate())).thenReturn(testComponentsBundle);
+
+ final BundleCoordinate coordinate =
testComponentsBundle.getBundleDetails().getCoordinate();
+ final Map<String, File> stepDocs =
runtimeManifestService.discoverStepDocumentation(
+ coordinate.getGroup(), coordinate.getId(),
coordinate.getVersion(), "org.example.NonExistentConnector");
+
+ assertNotNull(stepDocs);
+ assertTrue(stepDocs.isEmpty());
+ }
+
+ @Test
+ public void testDiscoverStepDocumentationWithNonExistentBundle() {
+ when(extensionManager.getBundle(new BundleCoordinate("org.example",
"nonexistent", "1.0.0"))).thenReturn(null);
+
+ assertThrows(ResourceNotFoundException.class, () ->
+
runtimeManifestService.discoverStepDocumentation("org.example", "nonexistent",
"1.0.0", "org.example.TestConnector"));
+ }
+
/**
* Override getFrameworkBundle to provide a mocked Bundle.
*/
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/TestRuntimeManifest/nifi-test-components-nar/META-INF/docs/step-documentation/org.example.TestConnector/Another_Test_Step.md
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/TestRuntimeManifest/nifi-test-components-nar/META-INF/docs/step-documentation/org.example.TestConnector/Another_Test_Step.md
new file mode 100644
index 0000000000..4673395a0c
--- /dev/null
+++
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/TestRuntimeManifest/nifi-test-components-nar/META-INF/docs/step-documentation/org.example.TestConnector/Another_Test_Step.md
@@ -0,0 +1,4 @@
+# Another Test Step
+
+Documentation for another step.
+
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/TestRuntimeManifest/nifi-test-components-nar/META-INF/docs/step-documentation/org.example.TestConnector/Test_Step.md
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/TestRuntimeManifest/nifi-test-components-nar/META-INF/docs/step-documentation/org.example.TestConnector/Test_Step.md
new file mode 100644
index 0000000000..22a6c822b8
--- /dev/null
+++
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/TestRuntimeManifest/nifi-test-components-nar/META-INF/docs/step-documentation/org.example.TestConnector/Test_Step.md
@@ -0,0 +1,8 @@
+# Test Step Documentation
+
+This is test documentation for the Test Step.
+
+## Configuration
+
+Configure the test step properties as needed.
+
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
index b10cf18a45..14472b78bd 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
@@ -654,6 +654,18 @@ public interface NiFiServiceFacade {
*/
String getAdditionalDetails(String group, String artifact, String version,
String type);
+ /**
+ * Return the step documentation for the specified Connector configuration
step.
+ *
+ * @param group The bundle group
+ * @param artifact The bundle artifact
+ * @param version The bundle version
+ * @param connectorType The fully qualified class name of the Connector
+ * @param stepName The name of the configuration step
+ * @return The step documentation markdown content
+ */
+ String getStepDocumentation(String group, String artifact, String version,
String connectorType, String stepName);
+
/**
* Returns the list of parameter provider types.
*
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
index 9155e6f55b..a472a18178 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
@@ -4373,6 +4373,11 @@ public class StandardNiFiServiceFacade implements
NiFiServiceFacade {
return controllerFacade.getAdditionalDetails(group, artifact, version,
type);
}
+ @Override
+ public String getStepDocumentation(final String group, final String
artifact, final String version, final String connectorType, final String
stepName) {
+ return controllerFacade.getStepDocumentation(group, artifact, version,
connectorType, stepName);
+ }
+
@Override
public Set<DocumentedTypeDTO> getParameterProviderTypes(final String
bundleGroup, final String bundleArtifact, final String type) {
return controllerFacade.getParameterProviderTypes(bundleGroup,
bundleArtifact, type);
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowResource.java
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowResource.java
index 1b73a9bf57..8a4d2f8fb0 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowResource.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowResource.java
@@ -106,6 +106,7 @@ import org.apache.nifi.web.api.entity.AboutEntity;
import org.apache.nifi.web.api.entity.ActionEntity;
import org.apache.nifi.web.api.entity.ActivateControllerServicesEntity;
import org.apache.nifi.web.api.entity.AdditionalDetailsEntity;
+import org.apache.nifi.web.api.entity.StepDocumentationEntity;
import org.apache.nifi.web.api.entity.BannerEntity;
import org.apache.nifi.web.api.entity.BulletinBoardEntity;
import org.apache.nifi.web.api.entity.ClearBulletinsForGroupRequestEntity;
@@ -2330,6 +2331,50 @@ public class FlowResource extends ApplicationResource {
return generateOkResponse(entity).build();
}
+ @GET
+ @Consumes(MediaType.WILDCARD)
+ @Produces(MediaType.APPLICATION_JSON)
+
@Path("step-documentation/{group}/{artifact}/{version}/{connectorType}/{stepName}")
+ @Operation(
+ summary = "Retrieves the step documentation for the specified
Connector configuration step.",
+ description = NON_GUARANTEED_ENDPOINT,
+ responses = {
+ @ApiResponse(responseCode = "200", content =
@Content(schema = @Schema(implementation = StepDocumentationEntity.class))),
+ @ApiResponse(responseCode = "400", description = "NiFi was
unable to complete the request because it was invalid. The request should not
be retried without modification."),
+ @ApiResponse(responseCode = "401", description = "Client
could not be authenticated."),
+ @ApiResponse(responseCode = "403", description = "Client
is not authorized to make this request."),
+ @ApiResponse(responseCode = "404", description = "The step
documentation for the coordinates could not be located.")
+ },
+ security = {
+ @SecurityRequirement(name = "Read - /flow")
+ }
+ )
+ public Response getStepDocumentation(
+ @Parameter(description = "The bundle group", required = true)
+ @PathParam("group") final String group,
+ @Parameter(description = "The bundle artifact", required = true)
+ @PathParam("artifact") final String artifact,
+ @Parameter(description = "The bundle version", required = true)
+ @PathParam("version") final String version,
+ @Parameter(description = "The fully qualified Connector type",
required = true)
+ @PathParam("connectorType") final String connectorType,
+ @Parameter(description = "The configuration step name", required =
true)
+ @PathParam("stepName") final String stepName
+ ) throws InterruptedException {
+
+ authorizeFlow();
+
+ if (isReplicateRequest()) {
+ return replicate(HttpMethod.GET);
+ }
+
+ final String stepDocumentation =
serviceFacade.getStepDocumentation(group, artifact, version, connectorType,
stepName);
+ final StepDocumentationEntity entity = new StepDocumentationEntity();
+ entity.setStepDocumentation(stepDocumentation);
+
+ return generateOkResponse(entity).build();
+ }
+
/**
* Retrieves the types of parameter providers that this NiFi supports.
*
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
index 8af59df8ff..fdfce9b68a 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
@@ -145,6 +145,7 @@ import org.apache.nifi.groups.RemoteProcessGroup;
import org.apache.nifi.groups.RemoteProcessGroupCounts;
import org.apache.nifi.history.History;
import org.apache.nifi.nar.ExtensionDefinition;
+import org.apache.nifi.manifest.RuntimeManifestService;
import org.apache.nifi.nar.ExtensionManager;
import org.apache.nifi.nar.NarClassLoadersHolder;
import org.apache.nifi.nar.NarManifest;
@@ -318,6 +319,7 @@ public final class DtoFactory {
private Authorizer authorizer;
private ExtensionManager extensionManager;
private ConnectorAssetRepository connectorAssetRepository;
+ private RuntimeManifestService runtimeManifestService;
public ControllerConfigurationDTO createControllerConfigurationDto(final
ControllerFacade controllerFacade) {
final ControllerConfigurationDTO dto = new ControllerConfigurationDTO();
@@ -5218,6 +5220,10 @@ public final class DtoFactory {
this.connectorAssetRepository = connectorAssetRepository;
}
+ public void setRuntimeManifestService(RuntimeManifestService
runtimeManifestService) {
+ this.runtimeManifestService = runtimeManifestService;
+ }
+
private ProcessingPerformanceStatusDTO
createProcessingPerformanceStatusDTO(final ProcessingPerformanceStatus
performanceStatus) {
final ProcessingPerformanceStatusDTO performanceStatusDTO = new
ProcessingPerformanceStatusDTO();
@@ -5272,16 +5278,38 @@ public final class DtoFactory {
return null;
}
+ final BundleCoordinate bundleCoordinate =
connector.getBundleCoordinate();
+ final String connectorType = connector.getCanonicalClassName();
+
+ final Set<String> stepsWithDocumentation =
discoverStepsWithDocumentation(bundleCoordinate, connectorType);
+
final ConnectorConfiguration configuration =
flowContext.getConfigurationContext().toConnectorConfiguration();
final ConnectorConfigurationDTO dto = new ConnectorConfigurationDTO();
final List<ConfigurationStepConfigurationDTO> configurationStepDtos =
configurationSteps.stream()
- .map(step ->
createConfigurationStepConfigurationDtoFromStep(step, configuration))
+ .map(step ->
createConfigurationStepConfigurationDtoFromStep(step, configuration,
stepsWithDocumentation))
.collect(Collectors.toList());
dto.setConfigurationStepConfigurations(configurationStepDtos);
return dto;
}
- private ConfigurationStepConfigurationDTO
createConfigurationStepConfigurationDtoFromStep(final ConfigurationStep step,
final ConnectorConfiguration configuration) {
+ private Set<String> discoverStepsWithDocumentation(final BundleCoordinate
bundleCoordinate, final String connectorType) {
+ if (runtimeManifestService == null) {
+ return Collections.emptySet();
+ }
+
+ try {
+ final Map<String, File> stepDocs =
runtimeManifestService.discoverStepDocumentation(
+ bundleCoordinate.getGroup(), bundleCoordinate.getId(),
bundleCoordinate.getVersion(), connectorType);
+ return stepDocs.keySet();
+ } catch (final Exception e) {
+ logger.debug("Unable to discover step documentation for connector
[{}]: {}", connectorType, e.getMessage());
+ return Collections.emptySet();
+ }
+ }
+
+ private ConfigurationStepConfigurationDTO
createConfigurationStepConfigurationDtoFromStep(final ConfigurationStep step,
+
final ConnectorConfiguration configuration,
+
final Set<String> stepsWithDocumentation) {
if (step == null) {
return null;
}
@@ -5289,7 +5317,7 @@ public final class DtoFactory {
final ConfigurationStepConfigurationDTO dto = new
ConfigurationStepConfigurationDTO();
dto.setConfigurationStepName(step.getName());
dto.setConfigurationStepDescription(step.getDescription());
- dto.setConfigurationStepDocumentation(step.getDocumentation());
+ dto.setDocumented(stepsWithDocumentation.contains(step.getName()));
// Get the current configuration values for this step from the flat
StepConfiguration
final StepConfiguration stepConfig =
configuration.getNamedStepConfigurations().stream()
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/configuration/WebApplicationConfiguration.java
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/configuration/WebApplicationConfiguration.java
index f51fb0c0c2..0659d8ef72 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/configuration/WebApplicationConfiguration.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/configuration/WebApplicationConfiguration.java
@@ -179,6 +179,7 @@ public class WebApplicationConfiguration {
dtoFactory.setEntityFactory(entityFactory());
dtoFactory.setExtensionManager(extensionManager);
dtoFactory.setConnectorAssetRepository(flowController.getConnectorRepository().getAssetRepository());
+ dtoFactory.setRuntimeManifestService(runtimeManifestService);
return dtoFactory;
}
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java
index 63611f6a1f..95d6f7f9e8 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java
@@ -695,6 +695,22 @@ public class ControllerFacade implements Authorizable {
}
}
+ public String getStepDocumentation(final String group, final String
artifact, final String version, final String connectorType, final String
stepName) {
+ final Map<String, File> stepDocsMap =
runtimeManifestService.discoverStepDocumentation(group, artifact, version,
connectorType);
+ final File stepDocFile = stepDocsMap.get(stepName);
+
+ if (stepDocFile == null) {
+ throw new ResourceNotFoundException("Unable to find step
documentation for step [%s] in connector [%s]".formatted(stepName,
connectorType));
+ }
+
+ try (final Stream<String> stepDocLines =
Files.lines(stepDocFile.toPath())) {
+ return stepDocLines.collect(Collectors.joining("\n"));
+ } catch (final IOException e) {
+ throw new RuntimeException("Unable to load step documentation
content for "
+ + stepDocFile.getAbsolutePath() + " due to: " +
e.getMessage(), e);
+ }
+ }
+
/**
* Gets the ParameterProvider types that this controller supports.
*
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardConnectorDAOTest.java
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardConnectorDAOTest.java
index 420c650620..f3c480efd5 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardConnectorDAOTest.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardConnectorDAOTest.java
@@ -18,10 +18,13 @@ package org.apache.nifi.web.dao.impl;
import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.connector.ConnectorAssetRepository;
+import org.apache.nifi.components.connector.ConnectorConfiguration;
import org.apache.nifi.components.connector.ConnectorNode;
import org.apache.nifi.components.connector.ConnectorRepository;
import org.apache.nifi.components.connector.ConnectorUpdateContext;
import org.apache.nifi.components.connector.FlowUpdateException;
+import org.apache.nifi.components.connector.FrameworkFlowContext;
+import
org.apache.nifi.components.connector.MutableConnectorConfigurationContext;
import org.apache.nifi.controller.FlowController;
import org.apache.nifi.web.NiFiCoreException;
import org.apache.nifi.web.ResourceNotFoundException;
@@ -31,6 +34,7 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
+import java.util.Collections;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -61,6 +65,15 @@ class StandardConnectorDAOTest {
@Mock
private ConnectorAssetRepository connectorAssetRepository;
+ @Mock
+ private FrameworkFlowContext frameworkFlowContext;
+
+ @Mock
+ private MutableConnectorConfigurationContext configurationContext;
+
+ @Mock
+ private ConnectorConfiguration connectorConfiguration;
+
private static final String CONNECTOR_ID = "test-connector-id";
private static final String STEP_NAME = "test-step";
private static final String PROPERTY_NAME = "test-property";
@@ -71,12 +84,18 @@ class StandardConnectorDAOTest {
connectorDAO.setFlowController(flowController);
when(flowController.getConnectorRepository()).thenReturn(connectorRepository);
-
when(connectorRepository.getAssetRepository()).thenReturn(connectorAssetRepository);
}
@Test
void testApplyConnectorUpdate() throws Exception {
when(connectorRepository.getConnector(CONNECTOR_ID)).thenReturn(connectorNode);
+
when(connectorRepository.getAssetRepository()).thenReturn(connectorAssetRepository);
+
when(connectorNode.getActiveFlowContext()).thenReturn(frameworkFlowContext);
+
when(frameworkFlowContext.getConfigurationContext()).thenReturn(configurationContext);
+
when(configurationContext.toConnectorConfiguration()).thenReturn(connectorConfiguration);
+
when(connectorConfiguration.getNamedStepConfigurations()).thenReturn(Collections.emptySet());
+ when(connectorNode.getIdentifier()).thenReturn(CONNECTOR_ID);
+
when(connectorAssetRepository.getAssets(CONNECTOR_ID)).thenReturn(Collections.emptyList());
connectorDAO.applyConnectorUpdate(CONNECTOR_ID,
connectorUpdateContext);
@@ -220,6 +239,8 @@ class StandardConnectorDAOTest {
@Test
void testDeleteConnectorRemovesConnectorAndAssets() {
+
when(connectorRepository.getAssetRepository()).thenReturn(connectorAssetRepository);
+
connectorDAO.deleteConnector(CONNECTOR_ID);
verify(connectorRepository).removeConnector(CONNECTOR_ID);
