This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new e088bca1d8 issue #6295 (#6316)
e088bca1d8 is described below
commit e088bca1d81df52d84c085083a103386f71ed9b5
Author: solarflare <[email protected]>
AuthorDate: Fri Jan 9 15:06:09 2026 +0100
issue #6295 (#6316)
* issue #6295
* spotless fix
---------
Co-authored-by: Hans Van Akelyen <[email protected]>
---
.../modules/ROOT/pages/hop-tools/hop-doc.adoc | 158 +++++++++++++++++++++
.../modules/ROOT/pages/hop-tools/index.adoc | 1 +
.../ROOT/pages/workflow/actions/documentation.adoc | 5 +-
plugins/misc/documentation/pom.xml | 24 ++++
.../misc/documentation/src/assembly/assembly.xml | 12 ++
.../org/apache/hop/documentation/ActionDoc.java | 31 +++-
.../org/apache/hop/documentation/DocBuilder.java | 97 ++++++++++++-
.../messages/messages_en_US.properties | 2 +
8 files changed, 323 insertions(+), 7 deletions(-)
diff --git a/docs/hop-user-manual/modules/ROOT/pages/hop-tools/hop-doc.adoc
b/docs/hop-user-manual/modules/ROOT/pages/hop-tools/hop-doc.adoc
new file mode 100644
index 0000000000..d8c2f784b1
--- /dev/null
+++ b/docs/hop-user-manual/modules/ROOT/pages/hop-tools/hop-doc.adoc
@@ -0,0 +1,158 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing
+permissions and limitations under the License.
+////
+:description: Hop Doc generates project documentation
+= Hop Doc
+
+Hop Doc generates documentation for Hop projects in Markdown format with
optional HTML output.
+
+== Usage
+
+Generate documentation for a Hop project:
+
+[source,command]
+----
+./hop.bat doc -s /path/to/project -t /path/to/docs [--generate-html]
[--remove-markdown]
+----
+
+Or using Hop project variables:
+[source,command]
+----
+./hop.bat run -j my-project doc -t /path/to/docs --generate-html
+----
+
+== Options
+
+[options="header"]
+|===
+|Option |Description |Default
+|-s, --source-folder |Source project folder to document |Required (or use
project)
+|-t, --target-folder |Target documentation folder |Required
+|-ip, --include-parameters |Include pipeline/workflow parameters |true
+|-in, --include-notes |Include notes text |false
+|-im, --include-metadata |Include metadata overview |true
+|-n, --project-name |Project name |"Hop" (or HOP_PROJECT_NAME)
+|-gh, --generate-html |Generate HTML versions of all docs |false
+|-rmmd, --remove-markdown |Remove .md files after HTML generation |false
+|===
+
+== Examples
+
+=== Basic Documentation
+
+Generate Markdown documentation only:
+
+[tabs]
+====
+Windows::
++
+--
+[source,shell]
+----
+./hop.bat doc -s "C:\Projects\my-hop-project" -t "C:\yourLocation\docs"
+----
+
+Linux/macOS::
++
+--
+[source,shell]
+----
+./hop.sh doc -s ~/projects/my-hop-project -t ~/docs
+----
+====
+
+=== HTML Documentation
+
+Generate HTML + remove Markdown source:
+
+[tabs]
+====
+Windows::
++
+--
+[source,shell]
+----
+./hop.bat doc -s "C:\Projects\my-hop-project" -t "C:\yourLocation\docs"
--generate-html --remove-markdown
+----
+
+Generates:
+[source]
+----
+C:\docs\
+├── index.html (project TOC)
+├── pipelines\
+│ └── my-pipeline.html
+├── workflows\
+│ └── my-workflow.html
+└── assets\
+ ├── styles.css
+ └── images\
+----
+
+Linux/macOS::
++
+--
+[source,shell]
+----
+./hop.sh doc -s ~/projects/my-hop-project -t ~/docs --generate-html
--remove-markdown
+----
+====
+
+=== Using Hop Project Context
+
+[source,shell]
+----
+./hop.bat run -j IT-mongo doc -t "C:\yourLocation\docs" --generate-html
+----
+
+Uses `HOP_PROJECT_HOME` and `HOP_PROJECT_NAME` automatically.
+
+== Workflow Action Usage
+
+Add "Documentation" action to any Hop workflow:
+
+1. Drag **Documentation** action from Utility category
+2. Set **Target folder** (`C:\yourLocation\docs`)
+3. Optional: Check **Generate HTML**
+4. Optional: Check **Remove markdown** (only when Generate HTML enabled)
+5. Configure content options:
+ - ☑ **Include parameters** (optional)
+ - ☑ **Include metadata** (optional)
+ - ☐ **Include notes** (optional)
+
+== Generated Structure
+
+docs/
+├── index.html (Table of Contents)
+├── pipelines/ (all .hpl files)
+│ ├── pipeline1.html
+│ └── subfolder/
+│ └── pipeline2.html
+├── workflows/ (all .hwf files)
+│ └── workflow1.html
+├── metadata/ (metadata overview)
+└── assets/
+├── styles.css
+└── images/
+
+== Notes
+
+* `--remove-markdown` is **automatically disabled** if `--generate-html` is
false
+* HTML output includes custom CSS styling and responsive images
+* All links in `index.html` automatically point to `.html` files
+* Supports nested folders and metadata documentation
+* Uses CommonMark for Markdown → HTML conversion
\ No newline at end of file
diff --git a/docs/hop-user-manual/modules/ROOT/pages/hop-tools/index.adoc
b/docs/hop-user-manual/modules/ROOT/pages/hop-tools/index.adoc
index 9c308af0a6..3bae65388c 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/hop-tools/index.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/hop-tools/index.adoc
@@ -28,4 +28,5 @@ This page provides an overview of the tools that are
available in the platform.
* xref:hop-run/index.adoc[Hop Run]
* xref:hop-tools/hop-search.adoc[Hop Search]
* xref:hop-tools/hop-import.adoc[Hop Import]
+* xref:hop-tools/hop-doc.adoc[Hop Documentation]
* xref:hop-server/index.adoc[Hop Server]
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/workflow/actions/documentation.adoc
b/docs/hop-user-manual/modules/ROOT/pages/workflow/actions/documentation.adoc
index cded00b783..3cc1f9344c 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/workflow/actions/documentation.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/workflow/actions/documentation.adoc
@@ -29,8 +29,11 @@ This action builds documentation of your Hop files or
project.
* The target folder in which you want your project documentation to end up in.
* Include notes?
* Include parameters?
+* Include metadata?
+* Generate HTML?
+* Remove MarkDown after HTML is generated?
-== Generating HTML
+== Generating HTML from MarkDown
The links from the index are currently set to the .md.html files.
I ran the following command in the target folder to generate the HTML files:
diff --git a/plugins/misc/documentation/pom.xml
b/plugins/misc/documentation/pom.xml
index 33f4adeb03..de3b796a72 100644
--- a/plugins/misc/documentation/pom.xml
+++ b/plugins/misc/documentation/pom.xml
@@ -29,4 +29,28 @@
<packaging>jar</packaging>
<name>Hop Plugins Miscellaneous Documentation</name>
+ <properties>
+ <commonmark.version>0.27.0</commonmark.version>
+ </properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-libs</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.commonmark</groupId>
+ <artifactId>commonmark</artifactId>
+ <version>${commonmark.version}</version>
+ </dependency>
+ </dependencies>
+
</project>
diff --git a/plugins/misc/documentation/src/assembly/assembly.xml
b/plugins/misc/documentation/src/assembly/assembly.xml
index 858f25610d..86202c5d7e 100644
--- a/plugins/misc/documentation/src/assembly/assembly.xml
+++ b/plugins/misc/documentation/src/assembly/assembly.xml
@@ -47,5 +47,17 @@
</includes>
<outputDirectory>plugins/misc/documentation</outputDirectory>
</dependencySet>
+ <dependencySet>
+ <scope>runtime</scope>
+ <excludes>
+ <exclude>commons-codec:*</exclude>
+ <exclude>commons-logging:*</exclude>
+ <exclude>org.slf4j:*</exclude>
+ <exclude>org.apache.httpcomponents:*</exclude>
+ </excludes>
+ <outputDirectory>plugins/misc/documentation/lib</outputDirectory>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <useProjectArtifact>false</useProjectArtifact>
+ </dependencySet>
</dependencySets>
</assembly>
\ No newline at end of file
diff --git
a/plugins/misc/documentation/src/main/java/org/apache/hop/documentation/ActionDoc.java
b/plugins/misc/documentation/src/main/java/org/apache/hop/documentation/ActionDoc.java
index 909c0e6dc3..5a1c155372 100644
---
a/plugins/misc/documentation/src/main/java/org/apache/hop/documentation/ActionDoc.java
+++
b/plugins/misc/documentation/src/main/java/org/apache/hop/documentation/ActionDoc.java
@@ -83,6 +83,24 @@ public class ActionDoc extends ActionBase implements
Cloneable, IAction {
@HopMetadataProperty
private boolean includingMetadata = true;
+ @GuiWidgetElement(
+ id = "1040-action-doc-dialog-generate-Html",
+ parentId = GUI_WIDGETS_PARENT_ID,
+ type = GuiElementType.CHECKBOX,
+ variables = false,
+ label = "i18n::ActionDoc.generateHtml.Label")
+ @HopMetadataProperty
+ private boolean generateHtml = true;
+
+ @GuiWidgetElement(
+ id = "1050-action-doc-dialog-remove-markdown",
+ parentId = GUI_WIDGETS_PARENT_ID,
+ type = GuiElementType.CHECKBOX,
+ variables = false,
+ label = "i18n:ActionDoc.RemoveMarkdown.Label")
+ @HopMetadataProperty
+ private boolean removeMarkdown = false;
+
public ActionDoc() {}
public ActionDoc(ActionDoc other) {
@@ -90,6 +108,9 @@ public class ActionDoc extends ActionBase implements
Cloneable, IAction {
this.targetParentFolder = other.targetParentFolder;
this.includingNotes = other.includingNotes;
this.includingParameters = other.includingParameters;
+ this.includingMetadata = other.includingMetadata;
+ this.generateHtml = other.generateHtml;
+ this.removeMarkdown = other.removeMarkdown;
}
@Override
@@ -107,6 +128,12 @@ public class ActionDoc extends ActionBase implements
Cloneable, IAction {
@Override
public Result execute(Result prevResult, int nr) {
+ // SAFETY CHECK: Disable removeMarkdown if generateHtml=false
+ if (removeMarkdown && !generateHtml) {
+ getLogChannel().logBasic("Remove Markdown disabled - Generate HTML not
selected");
+ removeMarkdown = false;
+ }
+
MultiMetadataProvider provider;
if (getMetadataProvider() instanceof MultiMetadataProvider) {
provider = (MultiMetadataProvider) getMetadataProvider();
@@ -124,7 +151,9 @@ public class ActionDoc extends ActionBase implements
Cloneable, IAction {
resolve(targetParentFolder),
includingParameters,
includingNotes,
- includingMetadata);
+ includingMetadata,
+ generateHtml,
+ removeMarkdown);
docBuilder.buildDocumentation(prevResult);
return prevResult;
}
diff --git
a/plugins/misc/documentation/src/main/java/org/apache/hop/documentation/DocBuilder.java
b/plugins/misc/documentation/src/main/java/org/apache/hop/documentation/DocBuilder.java
index 29719623d2..698a8acf9f 100644
---
a/plugins/misc/documentation/src/main/java/org/apache/hop/documentation/DocBuilder.java
+++
b/plugins/misc/documentation/src/main/java/org/apache/hop/documentation/DocBuilder.java
@@ -18,7 +18,9 @@
package org.apache.hop.documentation;
+import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@@ -43,6 +45,9 @@ import org.apache.hop.hop.plugin.HopCommand;
import org.apache.hop.hop.plugin.IHopCommand;
import org.apache.hop.metadata.api.IHasHopMetadataProvider;
import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
+import org.commonmark.node.Node;
+import org.commonmark.parser.Parser;
+import org.commonmark.renderer.html.HtmlRenderer;
import picocli.CommandLine;
@Getter
@@ -100,6 +105,16 @@ public class DocBuilder implements Runnable, IHopCommand,
IHasHopMetadataProvide
description = "The source folder to document")
private String sourceFolder;
+ @CommandLine.Option(
+ names = {"-gh", "--generate-html"},
+ description = "Generate HTML versions of documentation files")
+ private boolean generateHtml;
+
+ @CommandLine.Option(
+ names = {"-rmmd", "--remove-markdown"},
+ description = "Removes markdown after generating html files")
+ private boolean removeMarkdown;
+
public DocBuilder() {}
public DocBuilder(
@@ -111,7 +126,9 @@ public class DocBuilder implements Runnable, IHopCommand,
IHasHopMetadataProvide
String targetParentFolder,
boolean includingParameters,
boolean includingNotes,
- boolean includingMetadata) {
+ boolean includingMetadata,
+ boolean generateHtml,
+ boolean removeMarkdown) {
this.log = log;
this.variables = variables;
this.metadataProvider = metadataProvider;
@@ -121,6 +138,8 @@ public class DocBuilder implements Runnable, IHopCommand,
IHasHopMetadataProvide
this.includingParameters = includingParameters;
this.includingNotes = includingNotes;
this.includingMetadata = includingMetadata;
+ this.generateHtml = generateHtml;
+ this.removeMarkdown = removeMarkdown;
}
@Override
@@ -182,7 +201,9 @@ public class DocBuilder implements Runnable, IHopCommand,
IHasHopMetadataProvide
targetParentFolder,
includingParameters,
includingNotes,
- includingMetadata);
+ includingMetadata,
+ generateHtml,
+ removeMarkdown);
docBuilder.buildDocumentation(new Result());
} catch (Exception e) {
log.logError("Error generating documentation", e);
@@ -220,6 +241,8 @@ public class DocBuilder implements Runnable, IHopCommand,
IHasHopMetadataProvide
}
}
writeToc(toc, targetParentFolder, projectName);
+ generateHtmlFromToc(toc, targetParentFolder);
+
log.logBasic("Finished generating documentation for project " +
projectName);
} catch (Exception e) {
log.logError("Error building documentation", e);
@@ -410,13 +433,77 @@ public class DocBuilder implements Runnable, IHopCommand,
IHasHopMetadataProvide
}
}
- public void saveFile(String svgFilename, String pipelineSvg) throws
Exception {
- try (OutputStream outputStream = HopVfs.getOutputStream(svgFilename,
false)) {
- outputStream.write(pipelineSvg.getBytes());
+ public void saveFile(String filename, String content) throws Exception {
+ try (OutputStream outputStream = HopVfs.getOutputStream(filename, false)) {
+ outputStream.write(content.getBytes());
outputStream.flush();
}
}
+ public String readFile(String fileName) throws HopException {
+ try (InputStream inputStream = HopVfs.getInputStream(fileName)) {
+ return new String(inputStream.readAllBytes(), StandardCharsets.UTF_8);
+ } catch (Exception e) {
+ throw new HopException("Unable to read file " + fileName, e);
+ }
+ }
+
+ private String markdownToHtml(String markdown) {
+ Parser parser = Parser.builder().build();
+ HtmlRenderer renderer = HtmlRenderer.builder().build();
+
+ Node document = parser.parse(markdown);
+ return renderer.render(document);
+ }
+
+ private void generateHtmlFromToc(Toc toc, String targetParentFolder) throws
Exception {
+ log.logBasic("generateHtml flag = " + generateHtml);
+ log.logBasic("removeMarkdown flag = " + removeMarkdown);
+ if (!generateHtml) {
+ return;
+ }
+
+ // Generate index.html from index.md
+ String indexMdFile = targetParentFolder + "/" + INDEX_MD;
+ if (new java.io.File(indexMdFile).exists()) {
+ String indexMarkdown = readFile(indexMdFile);
+ String indexHtml = markdownToHtml(indexMarkdown);
+ indexHtml = indexHtml.replaceAll("\\.md([^a-zA-Z0-9])", ".html$1");
+ String indexHtmlFile = indexMdFile.replace(".md", ".html");
+ saveFile(indexHtmlFile, indexHtml);
+ log.logBasic("Generated: " + indexHtmlFile + " (with fixed links)");
+ }
+
+ // DELETE index.md if removeMarkdown flag is set
+ if (removeMarkdown) {
+ java.io.File indexFile = new java.io.File(indexMdFile);
+ if (indexFile.exists() && indexFile.delete()) {
+ log.logBasic("Deleted index markdown file: " + indexMdFile);
+ } else {
+ log.logBasic("Failed to delete index markdown file: " + indexMdFile);
+ }
+ }
+
+ // Generate other HTML files & removing the markdown if applicable
+ for (TocEntry entry : toc.getEntries()) {
+ String mdFile = targetParentFolder + "/" + entry.targetDocFile();
+ String markdown = readFile(mdFile);
+ String html = markdownToHtml(markdown);
+ String htmlFile = mdFile.replace(".md", ".html");
+ saveFile(htmlFile, html);
+ log.logBasic("Generated: " + htmlFile);
+
+ if (removeMarkdown) {
+ java.io.File file = new java.io.File(mdFile);
+ if (file.exists() && file.delete()) {
+ log.logBasic("Deleted markdown file: " + mdFile);
+ } else {
+ log.logBasic("Failed to delete markdown file (or not found): " +
mdFile);
+ }
+ }
+ }
+ }
+
private void processDatasetsFolder(
Toc toc, FileObject sourceFolder, FileObject targetFolder, String
relativeName) {}
diff --git
a/plugins/misc/documentation/src/main/resources/org/apache/hop/documentation/messages/messages_en_US.properties
b/plugins/misc/documentation/src/main/resources/org/apache/hop/documentation/messages/messages_en_US.properties
index fdc6abcd80..aefa9535d8 100644
---
a/plugins/misc/documentation/src/main/resources/org/apache/hop/documentation/messages/messages_en_US.properties
+++
b/plugins/misc/documentation/src/main/resources/org/apache/hop/documentation/messages/messages_en_US.properties
@@ -25,3 +25,5 @@ ActionDoc.Target.Label=The target folder
ActionDoc.Notes.Label=Include notes text?
ActionDoc.Parameters.Label=List parameters?
ActionDoc.Metadata.Label=Include metadata?
+ActionDoc.generateHtml.Label=Generate HTML?
+ActionDoc.RemoveMarkdown.Label=Remove MarkDown files?
\ No newline at end of file
