This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new fe19a27ce1c HDDS-11072. Publish user-facing configs to the website
(#6916)
fe19a27ce1c is described below
commit fe19a27ce1caba8381e4e893efd15d30c789c937
Author: Sarveksha Yeshavantha Raju
<[email protected]>
AuthorDate: Sun Feb 22 16:24:45 2026 +0530
HDDS-11072. Publish user-facing configs to the website (#6916)
---
.github/workflows/ci.yml | 12 ++
.github/workflows/generate-config-doc.yml | 58 +++++++
.github/workflows/update-ozone-site-config-doc.yml | 188 +++++++++++++++++++++
dev-support/ci/pr_body_config_doc.sh | 54 ++++++
dev-support/ci/xml_to_md.py | 158 +++++++++++++++++
5 files changed, 470 insertions(+)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 71af2b9824e..0d35d03c2ba 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -123,6 +123,18 @@ jobs:
with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }}
secrets: inherit
+ generate-config-doc:
+ needs:
+ - build-info
+ - build
+ if: |
+ needs.build-info.outputs.needs-build == 'true' &&
+ (github.repository != 'apache/ozone' || github.event_name ==
'pull_request' ||
+ (github.event_name == 'push' && github.ref_name == 'master'))
+ uses: ./.github/workflows/generate-config-doc.yml
+ with:
+ sha: ${{ needs.build-info.outputs.sha }}
+
compile:
needs:
- build-info
diff --git a/.github/workflows/generate-config-doc.yml
b/.github/workflows/generate-config-doc.yml
new file mode 100644
index 00000000000..36452e497fd
--- /dev/null
+++ b/.github/workflows/generate-config-doc.yml
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: generate-config-doc
+on:
+ workflow_call:
+ inputs:
+ sha:
+ type: string
+ required: true
+
+jobs:
+ generate:
+ runs-on: ubuntu-24.04
+ steps:
+ - name: Checkout project
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.sha }}
+
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: '3.x'
+
+ - name: Download ozone-bin artifact
+ uses: actions/download-artifact@v4
+ with:
+ name: ozone-bin
+ path: ozone-bin
+
+ - name: Extract tarball
+ run: |
+ mkdir -p ozone-bin/extracted
+ tar -xzf ozone-bin/ozone-*.tar.gz -C ozone-bin/extracted
+
+ - name: Generate configuration documentation
+ run: |
+ python3 dev-support/ci/xml_to_md.py ozone-bin/extracted
Configurations.md
+
+ - name: Upload generated documentation
+ uses: actions/upload-artifact@v4
+ with:
+ name: config-documentation
+ path: Configurations.md
+ retention-days: 7
diff --git a/.github/workflows/update-ozone-site-config-doc.yml
b/.github/workflows/update-ozone-site-config-doc.yml
new file mode 100644
index 00000000000..9488a998383
--- /dev/null
+++ b/.github/workflows/update-ozone-site-config-doc.yml
@@ -0,0 +1,188 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: update-ozone-site-config-doc
+on:
+ workflow_run:
+ workflows: ["ci"]
+ types: [completed]
+ branches: [master]
+
+jobs:
+ update-ozone-site-config-doc:
+ if: |
+ github.event.workflow_run.conclusion == 'success' &&
+ github.repository == 'apache/ozone' &&
+ !startsWith(github.event.workflow_run.head_commit.message, '[Auto]')
+ runs-on: ubuntu-24.04
+ steps:
+ - name: Download generated documentation
+ uses: actions/download-artifact@v4
+ with:
+ name: config-documentation
+ path: .
+ run-id: ${{ github.event.workflow_run.id }}
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Check if ozone-site repository exists
+ id: check-site-repo
+ run: |
+ REPO_OWNER="${{ github.repository_owner }}"
+ if gh repo view "$REPO_OWNER/ozone-site" >/dev/null 2>&1; then
+ echo "exists=true" >> $GITHUB_OUTPUT
+ else
+ echo "exists=false" >> $GITHUB_OUTPUT
+ echo "ozone-site repository not found for $REPO_OWNER, skipping"
+ fi
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Checkout ozone-site repository
+ if: steps.check-site-repo.outputs.exists == 'true'
+ uses: actions/checkout@v4
+ with:
+ repository: ${{ github.repository_owner }}/ozone-site
+ ref: 'master'
+ path: ozone-site
+ token: ${{ secrets.OZONE_WEBSITE_BUILD }}
+
+ - name: Check if documentation changed
+ if: steps.check-site-repo.outputs.exists == 'true'
+ id: check-changes
+ run: |
+ cd ozone-site
+
+ # Find target directory
+ TARGET_DIR=$(ls -d docs/*-administrator-guide/*-configuration
2>/dev/null | head -1)
+
+ if [ -z "$TARGET_DIR" ]; then
+ echo "changed=false" >> $GITHUB_OUTPUT
+ echo "Target directory not found, skipping"
+ exit 0
+ fi
+
+ TARGET_FILE="$TARGET_DIR/05-appendix.md"
+ echo "Checking against $TARGET_FILE"
+
+ # Copy new file to target location
+ cp ../Configurations.md "$TARGET_FILE"
+
+ # Check if file changed using git status
+ if git status --porcelain "$TARGET_FILE" | grep -q .; then
+ echo "changed=true" >> $GITHUB_OUTPUT
+ echo "target_file=$TARGET_FILE" >> $GITHUB_OUTPUT
+ echo "Configurations.md has changed in ozone-site"
+ else
+ echo "changed=false" >> $GITHUB_OUTPUT
+ echo "No changes in ozone-site documentation"
+ fi
+
+ - name: Checkout ozone repository for script access
+ if: steps.check-changes.outputs.changed == 'true'
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event.workflow_run.head_sha }}
+
+ - name: Extract JIRA ID from commit message
+ if: steps.check-changes.outputs.changed == 'true'
+ id: extract-jira
+ run: |
+ COMMIT_MSG=$(git log -1 --format='%s' ${{
github.event.workflow_run.head_sha }})
+ echo "Commit message: $COMMIT_MSG"
+
+ JIRA_ID=$(echo "$COMMIT_MSG" | grep -oE '(HDDS|OZONE)-[0-9]+' | head
-1 || echo "")
+
+ if [ -n "$JIRA_ID" ]; then
+ echo "jira_id=$JIRA_ID" >> $GITHUB_OUTPUT
+ echo "Using JIRA ID: $JIRA_ID"
+ else
+ echo "jira_id=" >> $GITHUB_OUTPUT
+ echo "No JIRA ID found"
+ fi
+
+ - name: Commit and push changes
+ if: steps.check-changes.outputs.changed == 'true'
+ run: |
+ cd ozone-site
+
+ git config user.name 'github-actions[bot]'
+ git config user.email 'github-actions[bot]@users.noreply.github.com'
+
+ # Use single branch name
+ BRANCH_NAME="automated-config-doc-update"
+
+ echo "Current branch: $(git branch --show-current)"
+ echo "Current commit: $(git rev-parse HEAD)"
+
+ # Create/reset branch at current commit
+ git checkout -B "$BRANCH_NAME"
+ echo "Created/reset branch $BRANCH_NAME at current commit"
+
+ TARGET_FILE="${{ steps.check-changes.outputs.target_file }}"
+ git add "$TARGET_FILE"
+
+ # Build commit message with JIRA ID if available
+ JIRA_ID="${{ steps.extract-jira.outputs.jira_id }}"
+ COMMIT_MSG="[Auto] Update configuration documentation from ozone ${{
github.event.workflow_run.head_sha }}"
+ if [ -n "$JIRA_ID" ]; then
+ COMMIT_MSG="$JIRA_ID. $COMMIT_MSG"
+ fi
+
+ git commit -m "$COMMIT_MSG"
+
+ echo "Pushing $BRANCH_NAME to origin"
+ git push -f origin "$BRANCH_NAME"
+
+ - name: Create or update Pull Request in ozone-site
+ if: steps.check-changes.outputs.changed == 'true' && github.repository
== 'apache/ozone'
+ env:
+ GH_TOKEN: ${{ secrets.OZONE_WEBSITE_BUILD }}
+ run: |
+ cd ozone-site
+
+ BRANCH_NAME="automated-config-doc-update"
+ REPO="${{ github.repository }}"
+ JIRA_ID="${{ steps.extract-jira.outputs.jira_id }}"
+
+ # Generate PR body
+ cd ..
+ dev-support/ci/pr_body_config_doc.sh \
+ "$REPO" \
+ "${{ github.workflow }}" \
+ "${{ github.run_id }}" \
+ "${{ github.event.workflow_run.head_branch }}" \
+ "${{ github.event.workflow_run.head_sha }}" \
+ "$JIRA_ID" > ozone-site/pr_body.txt
+
+ cd ozone-site
+
+ # Check if PR already exists
+ EXISTING_PR=$(gh pr list --repo "${{ github.repository_owner
}}/ozone-site" \
+ --head "$BRANCH_NAME" --base master --json number --jq
'.[0].number' || echo "")
+
+ if [ -n "$EXISTING_PR" ]; then
+ echo "Updating existing PR #$EXISTING_PR"
+ gh pr edit "$EXISTING_PR" --repo "${{ github.repository_owner
}}/ozone-site" --body-file pr_body.txt
+ else
+ echo "Creating new PR"
+ TITLE="[Auto] Update configuration documentation"
+ if [ -n "$JIRA_ID" ]; then
+ TITLE="$JIRA_ID. $TITLE"
+ fi
+ gh pr create --repo "${{ github.repository_owner }}/ozone-site" \
+ --base master --head "$BRANCH_NAME" \
+ --title "$TITLE" \
+ --body-file pr_body.txt
+ fi
diff --git a/dev-support/ci/pr_body_config_doc.sh
b/dev-support/ci/pr_body_config_doc.sh
new file mode 100755
index 00000000000..49ca4e67fb2
--- /dev/null
+++ b/dev-support/ci/pr_body_config_doc.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generate PR body for configuration documentation update
+# Usage: pr_body_config_doc.sh <repo> <workflow_name> <workflow_run_id>
<branch_name> <commit_sha> <jira_id>
+
+set -euo pipefail
+
+REPO="${1}"
+WORKFLOW_NAME="${2}"
+WORKFLOW_RUN_ID="${3}"
+BRANCH_NAME="${4}"
+COMMIT_SHA="${5}"
+JIRA_ID="${6:-}"
+
+cat <<EOF
+## What changes were proposed in this pull request?
+
+This is an automated pull request triggered by the
[$WORKFLOW_NAME](https://github.com/${REPO}/actions/runs/${WORKFLOW_RUN_ID})
workflow run from
[\`${COMMIT_SHA}\`](https://github.com/${REPO}/commit/${COMMIT_SHA}) on
[\`${BRANCH_NAME}\`](https://github.com/${REPO}/tree/${BRANCH_NAME}).
+
+The configuration documentation has been automatically regenerated from the
latest \`ozone-default.xml\` files.
+
+EOF
+
+if [ -n "${JIRA_ID}" ]; then
+ cat <<EOF
+## What is the link to the Apache JIRA?
+
+[${JIRA_ID}](https://issues.apache.org/jira/browse/${JIRA_ID})
+
+EOF
+fi
+
+cat <<EOF
+## How was this patch tested?
+
+This is an auto-generated documentation update. Reviewers should verify:
+- The markdown file is properly formatted
+- Configuration keys are correctly extracted
+- No unexpected changes in existing configurations
+EOF
diff --git a/dev-support/ci/xml_to_md.py b/dev-support/ci/xml_to_md.py
new file mode 100644
index 00000000000..e3c6ef14ebc
--- /dev/null
+++ b/dev-support/ci/xml_to_md.py
@@ -0,0 +1,158 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Python file to convert XML properties into Markdown
+import os
+import re
+import zipfile
+import xml.etree.ElementTree as ET
+from collections import namedtuple
+from pathlib import Path
+import sys
+
+Property = namedtuple('Property', ['name', 'value', 'tag', 'description'])
+
+def extract_xml_from_jar(jar_path, xml_filename):
+ xml_files = []
+ with zipfile.ZipFile(jar_path, 'r') as jar:
+ for file_info in jar.infolist():
+ if file_info.filename.endswith(xml_filename):
+ with jar.open(file_info.filename) as xml_file:
+ xml_files.append(xml_file.read())
+ return xml_files
+
+def wrap_config_keys_in_description(description, config_keys):
+ words = description.split()
+ wrapped_words = []
+
+ for word in words:
+ # Strip punctuation to check if the word is a config key
+ stripped_word = word.strip('.,;:!?()[]{}')
+ if stripped_word in config_keys:
+ # Preserve punctuation around the wrapped key
+ prefix = word[:len(word) - len(word.lstrip('.,;:!?()[]{}'))]
+ suffix = word[len(stripped_word) + len(prefix):]
+ wrapped_words.append(f'{prefix}`{stripped_word}`{suffix}')
+ else:
+ wrapped_words.append(word)
+
+ return ' '.join(wrapped_words)
+
+def parse_xml_file(xml_content, properties):
+ root = ET.fromstring(xml_content)
+ for prop in root.findall('property'):
+ name = prop.findtext('name')
+ if not name:
+ raise ValueError("Property 'name' is required but missing in XML.")
+ description = prop.findtext('description', '')
+ if not description:
+ raise ValueError(f"Property '{name}' is missing a description.")
+ tag = prop.findtext('tag', '')
+
+ properties[name] = Property(
+ name=name,
+ value=prop.findtext('value', ''),
+ tag=tag,
+ description=' '.join(description.split()).strip()
+ )
+ return properties
+
+def format_properties(properties):
+ config_keys = set(properties.keys())
+ formatted_properties = {}
+
+ for name, prop in properties.items():
+ if prop.tag:
+ formatted_tag = ', '.join(f'`{t.strip()}`' for t in prop.tag.split(','))
+ else:
+ formatted_tag = ''
+
+ # Wrap config keys in description now that we have all configs
+ formatted_description = wrap_config_keys_in_description(prop.description,
config_keys)
+
+ formatted_properties[name] = Property(
+ name=prop.name,
+ value=prop.value,
+ tag=formatted_tag,
+ description=formatted_description
+ )
+ return formatted_properties
+
+def generate_markdown(properties):
+ markdown = """---
+sidebar_label: Appendix
+---
+
+# Configuration Key Appendix
+
+This page provides a comprehensive overview of the configuration keys
available in Ozone.
+
+| Name | Default Value | Tags | Description |
+|:-----|:--------------|:-----|:------------|
+"""
+
+ for prop in sorted(properties.values(), key=lambda p: p.name):
+ # Escape pipe characters in description to prevent breaking the table
+ description = prop.description.replace('|', '\\|')
+ value = prop.value if prop.value else ''
+
+ markdown += f"| `{prop.name}` | {value} | {prop.tag} | {description} |\n"
+
+ return markdown
+
+def main():
+ if len(sys.argv) < 2 or len(sys.argv) > 3:
+ print("Usage: python3 xml_to_md.py <base_path> [<output_path>]")
+ sys.exit(1)
+
+ base_path = sys.argv[1]
+ output_path = sys.argv[2] if len(sys.argv) == 3 else None
+
+ # Find ozone SNAPSHOT directory dynamically using regex
+ snapshot_dir = next(
+ (os.path.join(base_path, d) for d in os.listdir(base_path) if
re.match(r'ozone-[\d.]+\d-SNAPSHOT', d)),
+ None
+ )
+
+ if not snapshot_dir:
+ raise ValueError("SNAPSHOT directory not found in the specified base
path.")
+
+ extract_path = os.path.join(snapshot_dir, 'share', 'ozone', 'lib')
+ xml_filename = 'ozone-default.xml'
+
+ property_map = {}
+ for file_name in os.listdir(extract_path):
+ if file_name.startswith('hdds-common-') and file_name.endswith('.jar'):
+ jar_path = os.path.join(extract_path, file_name)
+ xml_contents = extract_xml_from_jar(jar_path, xml_filename)
+ for xml_content in xml_contents:
+ parse_xml_file(xml_content, property_map)
+
+ formatted_properties = format_properties(property_map)
+ markdown_content = generate_markdown(formatted_properties)
+
+ if output_path:
+ output_path = Path(output_path)
+ output_path.parent.mkdir(parents=True, exist_ok=True)
+ with output_path.open('w', encoding='utf-8') as file:
+ file.write(markdown_content)
+ else:
+ print(markdown_content)
+
+if __name__ == '__main__':
+ main()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
