errose28 commented on code in PR #6916: URL: https://github.com/apache/ozone/pull/6916#discussion_r2748502944
########## dev-support/ci/xml_to_md.py: ########## @@ -0,0 +1,127 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Python file to convert XML properties into Markdown +import os +import re +import zipfile +import xml.etree.ElementTree as ET +from collections import namedtuple +from pathlib import Path +import sys + +Property = namedtuple('Property', ['name', 'value', 'tag', 'description']) + +def extract_xml_from_jar(jar_path, xml_filename): + xml_files = [] + with zipfile.ZipFile(jar_path, 'r') as jar: + for file_info in jar.infolist(): + if file_info.filename.endswith(xml_filename): + with jar.open(file_info.filename) as xml_file: + xml_files.append(xml_file.read()) + return xml_files + +def wrap_config_keys_in_description(description, properties): + for key in properties.keys(): + description = re.sub(r'\b' + re.escape(key) + r'\b', f'`{key}`', description) + return description + +def parse_xml_file(xml_content, properties): + root = ET.fromstring(xml_content) + for prop in root.findall('property'): + name = prop.findtext('name') + if not name: + raise ValueError("Property 'name' is required but missing in XML.") + description = prop.findtext('description', '') + if not description: + raise ValueError(f"Property '{name}' is missing a description.") + tag = prop.findtext('tag', '') + if tag: + formatted_tag = '<br/>'.join(f'`{t.strip()}`' for t in tag.split(',')) + else: + formatted_tag = '' + + properties[name] = Property( + name=name, + value=prop.findtext('value', ''), + tag=formatted_tag, + description=wrap_config_keys_in_description( + ' '.join(description.split()).strip(), + properties + ) + ) + return properties + +def generate_markdown(properties): + markdown = f""" +## Ozone Configuration Keys +This page provides da comprehensive overview of the configuration keys available in Ozone. +### Configuration Keys +""" + + for prop in sorted(properties.values(), key=lambda p: p.name): + markdown += f""" +| **Name** | `{prop.name}` | +|:----------------|:----------------------------| +| **Value** | {prop.value} | +| **Tag** | {prop.tag} | +| **Description** | {prop.description} | +-------------------------------------------------------------------------------- Review Comment: Should we create a header row and put each entry in its own column like [Hadoop](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/core-default.xml)? It looks like this doesn't actually create one table, it creates a small table for each config with a doc separator in between. <img width="1011" height="786" alt="Image" src="https://github.com/user-attachments/assets/80c8fe73-063e-4102-9d8d-910a42565905"; /> ########## dev-support/ci/xml_to_md.py: ########## @@ -0,0 +1,127 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Python file to convert XML properties into Markdown +import os +import re +import zipfile +import xml.etree.ElementTree as ET +from collections import namedtuple +from pathlib import Path +import sys + +Property = namedtuple('Property', ['name', 'value', 'tag', 'description']) + +def extract_xml_from_jar(jar_path, xml_filename): + xml_files = [] + with zipfile.ZipFile(jar_path, 'r') as jar: + for file_info in jar.infolist(): + if file_info.filename.endswith(xml_filename): + with jar.open(file_info.filename) as xml_file: + xml_files.append(xml_file.read()) + return xml_files + +def wrap_config_keys_in_description(description, properties): + for key in properties.keys(): + description = re.sub(r'\b' + re.escape(key) + r'\b', f'`{key}`', description) + return description + +def parse_xml_file(xml_content, properties): + root = ET.fromstring(xml_content) + for prop in root.findall('property'): + name = prop.findtext('name') + if not name: + raise ValueError("Property 'name' is required but missing in XML.") + description = prop.findtext('description', '') + if not description: + raise ValueError(f"Property '{name}' is missing a description.") + tag = prop.findtext('tag', '') + if tag: + formatted_tag = '<br/>'.join(f'`{t.strip()}`' for t in tag.split(',')) + else: + formatted_tag = '' + + properties[name] = Property( + name=name, + value=prop.findtext('value', ''), + tag=formatted_tag, + description=wrap_config_keys_in_description( + ' '.join(description.split()).strip(), + properties + ) + ) + return properties + +def generate_markdown(properties): + markdown = f""" +## Ozone Configuration Keys +This page provides da comprehensive overview of the configuration keys available in Ozone. +### Configuration Keys +""" + + for prop in sorted(properties.values(), key=lambda p: p.name): + markdown += f""" +| **Name** | `{prop.name}` | +|:----------------|:----------------------------| +| **Value** | {prop.value} | Review Comment: ```suggestion | **Default Value** | {prop.value} | ``` ########## dev-support/ci/xml_to_md.py: ########## @@ -0,0 +1,127 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Python file to convert XML properties into Markdown +import os +import re +import zipfile +import xml.etree.ElementTree as ET +from collections import namedtuple +from pathlib import Path +import sys + +Property = namedtuple('Property', ['name', 'value', 'tag', 'description']) + +def extract_xml_from_jar(jar_path, xml_filename): + xml_files = [] + with zipfile.ZipFile(jar_path, 'r') as jar: + for file_info in jar.infolist(): + if file_info.filename.endswith(xml_filename): + with jar.open(file_info.filename) as xml_file: + xml_files.append(xml_file.read()) + return xml_files + +def wrap_config_keys_in_description(description, properties): + for key in properties.keys(): + description = re.sub(r'\b' + re.escape(key) + r'\b', f'`{key}`', description) + return description + +def parse_xml_file(xml_content, properties): + root = ET.fromstring(xml_content) + for prop in root.findall('property'): + name = prop.findtext('name') + if not name: + raise ValueError("Property 'name' is required but missing in XML.") + description = prop.findtext('description', '') + if not description: + raise ValueError(f"Property '{name}' is missing a description.") + tag = prop.findtext('tag', '') + if tag: + formatted_tag = '<br/>'.join(f'`{t.strip()}`' for t in tag.split(',')) + else: + formatted_tag = '' + + properties[name] = Property( + name=name, + value=prop.findtext('value', ''), + tag=formatted_tag, + description=wrap_config_keys_in_description( + ' '.join(description.split()).strip(), + properties Review Comment: This is only going to replace config keys that have already been processed. If one config references another that has not yet been added to `properties`, it won't get replaced. The config processing is also O(n^2) because it is iterating the set of configs, instead of iterating the words in the description and checking if they are in the set. Instead we can process all the properties first and then do the description substitution at the end so we have all the configs to reference. When doing the substitution, we should check if each word in each description is in the config set. This will be two iterations through the config list instead of n^2. ########## dev-support/ci/xml_to_md.py: ########## @@ -0,0 +1,127 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Python file to convert XML properties into Markdown +import os +import re +import zipfile +import xml.etree.ElementTree as ET +from collections import namedtuple +from pathlib import Path +import sys + +Property = namedtuple('Property', ['name', 'value', 'tag', 'description']) + +def extract_xml_from_jar(jar_path, xml_filename): + xml_files = [] + with zipfile.ZipFile(jar_path, 'r') as jar: + for file_info in jar.infolist(): + if file_info.filename.endswith(xml_filename): + with jar.open(file_info.filename) as xml_file: + xml_files.append(xml_file.read()) + return xml_files + +def wrap_config_keys_in_description(description, properties): + for key in properties.keys(): + description = re.sub(r'\b' + re.escape(key) + r'\b', f'`{key}`', description) + return description + +def parse_xml_file(xml_content, properties): + root = ET.fromstring(xml_content) + for prop in root.findall('property'): + name = prop.findtext('name') + if not name: + raise ValueError("Property 'name' is required but missing in XML.") + description = prop.findtext('description', '') + if not description: + raise ValueError(f"Property '{name}' is missing a description.") + tag = prop.findtext('tag', '') + if tag: + formatted_tag = '<br/>'.join(f'`{t.strip()}`' for t in tag.split(',')) + else: + formatted_tag = '' + + properties[name] = Property( + name=name, + value=prop.findtext('value', ''), + tag=formatted_tag, + description=wrap_config_keys_in_description( + ' '.join(description.split()).strip(), + properties + ) + ) + return properties + +def generate_markdown(properties): + markdown = f""" +## Ozone Configuration Keys +This page provides da comprehensive overview of the configuration keys available in Ozone. +### Configuration Keys +""" + + for prop in sorted(properties.values(), key=lambda p: p.name): + markdown += f""" +| **Name** | `{prop.name}` | +|:----------------|:----------------------------| +| **Value** | {prop.value} | +| **Tag** | {prop.tag} | Review Comment: Configs can have multiple tags. Take `ozone.container.cache.size` as an example. We can still keep them in one cell of the table, but we should format them as a comma separated list. ########## dev-support/ci/xml_to_md.py: ########## @@ -0,0 +1,127 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Python file to convert XML properties into Markdown +import os +import re +import zipfile +import xml.etree.ElementTree as ET +from collections import namedtuple +from pathlib import Path +import sys + +Property = namedtuple('Property', ['name', 'value', 'tag', 'description']) + +def extract_xml_from_jar(jar_path, xml_filename): + xml_files = [] + with zipfile.ZipFile(jar_path, 'r') as jar: + for file_info in jar.infolist(): + if file_info.filename.endswith(xml_filename): + with jar.open(file_info.filename) as xml_file: + xml_files.append(xml_file.read()) + return xml_files + +def wrap_config_keys_in_description(description, properties): + for key in properties.keys(): + description = re.sub(r'\b' + re.escape(key) + r'\b', f'`{key}`', description) + return description + +def parse_xml_file(xml_content, properties): + root = ET.fromstring(xml_content) + for prop in root.findall('property'): + name = prop.findtext('name') + if not name: + raise ValueError("Property 'name' is required but missing in XML.") + description = prop.findtext('description', '') + if not description: + raise ValueError(f"Property '{name}' is missing a description.") + tag = prop.findtext('tag', '') + if tag: + formatted_tag = '<br/>'.join(f'`{t.strip()}`' for t in tag.split(',')) + else: + formatted_tag = '' + + properties[name] = Property( + name=name, + value=prop.findtext('value', ''), + tag=formatted_tag, + description=wrap_config_keys_in_description( + ' '.join(description.split()).strip(), + properties + ) + ) + return properties + +def generate_markdown(properties): + markdown = f""" +## Ozone Configuration Keys +This page provides da comprehensive overview of the configuration keys available in Ozone. +### Configuration Keys +""" + + for prop in sorted(properties.values(), key=lambda p: p.name): + markdown += f""" +| **Name** | `{prop.name}` | +|:----------------|:----------------------------| +| **Value** | {prop.value} | +| **Tag** | {prop.tag} | +| **Description** | {prop.description} | +-------------------------------------------------------------------------------- +""" + return markdown + +def main(): + if len(sys.argv) < 2 or len(sys.argv) > 3: + print("Usage: python3 xml_to_md.py <base_path> [<output_path>]") + sys.exit(1) + + base_path = sys.argv[1] + output_path = sys.argv[2] if len(sys.argv) == 3 else None + + # Find ozone SNAPSHOT directory dynamically using regex + snapshot_dir = next( + (os.path.join(base_path, d) for d in os.listdir(base_path) if re.match(r'ozone-[\d.]+\d-SNAPSHOT', d)), + None + ) + + if not snapshot_dir: + raise ValueError("SNAPSHOT directory not found in the specified base path.") + + extract_path = os.path.join(snapshot_dir, 'share', 'ozone', 'lib') + xml_filename = 'ozone-default.xml' + + property_map = {} + for file_name in os.listdir(extract_path): + if file_name.endswith('.jar'): Review Comment: Can we go right to the `hdds-common` jar to extract the file, or are there other configs in other jars we need? If there are multiple jars with configs, can we at least trim down the search space by only searching in jars that begin with `ozone` or `hdds`? ########## dev-support/ci/xml_to_md.py: ########## @@ -0,0 +1,127 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Python file to convert XML properties into Markdown +import os +import re +import zipfile +import xml.etree.ElementTree as ET +from collections import namedtuple +from pathlib import Path +import sys + +Property = namedtuple('Property', ['name', 'value', 'tag', 'description']) + +def extract_xml_from_jar(jar_path, xml_filename): + xml_files = [] + with zipfile.ZipFile(jar_path, 'r') as jar: + for file_info in jar.infolist(): + if file_info.filename.endswith(xml_filename): + with jar.open(file_info.filename) as xml_file: + xml_files.append(xml_file.read()) + return xml_files + +def wrap_config_keys_in_description(description, properties): + for key in properties.keys(): + description = re.sub(r'\b' + re.escape(key) + r'\b', f'`{key}`', description) + return description + +def parse_xml_file(xml_content, properties): + root = ET.fromstring(xml_content) + for prop in root.findall('property'): + name = prop.findtext('name') + if not name: + raise ValueError("Property 'name' is required but missing in XML.") + description = prop.findtext('description', '') + if not description: + raise ValueError(f"Property '{name}' is missing a description.") + tag = prop.findtext('tag', '') + if tag: + formatted_tag = '<br/>'.join(f'`{t.strip()}`' for t in tag.split(',')) + else: + formatted_tag = '' + + properties[name] = Property( + name=name, + value=prop.findtext('value', ''), + tag=formatted_tag, + description=wrap_config_keys_in_description( + ' '.join(description.split()).strip(), + properties + ) + ) + return properties + +def generate_markdown(properties): + markdown = f""" +## Ozone Configuration Keys +This page provides da comprehensive overview of the configuration keys available in Ozone. +### Configuration Keys Review Comment: ```suggestion # Ozone Configuration Keys This page provides a comprehensive overview of the configuration keys available in Ozone. ## Configuration Keys ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
