This is an automated email from the ASF dual-hosted git repository.
yuanfeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 25e49249d [AMORO-3981] Automatic generate amoro configuration docs
(#3982)
25e49249d is described below
commit 25e49249df4657bdb95b782e25ecf9f74d07d259
Author: Fei Wang <[email protected]>
AuthorDate: Fri Dec 5 20:19:29 2025 -0800
[AMORO-3981] Automatic generate amoro configuration docs (#3982)
* auto generate ams config
* add auto generate notice
* junit 5
* convertToString
* Update CONTRIBUTING.md
---------
Co-authored-by: big face cat <[email protected]>
---
CONTRIBUTING.md | 8 +
.../apache/amoro/config/ConfigurationsTest.java | 279 +++++++++++++++++++++
docs/configuration/ams-config.md | 137 ++++++++++
pom.xml | 3 +
site/amoro-docs/hugo.toml | 1 +
5 files changed, 428 insertions(+)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 46de0a962..e502ac705 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -205,6 +205,14 @@ After installing the plugin, format your code
automatically by applying the foll
7. Under “Formatting Actions”, select “Optimize imports” and “Reformat file”.
8. From the “All file types list” next to “Reformat code”, select Java and
Scala.
+### Updating Configuration Documentation
+
+If you modify ConfigOptions, please regenerate the configuration documentation
by running:
+
+```shell
+UPDATE=1 ./mvnw test -pl amoro-ams -am -Dtest=ConfigurationsTest
+```
+
### Copyright
All files (including source code, configuration files) in the project are
required to declare
diff --git
a/amoro-ams/src/test/java/org/apache/amoro/config/ConfigurationsTest.java
b/amoro-ams/src/test/java/org/apache/amoro/config/ConfigurationsTest.java
new file mode 100644
index 000000000..4019aa490
--- /dev/null
+++ b/amoro-ams/src/test/java/org/apache/amoro/config/ConfigurationsTest.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.config;
+
+import org.apache.amoro.config.shade.utils.ConfigShadeUtils;
+import org.apache.amoro.server.AmoroManagementConf;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * End-to-end test cases for configuration documentation.
+ *
+ * <p>The golden result file is "docs/configuration/".
+ *
+ * <p>To run the test suite:
+ *
+ * <pre>
+ * ./mvnw test -pl amoro-ams -am -Dtest=ConfigurationsTest
+ * </pre>
+ *
+ * <p>To re-generate golden file, run:
+ *
+ * <pre>
+ * UPDATE=1 ./mvnw test -pl amoro-ams -am -Dtest=ConfigurationsTest
+ * </pre>
+ */
+public class ConfigurationsTest {
+ public static String CONFIGURATION_DOCS_PATH = "docs/configuration";
+ public static String UPDATE_CMD =
+ "UPDATE=1 ./mvnw test -pl amoro-ams -am -Dtest=ConfigurationsTest";
+
+ @Test
+ public void testAmoroManagementConfDocumentation() throws Exception {
+ List<AmoroConfInfo> confInfoList = new ArrayList<>();
+ confInfoList.add(
+ new AmoroConfInfo(
+ AmoroManagementConf.class,
+ "Amoro Management Service Configuration",
+ "The configuration options for Amoro Management Service (AMS)."));
+ confInfoList.add(
+ new AmoroConfInfo(
+ ConfigShadeUtils.class,
+ "Shade Utils Configuration",
+ "The configuration options for Amoro Configuration Shade Utils."));
+ generateConfigurationMarkdown("ams-config.md", "AMS Configuration", 100,
confInfoList);
+ }
+
+ /**
+ * Generate configuration documentation for multiple configuration classes.
+ *
+ * @param markdown The markdown file name to generate
+ * @param title The title for the markdown document
+ * @param weight The weight for ordering in the menu
+ * @param confInfoList List of configuration class information
+ * @throws Exception if generation fails
+ */
+ protected void generateConfigurationMarkdown(
+ String markdown, String title, int weight, List<AmoroConfInfo>
confInfoList)
+ throws Exception {
+ Path markdownFile = getMarkdownFilePath(CONFIGURATION_DOCS_PATH + "/" +
markdown);
+ List<String> output = new ArrayList<>();
+
+ appendFrontHeader(output, title, markdown, weight);
+ appendLicenseHeader(output);
+ output.add("<!-- This file is auto-generated. To update, run: " +
UPDATE_CMD + " -->");
+ output.add("");
+ output.add("# " + title);
+ output.add("");
+
+ for (AmoroConfInfo confInfo : confInfoList) {
+ output.add("## " + confInfo.title);
+ output.add("");
+ output.add(confInfo.description);
+ output.add("");
+ appendTableStyle(output);
+ appendConfigurationTableHeader(output);
+
+ List<ConfigOptionInfo> configOptions =
extractConfigOptions(confInfo.confClass);
+ configOptions.sort(Comparator.comparing(c -> c.key));
+
+ for (ConfigOptionInfo info : configOptions) {
+ String row =
+ String.format(
+ "| %s | %s | %s |",
+ escape(info.key), escape(info.defaultValue),
escape(info.description));
+ output.add(row);
+ }
+
+ // Add some space between different configuration sections
+ output.add("");
+ output.add("");
+ }
+
+ verifyOutput(markdownFile, output);
+ }
+
+ private Path getMarkdownFilePath(String relativePath) {
+ // Get the project root directory
+ String projectRoot = System.getProperty("user.dir");
+ if (projectRoot.endsWith("amoro-ams")) {
+ projectRoot = Paths.get(projectRoot).getParent().toString();
+ }
+ return Paths.get(projectRoot, relativePath);
+ }
+
+ private List<ConfigOptionInfo> extractConfigOptions(Class<?> confClass) {
+ List<ConfigOptionInfo> options = new ArrayList<>();
+
+ for (Field field : confClass.getDeclaredFields()) {
+ // Only process static ConfigOption fields
+ if (!Modifier.isStatic(field.getModifiers())) {
+ continue;
+ }
+
+ if (!ConfigOption.class.isAssignableFrom(field.getType())) {
+ continue;
+ }
+
+ try {
+ field.setAccessible(true);
+ ConfigOption<?> configOption = (ConfigOption<?>) field.get(null);
+
+ ConfigOptionInfo info = new ConfigOptionInfo();
+ info.key = configOption.key();
+ info.defaultValue =
+ Optional.ofNullable(configOption.defaultValue())
+ .map(ConfigHelpers::convertToString)
+ .orElse("<undefined>");
+ info.description = configOption.description();
+
+ options.add(info);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException("Failed to access field: " +
field.getName(), e);
+ }
+ }
+
+ return options;
+ }
+
+ private String escape(String s) {
+ if (s == null) {
+ return "";
+ }
+ return s.replace("|", "\\|").replace("<", "<").replace(">",
">").replace("\n", "<br>");
+ }
+
+ private void appendFrontHeader(List<String> output, String title, String
markdown, int weight) {
+ String baseName = markdown.replace(".md", "");
+ output.add("---");
+ output.add("title: " + title);
+ output.add("url: " + baseName);
+ output.add("aliases:");
+ output.add(" - \"configuration/" + baseName + "\"");
+ output.add("menu:");
+ output.add(" main:");
+ output.add(" parent: Configuration");
+ output.add(" weight: " + weight);
+ output.add("---");
+ }
+
+ private void appendLicenseHeader(List<String> output) {
+ output.add("<!--");
+ output.add(" ~ Licensed to the Apache Software Foundation (ASF) under
one");
+ output.add(" ~ or more contributor license agreements. See the NOTICE
file");
+ output.add(" ~ distributed with this work for additional information");
+ output.add(" ~ regarding copyright ownership. The ASF licenses this
file");
+ output.add(" ~ to you under the Apache License, Version 2.0 (the");
+ output.add(" ~ \"License\"); you may not use this file except in
compliance");
+ output.add(" ~ with the License. You may obtain a copy of the License
at");
+ output.add(" ~");
+ output.add(" ~ http://www.apache.org/licenses/LICENSE-2.0";);
+ output.add(" ~");
+ output.add(" ~ Unless required by applicable law or agreed to in writing,
software");
+ output.add(" ~ distributed under the License is distributed on an \"AS
IS\" BASIS,");
+ output.add(" ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
express or implied.");
+ output.add(" ~ See the License for the specific language governing
permissions and");
+ output.add(" ~ limitations under the License.");
+ output.add(" -->");
+ }
+
+ private void appendTableStyle(List<String> output) {
+ output.add("<style>");
+ output.add("table { width: 100%; table-layout: fixed; }");
+ output.add("table th, table td { vertical-align: top; }");
+ output.add(
+ "table td:first-child, table th:first-child { word-break: normal;
width: 40%; }"); // Key
+ output.add(
+ "table td:nth-child(2), table th:nth-child(2) { width: 20%;
word-break: break-all; }"); // Default
+ output.add(
+ "table td:last-child, table th:last-child { width: 40%; word-break:
break-all; }"); // Description
+ output.add("</style>");
+ }
+
+ private void appendConfigurationTableHeader(List<String> output) {
+ output.add("| Key | Default | Description |");
+ output.add("| --- | ------- | ----------- |");
+ }
+
+ private void verifyOutput(Path goldenFile, List<String> output) throws
IOException {
+ if ("1".equals(System.getenv("UPDATE"))) {
+ // Update mode: write the golden file
+ Files.createDirectories(goldenFile.getParent());
+ Files.write(
+ goldenFile,
+ output,
+ StandardCharsets.UTF_8,
+ StandardOpenOption.CREATE,
+ StandardOpenOption.TRUNCATE_EXISTING);
+ System.out.println("Updated golden file: " + goldenFile);
+ } else {
+ // Verify mode: compare with existing golden file
+ if (!Files.exists(goldenFile)) {
+ Assertions.fail(
+ String.format("%s does not exist. Generate it with:\n%s",
goldenFile, UPDATE_CMD));
+ }
+
+ List<String> expected = Files.readAllLines(goldenFile,
StandardCharsets.UTF_8);
+ String hint =
+ String.format(
+ "\n%s is out of date, please update the golden file
with:\n\n%s\n",
+ goldenFile, UPDATE_CMD);
+
+ Assertions.assertEquals(expected.size(), output.size(), hint + "Number
of lines mismatch");
+
+ for (int i = 0; i < expected.size(); i++) {
+ Assertions.assertEquals(
+ expected.get(i), output.get(i), hint + "Line " + (i + 1) + "
mismatch");
+ }
+ }
+ }
+
+ private static class ConfigOptionInfo {
+ String key;
+ String defaultValue;
+ String description;
+ }
+
+ /** Class to hold configuration class information for documentation
generation. */
+ public static class AmoroConfInfo {
+ Class<?> confClass;
+ String title;
+ String description;
+
+ public AmoroConfInfo(Class<?> confClass, String title, String description)
{
+ this.confClass = confClass;
+ this.title = title;
+ this.description = description;
+ }
+ }
+}
diff --git a/docs/configuration/ams-config.md b/docs/configuration/ams-config.md
new file mode 100644
index 000000000..7a99924ef
--- /dev/null
+++ b/docs/configuration/ams-config.md
@@ -0,0 +1,137 @@
+---
+title: AMS Configuration
+url: ams-config
+aliases:
+ - "configuration/ams-config"
+menu:
+ main:
+ parent: Configuration
+ weight: 100
+---
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<!-- This file is auto-generated. To update, run: UPDATE=1 ./mvnw test -pl
amoro-ams -am -Dtest=ConfigurationsTest -->
+
+# AMS Configuration
+
+## Amoro Management Service Configuration
+
+The configuration options for Amoro Management Service (AMS).
+
+<style>
+table { width: 100%; table-layout: fixed; }
+table th, table td { vertical-align: top; }
+table td:first-child, table th:first-child { word-break: normal; width: 40%; }
+table td:nth-child(2), table th:nth-child(2) { width: 20%; word-break:
break-all; }
+table td:last-child, table th:last-child { width: 40%; word-break: break-all; }
+</style>
+| Key | Default | Description |
+| --- | ------- | ----------- |
+| admin-password | admin | The administrator password |
+| admin-username | admin | The administrator account name. |
+| auto-create-tags.enabled | true | Enable creating tags. |
+| auto-create-tags.interval | 60000000000 ns | Interval for creating tags. |
+| auto-create-tags.thread-count | 3 | The number of threads used for creating
tags. |
+| blocker.timeout | 60000000000 ns | Session timeout. Default unit is
milliseconds if not specified. |
+| catalog-meta-cache.expiration-interval | 60000000000 ns | TTL for catalog
metadata. |
+| clean-dangling-delete-files.enabled | true | Enable dangling delete files
cleaning. |
+| clean-dangling-delete-files.thread-count | 10 | The number of threads used
for dangling delete files cleaning. |
+| clean-orphan-files.enabled | true | Enable orphan files cleaning. |
+| clean-orphan-files.interval | 86400000000000 ns | Interval for cleaning
orphan files. |
+| clean-orphan-files.thread-count | 10 | The number of threads used for orphan
files cleaning. |
+| data-expiration.enabled | true | Enable data expiration |
+| data-expiration.interval | 86400000000000 ns | Execute interval for data
expiration |
+| data-expiration.thread-count | 10 | The number of threads used for data
expiring |
+| database.auto-create-tables | true | Auto init table schema when started |
+| database.connection-pool-max-idle | 16 | Max idle connect count of database
connect pool. |
+| database.connection-pool-max-total | 20 | Max connect count of database
connect pool. |
+| database.connection-pool-max-wait-millis | 30000 | Max wait time before
getting a connection timeout. |
+| database.jdbc-driver-class | org.apache.derby.jdbc.EmbeddedDriver | The JDBC
driver class name for connecting to the database. |
+| database.password | | The password for connecting to the database. |
+| database.type | derby | Database type. |
+| database.url | jdbc:derby:/tmp/amoro/derby;create=true | Database connection
address |
+| database.username | root | The username for connecting to the database. |
+| expire-snapshots.enabled | true | Enable snapshots expiring. |
+| expire-snapshots.thread-count | 10 | The number of threads used for
snapshots expiring. |
+| ha.cluster-name | default | Amoro management service cluster name. |
+| ha.connection-timeout | 300000000000 ns | The Zookeeper connection timeout
in milliseconds. |
+| ha.enabled | false | Whether to enable high availability mode. |
+| ha.session-timeout | 30000000000 ns | The Zookeeper session timeout in
milliseconds. |
+| ha.zookeeper-address | | The Zookeeper address used for high availability. |
+| http-server.auth-basic-provider |
org.apache.amoro.server.authentication.DefaultPasswdAuthenticationProvider |
User-defined password authentication implementation of
org.apache.amoro.authentication.PasswdAuthenticationProvider |
+| http-server.auth-jwt-provider | <undefined> | User-defined JWT (JSON
Web Token) authentication implementation of
org.apache.amoro.authentication.TokenAuthenticationProvider |
+| http-server.bind-port | 19090 | Port that the Http server is bound to. |
+| http-server.proxy-client-ip-header | X-Real-IP | The HTTP header to record
the real client IP address. If your server is behind a load balancer or other
proxy, the server will see this load balancer or proxy IP address as the client
IP address, to get around this common issue, most load balancers or proxies
offer the ability to record the real remote IP address in an HTTP header that
will be added to the request for other devices to use. |
+| http-server.rest-auth-type | token | The authentication used by REST APIs,
token (default), basic or jwt. |
+| http-server.session-timeout | 604800000000000 ns | Timeout for http session.
|
+| optimizer.heart-beat-timeout | 60000000000 ns | Timeout duration for
Optimizer heartbeat. |
+| optimizer.max-planning-parallelism | 1 | Max planning parallelism in one
optimizer group. |
+| optimizer.polling-timeout | 3000000000 ns | Optimizer polling task timeout. |
+| optimizer.task-ack-timeout | 30000000000 ns | Timeout duration for task
acknowledgment. |
+| optimizer.task-execute-timeout | 3600000000000 ns | Timeout duration for
task execution, default to 1 hour. |
+| overview-cache.max-size | 3360 | Max size of overview cache. |
+| overview-cache.refresh-interval | 180000000000 ns | Interval for refreshing
overview cache. |
+| refresh-external-catalogs.interval | 180000000000 ns | Interval to refresh
the external catalog. |
+| refresh-external-catalogs.queue-size | 1000000 | The queue size of the
executors of the external catalog explorer. |
+| refresh-external-catalogs.thread-count | 10 | The number of threads used for
discovering tables in external catalogs. |
+| refresh-tables.interval | 60000000000 ns | Interval for refreshing table
metadata. |
+| refresh-tables.max-pending-partition-count | 100 | Filters will not be used
beyond that number of partitions. |
+| refresh-tables.thread-count | 10 | The number of threads used for refreshing
tables. |
+| self-optimizing.break-quota-limit-enabled | true | Allow the table to break
the quota limit when the resource is sufficient. |
+| self-optimizing.commit-manifest-io-thread-count | 10 | Sets the size of the
worker pool. The worker pool limits the number of tasks concurrently processing
manifests in the base table implementation across all concurrent commit
operations. |
+| self-optimizing.commit-thread-count | 10 | The number of threads that
self-optimizing uses to submit results. |
+| self-optimizing.plan-manifest-io-thread-count | 10 | Sets the size of the
worker pool. The worker pool limits the number of tasks concurrently processing
manifests in the base table implementation across all concurrent planning
operations. |
+| self-optimizing.refresh-group-interval | 30000000000 ns | Optimizer group
refresh interval. |
+| self-optimizing.runtime-data-expire-interval-hours | 1 | The number of hours
that self-optimizing runtime data expire interval. |
+| self-optimizing.runtime-data-keep-days | 30 | The number of days that
self-optimizing runtime data keeps the runtime. |
+| server-bind-host | 0.0.0.0 | The host bound to the server. |
+| server-expose-host | | The exposed host of the server. |
+| sync-hive-tables.enabled | false | Enable synchronizing Hive tables. |
+| sync-hive-tables.thread-count | 10 | The number of threads used for
synchronizing Hive tables. |
+| table-manifest-io.thread-count | 20 | Sets the size of the worker pool. The
worker pool limits the number of tasks concurrently processing manifests in the
base table implementation across all concurrent planning or commit operations. |
+| terminal.backend | local | Terminal backend implementation. local, kyuubi
and custom are valid values. |
+| terminal.factory | <undefined> | Session factory implement of
terminal, `terminal.backend` must be `custom` if this is set. |
+| terminal.result.limit | 1000 | Row limit of result-set |
+| terminal.sensitive-conf-keys | | Comma-separated list of sensitive conf
keys used to desensitize related value. |
+| terminal.session.timeout | 1800000000000 ns | Session timeout. Default unit
is milliseconds if not specified (** Note: default units are minutes when
version < 0.8). |
+| terminal.stop-on-error | false | When a statement fails to execute, stop
execution or continue executing the remaining statements. |
+| thrift-server.max-message-size | 100 mb | Maximum message size that the
Thrift server can accept. Default unit is bytes if not specified. |
+| thrift-server.optimizing-service.bind-port | 1261 | Port that the optimizing
service thrift server is bound to. |
+| thrift-server.selector-queue-size | 4 | The number of queue size per
selector thread for the Thrift server |
+| thrift-server.selector-thread-count | 2 | The number of selector threads for
the Thrift server. |
+| thrift-server.table-service.bind-port | 1260 | Port that the table service
thrift server is bound to. |
+| thrift-server.table-service.worker-thread-count | 20 | The number of worker
threads for the Thrift server. |
+
+
+## Shade Utils Configuration
+
+The configuration options for Amoro Configuration Shade Utils.
+
+<style>
+table { width: 100%; table-layout: fixed; }
+table th, table td { vertical-align: top; }
+table td:first-child, table th:first-child { word-break: normal; width: 40%; }
+table td:nth-child(2), table th:nth-child(2) { width: 20%; word-break:
break-all; }
+table td:last-child, table th:last-child { width: 40%; word-break: break-all; }
+</style>
+| Key | Default | Description |
+| --- | ------- | ----------- |
+| shade.identifier | default | The identifier of the encryption method for
decryption. Defaults to "default", indicating no encryption |
+| shade.sensitive-keywords | admin-password;database.password | A
semicolon-separated list of keywords for the configuration items to be
decrypted. |
+
+
diff --git a/pom.xml b/pom.xml
index 977595f7d..c0793b807 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1324,6 +1324,9 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>${maven-surefire-plugin.version}</version>
+ <configuration>
+ <failIfNoSpecifiedTests>false</failIfNoSpecifiedTests>
+ </configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
diff --git a/site/amoro-docs/hugo.toml b/site/amoro-docs/hugo.toml
index 85890d9f8..92e414beb 100644
--- a/site/amoro-docs/hugo.toml
+++ b/site/amoro-docs/hugo.toml
@@ -52,6 +52,7 @@ home = [ "HTML", "RSS", "SearchIndex" ]
main = [ # Main menu, docs site left nav
{ name = "Introduction", url = "/", weight = 1},
{ name = "Concepts", weight = 100 },
+ { name = "Configuration", weight = 150 },
{ name = "Admin Guides", weight = 200},
{ name = "User Guides", weight = 300},
{ name = "Formats", weight = 400 },
