This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-kubernetes-operator.git
The following commit(s) were added to refs/heads/main by this push:
new e842e3b [SPARK-49527] Add `ConfOptionDocGenerator` to generate Spark
Operator Config Property Doc
e842e3b is described below
commit e842e3bf8f08c5999d050aac059414424def5fa7
Author: zhou-jiang <[email protected]>
AuthorDate: Wed Sep 11 08:48:18 2024 -0700
[SPARK-49527] Add `ConfOptionDocGenerator` to generate Spark Operator
Config Property Doc
### What changes were proposed in this pull request?
This PR adds a module `docs-utils` to automatically generate config
properties doc page from source.
### Why are the changes needed?
This helps to keep config property docs up-to-date for properties by adding
it as a gradle task.
### Does this PR introduce _any_ user-facing change?
No (doc only, not released)
### How was this patch tested?
Pass the CIs
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #118 from jiangzho/doc_utils.
Lead-authored-by: zhou-jiang <[email protected]>
Co-authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../docs-utils/build.gradle | 26 ++++++-
.../k8s/operator/utils/ConfOptionDocGenerator.java | 87 ++++++++++++++++++++++
.../apache/spark/k8s/operator/utils/DocTable.java | 69 +++++++++++++++++
settings.gradle | 2 +
.../spark/k8s/operator/config/ConfigOption.java | 4 +-
5 files changed, 182 insertions(+), 6 deletions(-)
diff --git a/settings.gradle b/build-tools/docs-utils/build.gradle
similarity index 57%
copy from settings.gradle
copy to build-tools/docs-utils/build.gradle
index 8b2b816..2cdde29 100644
--- a/settings.gradle
+++ b/build-tools/docs-utils/build.gradle
@@ -16,7 +16,25 @@
* specific language governing permissions and limitations
* under the License.
*/
-rootProject.name = 'apache-spark-kubernetes-operator'
-include 'spark-operator-api'
-include 'spark-submission-worker'
-include 'spark-operator'
+
+ext {
+ javaMainClass =
"org.apache.spark.k8s.operator.utils.ConfOptionDocGenerator"
+ docsPath = System.getProperty("user.dir") + "/docs"
+}
+
+dependencies {
+ implementation project(":spark-operator")
+ implementation(libs.log4j.core)
+ implementation(libs.log4j.slf4j.impl)
+ compileOnly(libs.lombok)
+ annotationProcessor(libs.lombok)
+}
+
+test {
+ useJUnitPlatform()
+}
+
+tasks.register('generateConfPropsDoc', Exec) {
+ description = "Generate config properties doc for operator"
+ commandLine "java", "-classpath",
sourceSets.main.runtimeClasspath.getAsPath(), javaMainClass, docsPath
+}
diff --git
a/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/ConfOptionDocGenerator.java
b/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/ConfOptionDocGenerator.java
new file mode 100644
index 0000000..c0f2f49
--- /dev/null
+++
b/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/ConfOptionDocGenerator.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.k8s.operator.utils;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.lang.reflect.Field;
+import java.util.List;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.spark.k8s.operator.config.ConfigOption;
+import org.apache.spark.k8s.operator.config.SparkOperatorConf;
+
+@Slf4j
+public class ConfOptionDocGenerator {
+ public static final String CONF_FILE_NAME = "config_properties.md";
+ public static final String DEFAULT_DOCS_PATH = "docs";
+ public static final String GENERATED_FILE_HEADER =
+ "This doc is automatically generated by gradle task, manual updates
would be overridden.";
+
+ public void generate(String docsPath) throws IOException,
IllegalAccessException {
+ Field[] fields = SparkOperatorConf.class.getDeclaredFields();
+ File docsDir = new File(docsPath);
+ if (!docsDir.exists() && docsDir.mkdir()) {
+ log.info("Creating docs directory at {}", docsPath);
+ }
+ File generated = new File(docsPath, CONF_FILE_NAME);
+ if (generated.createNewFile()) {
+ log.info("Creating props at {}/{}", docsPath, CONF_FILE_NAME);
+ }
+ PrintWriter printWriter = new PrintWriter(generated, "UTF-8");
+ printWriter.println(String.format("[//]: # (%s)", GENERATED_FILE_HEADER));
+ printWriter.println("# Spark Operator Config Properties");
+ DocTable table =
+ DocTable.builder()
+ .headers(List.of("Key", "Type", "Default Value", "Allow Hot
Reloading", "Description"))
+ .columns(5)
+ .build();
+ for (Field f : fields) {
+ if (ConfigOption.class.isAssignableFrom(f.getType())) {
+ ConfigOption<?> conf = (ConfigOption<?>) f.get(this);
+ table.addRow(
+ List.of(
+ conf.getKey(),
+ conf.getTypeParameterClass().getSimpleName(),
+ conf.getDefaultValue().toString(),
+ String.valueOf(conf.isEnableDynamicOverride()),
+ conf.getDescription()));
+ }
+ }
+ table.flush(printWriter);
+ printWriter.close();
+ }
+
+ public static void main(String[] args) throws IOException,
IllegalAccessException {
+ ConfOptionDocGenerator generator = new ConfOptionDocGenerator();
+ String docsPath = DEFAULT_DOCS_PATH;
+ if (args.length > 0) {
+ docsPath = args[0];
+ }
+ try {
+ generator.generate(docsPath);
+ } catch (IOException | IllegalAccessException e) {
+ log.error("Failed to generate docs for config props.", e);
+ throw e;
+ }
+ }
+}
diff --git
a/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/DocTable.java
b/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/DocTable.java
new file mode 100644
index 0000000..73e0597
--- /dev/null
+++
b/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/DocTable.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.k8s.operator.utils;
+
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+
+@Data
+@RequiredArgsConstructor
+@Builder
+public class DocTable {
+ // Separator among cells
+ public static final String ROW_SEPARATOR = " | ";
+ // Separator for header line
+ public static final String HEADER_SEPARATOR = "---";
+ private final List<String> headers;
+ private final int columns;
+ @Builder.Default private final List<List<String>> rows = new ArrayList<>();
+
+ public void addRow(List<String> row) {
+ rows.add(row);
+ }
+
+ public void flush(PrintWriter writer) {
+ writer.println(joinRow(headers));
+ writer.println(joinRow(Collections.nCopies(columns, HEADER_SEPARATOR)));
+ for (List<String> row : rows) {
+ writer.println(joinRow(row));
+ }
+ writer.println();
+ writer.flush();
+ }
+
+ private String joinRow(List<String> elements) {
+ StringBuilder stringBuilder = new StringBuilder(ROW_SEPARATOR);
+ for (String element : elements) {
+ stringBuilder.append(element);
+ stringBuilder.append(ROW_SEPARATOR);
+ }
+ if (elements.size() < columns) {
+ // Append empty cells to end if needed
+ stringBuilder.append(ROW_SEPARATOR.repeat(columns - elements.size()));
+ }
+ return stringBuilder.toString();
+ }
+}
diff --git a/settings.gradle b/settings.gradle
index 8b2b816..bf5289f 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -20,3 +20,5 @@ rootProject.name = 'apache-spark-kubernetes-operator'
include 'spark-operator-api'
include 'spark-submission-worker'
include 'spark-operator'
+include "build-tools-docs-utils"
+project(':build-tools-docs-utils').projectDir = file('build-tools/docs-utils')
diff --git
a/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/ConfigOption.java
b/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/ConfigOption.java
index 83091bd..e0726c4 100644
---
a/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/ConfigOption.java
+++
b/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/ConfigOption.java
@@ -42,8 +42,8 @@ public class ConfigOption<T> {
@Getter @Builder.Default private final boolean enableDynamicOverride = true;
@Getter private String key;
@Getter private String description;
- private T defaultValue;
- private Class<T> typeParameterClass;
+ @Getter private T defaultValue;
+ @Getter private Class<T> typeParameterClass;
public T getValue() {
T resolvedValue = resolveValue();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
