[hotfix][docs] Refactor ConfigOptionsDocGenerator
Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1f21af6f Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1f21af6f Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1f21af6f Branch: refs/heads/release-1.5 Commit: 1f21af6fd7c1c67015fad3b7a35d38d1b5b1a982 Parents: 21a66ea Author: zentol <ches...@apache.org> Authored: Fri Feb 16 23:09:54 2018 +0100 Committer: zentol <ches...@apache.org> Committed: Thu Apr 12 11:58:20 2018 +0200 ---------------------------------------------------------------------- flink-docs/README.md | 2 +- flink-docs/pom.xml | 62 ++++++------ .../ConfigOptionsDocGenerator.java | 99 ++++++++++++-------- .../configuration/OptionsClassLocation.java | 42 +++++++++ 4 files changed, 134 insertions(+), 71 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/1f21af6f/flink-docs/README.md ---------------------------------------------------------------------- diff --git a/flink-docs/README.md b/flink-docs/README.md index bad5444..61624fa 100644 --- a/flink-docs/README.md +++ b/flink-docs/README.md @@ -39,7 +39,7 @@ The documentation must be regenerated whenever The `ConfigOptionsDocGenerator` can be use to generate a reference of `ConfigOptions`. By default, a separate file is generated for each `*Options` class found in `org.apache.flink.configuration` and `org.apache.flink.yarn.configuration`. The `@ConfigGroups` annotation can be used to generate multiple files from a single class. -To integrate an `*Options` class from another package, add another module-package argument pair to the `maven-antrun-plugin` configuration in the `generate-config-docs` profile. +To integrate an `*Options` class from another package, add another module-package argument pair to `ConfigOptionsDocGenerator#LOCATIONS`. The files can be generated by running `mvn package -Dgenerate-config-docs -pl flink-docs -am -nsu`, and can be integrated into the documentation using `{% include generated/<file-name>.html %}`. http://git-wip-us.apache.org/repos/asf/flink/blob/1f21af6f/flink-docs/pom.xml ---------------------------------------------------------------------- diff --git a/flink-docs/pom.xml b/flink-docs/pom.xml index ad53007..c8747a5 100644 --- a/flink-docs/pom.xml +++ b/flink-docs/pom.xml @@ -118,6 +118,15 @@ under the License. </executions> </plugin> </plugins> + + <pluginManagement> + <plugins> + <plugin> + <artifactId>maven-antrun-plugin</artifactId> + <version>1.7</version> + </plugin> + </plugins> + </pluginManagement> </build> <profiles> @@ -132,24 +141,24 @@ under the License. <plugins> <plugin> <artifactId>maven-antrun-plugin</artifactId> - <version>1.7</version> <executions> <execution> + <id>generate-rest-docs</id> <phase>package</phase> <goals> <goal>run</goal> </goals> + <configuration> + <target> + <mkdir dir="${rootDir}/${generated.docs.dir}"/> + <java classname="org.apache.flink.docs.rest.RestAPIDocGenerator" fork="true" failonerror="true"> + <classpath refid="maven.compile.classpath"/> + <arg value="${rootDir}/${generated.docs.dir}/"/> + </java> + </target> + </configuration> </execution> </executions> - <configuration> - <target> - <mkdir dir="${rootDir}/${generated.docs.dir}"/> - <java classname="org.apache.flink.docs.rest.RestAPIDocGenerator" fork="true"> - <classpath refid="maven.compile.classpath"/> - <arg value="${rootDir}/${generated.docs.dir}/"/> - </java> - </target> - </configuration> </plugin> </plugins> </build> @@ -165,36 +174,25 @@ under the License. <plugins> <plugin> <artifactId>maven-antrun-plugin</artifactId> - <version>1.7</version> <executions> <execution> + <id>generate-config-docs</id> <phase>package</phase> <goals> <goal>run</goal> </goals> + <configuration> + <target> + <mkdir dir="${rootDir}/${generated.docs.dir}"/> + <java classname="org.apache.flink.docs.configuration.ConfigOptionsDocGenerator" fork="true" failonerror="true"> + <classpath refid="maven.compile.classpath" /> + <arg value="${rootDir}/${generated.docs.dir}/" /> + <arg value="${rootDir}" /> + </java> + </target> + </configuration> </execution> </executions> - <configuration> - <target> - <mkdir dir="${rootDir}/${generated.docs.dir}"/> - <java classname="org.apache.flink.docs.configuration.ConfigOptionsDocGenerator" fork="true"> - <classpath refid="maven.compile.classpath" /> - <arg value="${rootDir}/${generated.docs.dir}/" /> - <arg value="${rootDir}" /> - <!--packages with configuration classes--> - <arg value="flink-core" /> - <arg value="org.apache.flink.configuration" /> - <arg value="flink-runtime" /> - <arg value="org.apache.flink.runtime.io.network.netty" /> - <arg value="flink-yarn" /> - <arg value="org.apache.flink.yarn.configuration" /> - <arg value="flink-mesos" /> - <arg value="org.apache.flink.mesos.configuration" /> - <arg value="flink-mesos" /> - <arg value="org.apache.flink.mesos.runtime.clusterframework" /> - </java> - </target> - </configuration> </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/flink/blob/1f21af6f/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java ---------------------------------------------------------------------- diff --git a/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java b/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java index 3bec09c..6256a67 100644 --- a/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java +++ b/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java @@ -25,6 +25,7 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.CoreOptions; import org.apache.flink.configuration.WebOptions; +import org.apache.flink.util.function.ThrowingConsumer; import java.io.IOException; import java.lang.reflect.Field; @@ -35,6 +36,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -47,6 +49,18 @@ import java.util.regex.Pattern; */ public class ConfigOptionsDocGenerator { + private static final OptionsClassLocation[] LOCATIONS = new OptionsClassLocation[]{ + new OptionsClassLocation("flink-core", "org.apache.flink.configuration"), + new OptionsClassLocation("flink-runtime", "org.apache.flink.runtime.io.network.netty"), + new OptionsClassLocation("flink-yarn", "org.apache.flink.yarn.configuration"), + new OptionsClassLocation("flink-mesos", "org.apache.flink.mesos.configuration"), + new OptionsClassLocation("flink-mesos", "org.apache.flink.mesos.runtime.clusterframework"), + }; + + private static final String CLASS_NAME_GROUP = "className"; + private static final String CLASS_PREFIX_GROUP = "classPrefix"; + private static final Pattern CLASS_NAME_PATTERN = Pattern.compile("(?<" + CLASS_NAME_GROUP + ">(?<" + CLASS_PREFIX_GROUP + ">[a-zA-Z]*)(?:Options|Config|Parameters))(?:\\.java)?"); + /** * This method generates html tables from set of classes containing {@link ConfigOption ConfigOptions}. * @@ -57,39 +71,47 @@ public class ConfigOptionsDocGenerator { * @param args * [0] output directory for the generated files * [1] project root directory - * [x] module containing an *Options class - * [x+1] package to the * Options.classes */ public static void main(String[] args) throws IOException, ClassNotFoundException { String outputDirectory = args[0]; String rootDir = args[1]; - for (int x = 2; x + 1 < args.length; x += 2) { - createTable(rootDir, args[x], args[x + 1], outputDirectory); + + for (OptionsClassLocation location : LOCATIONS) { + createTable(rootDir, location.getModule(), location.getPackage(), outputDirectory); } } private static void createTable(String rootDir, String module, String packageName, String outputDirectory) throws IOException, ClassNotFoundException { + processConfigOptions(rootDir, module, packageName, optionsClass -> { + List<Tuple2<ConfigGroup, String>> tables = generateTablesForClass(optionsClass); + for (Tuple2<ConfigGroup, String> group : tables) { + String name; + if (group.f0 == null) { + Matcher matcher = CLASS_NAME_PATTERN.matcher(optionsClass.getSimpleName()); + if (!matcher.matches()) { + throw new RuntimeException("Pattern did not match for " + optionsClass.getSimpleName() + '.'); + } + name = matcher.group(CLASS_PREFIX_GROUP).replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase(); + } else { + name = group.f0.name().replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase(); + } + + String outputFile = name + "_configuration.html"; + Files.write(Paths.get(outputDirectory, outputFile), group.f1.getBytes(StandardCharsets.UTF_8)); + } + }); + } + + private static void processConfigOptions(String rootDir, String module, String packageName, ThrowingConsumer<Class<?>, IOException> classConsumer) throws IOException, ClassNotFoundException { Path configDir = Paths.get(rootDir, module, "src/main/java", packageName.replaceAll("\\.", "/")); - Pattern p = Pattern.compile("(([a-zA-Z]*)(Options|Config|Parameters))\\.java"); try (DirectoryStream<Path> stream = Files.newDirectoryStream(configDir)) { for (Path entry : stream) { String fileName = entry.getFileName().toString(); - Matcher matcher = p.matcher(fileName); + Matcher matcher = CLASS_NAME_PATTERN.matcher(fileName); if (!fileName.equals("ConfigOptions.java") && matcher.matches()) { - Class<?> optionsClass = Class.forName(packageName + "." + matcher.group(1)); - List<Tuple2<ConfigGroup, String>> tables = generateTablesForClass(optionsClass); - if (tables.size() > 0) { - for (Tuple2<ConfigGroup, String> group : tables) { - - String name = group.f0 == null - ? matcher.group(2).replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase() - : group.f0.name().replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase(); - - String outputFile = name + "_configuration.html"; - Files.write(Paths.get(outputDirectory, outputFile), group.f1.getBytes(StandardCharsets.UTF_8)); - } - } + Class<?> optionsClass = Class.forName(packageName + '.' + matcher.group(CLASS_NAME_GROUP)); + classConsumer.accept(optionsClass); } } } @@ -98,40 +120,41 @@ public class ConfigOptionsDocGenerator { @VisibleForTesting static List<Tuple2<ConfigGroup, String>> generateTablesForClass(Class<?> optionsClass) { ConfigGroups configGroups = optionsClass.getAnnotation(ConfigGroups.class); - List<Tuple2<ConfigGroup, String>> tables = new ArrayList<>(); - List<ConfigOption> allOptions = extractConfigOptions(optionsClass); + List<ConfigOption<?>> allOptions = extractConfigOptions(optionsClass); + List<Tuple2<ConfigGroup, String>> tables; if (configGroups != null) { + tables = new ArrayList<>(configGroups.groups().length + 1); Tree tree = new Tree(configGroups.groups(), allOptions); for (ConfigGroup group : configGroups.groups()) { - List<ConfigOption> configOptions = tree.findConfigOptions(group); + List<ConfigOption<?>> configOptions = tree.findConfigOptions(group); sortOptions(configOptions); tables.add(Tuple2.of(group, toHtmlTable(configOptions))); } - List<ConfigOption> configOptions = tree.getDefaultOptions(); + List<ConfigOption<?>> configOptions = tree.getDefaultOptions(); sortOptions(configOptions); tables.add(Tuple2.of(null, toHtmlTable(configOptions))); } else { sortOptions(allOptions); - tables.add(Tuple2.of(null, toHtmlTable(allOptions))); + tables = Collections.singletonList(Tuple2.of(null, toHtmlTable(allOptions))); } return tables; } - private static List<ConfigOption> extractConfigOptions(Class<?> clazz) { + private static List<ConfigOption<?>> extractConfigOptions(Class<?> clazz) { try { - List<ConfigOption> configOptions = new ArrayList<>(); + List<ConfigOption<?>> configOptions = new ArrayList<>(8); Field[] fields = clazz.getFields(); for (Field field : fields) { if (field.getType().equals(ConfigOption.class) && field.getAnnotation(Deprecated.class) == null) { - configOptions.add((ConfigOption) field.get(null)); + configOptions.add((ConfigOption<?>) field.get(null)); } } return configOptions; } catch (Exception e) { - throw new RuntimeException("Failed to extract config options from class " + clazz + ".", e); + throw new RuntimeException("Failed to extract config options from class " + clazz + '.', e); } } @@ -142,7 +165,7 @@ public class ConfigOptionsDocGenerator { * @param options list of options to include in this group * @return string containing HTML formatted table */ - private static String toHtmlTable(final List<ConfigOption> options) { + private static String toHtmlTable(final List<ConfigOption<?>> options) { StringBuilder htmlTable = new StringBuilder(); htmlTable.append("<table class=\"table table-bordered\">\n"); htmlTable.append(" <thead>\n"); @@ -154,7 +177,7 @@ public class ConfigOptionsDocGenerator { htmlTable.append(" </thead>\n"); htmlTable.append(" <tbody>\n"); - for (ConfigOption option : options) { + for (ConfigOption<?> option : options) { htmlTable.append(toHtmlString(option)); } @@ -203,7 +226,7 @@ public class ConfigOptionsDocGenerator { .replaceAll(">", ">"); } - private static void sortOptions(List<ConfigOption> configOptions) { + private static void sortOptions(List<ConfigOption<?>> configOptions) { configOptions.sort(Comparator.comparing(ConfigOption::key)); } @@ -214,7 +237,7 @@ public class ConfigOptionsDocGenerator { private static class Tree { private final Node root = new Node(); - Tree(ConfigGroup[] groups, Collection<ConfigOption> options) { + Tree(ConfigGroup[] groups, Collection<ConfigOption<?>> options) { // generate a tree based on all key prefixes for (ConfigGroup group : groups) { String[] keyComponents = group.keyPrefix().split("\\."); @@ -232,12 +255,12 @@ public class ConfigOptionsDocGenerator { } } - List<ConfigOption> findConfigOptions(ConfigGroup configGroup) { + List<ConfigOption<?>> findConfigOptions(ConfigGroup configGroup) { Node groupRoot = findGroupRoot(configGroup.keyPrefix()); return groupRoot.getConfigOptions(); } - List<ConfigOption> getDefaultOptions() { + List<ConfigOption<?>> getDefaultOptions() { return root.getConfigOptions(); } @@ -251,8 +274,8 @@ public class ConfigOptionsDocGenerator { } private static class Node { - private final List<ConfigOption> configOptions = new ArrayList<>(); - private final Map<String, Node> children = new HashMap<>(); + private final List<ConfigOption<?>> configOptions = new ArrayList<>(8); + private final Map<String, Node> children = new HashMap<>(8); private boolean isGroupRoot = false; private Node addChild(String keyComponent) { @@ -272,7 +295,7 @@ public class ConfigOptionsDocGenerator { return child; } - private void assignOption(ConfigOption option) { + private void assignOption(ConfigOption<?> option) { configOptions.add(option); } @@ -284,7 +307,7 @@ public class ConfigOptionsDocGenerator { this.isGroupRoot = true; } - private List<ConfigOption> getConfigOptions() { + private List<ConfigOption<?>> getConfigOptions() { return configOptions; } } http://git-wip-us.apache.org/repos/asf/flink/blob/1f21af6f/flink-docs/src/main/java/org/apache/flink/docs/configuration/OptionsClassLocation.java ---------------------------------------------------------------------- diff --git a/flink-docs/src/main/java/org/apache/flink/docs/configuration/OptionsClassLocation.java b/flink-docs/src/main/java/org/apache/flink/docs/configuration/OptionsClassLocation.java new file mode 100644 index 0000000..2fc11a9 --- /dev/null +++ b/flink-docs/src/main/java/org/apache/flink/docs/configuration/OptionsClassLocation.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.docs.configuration; + +import org.apache.flink.configuration.ConfigOption; + +/** + * Simple descriptor for the location of a class containing {@link ConfigOption ConfigOptions}. + */ +class OptionsClassLocation { + private final String module; + private final String pckg; + + OptionsClassLocation(String module, String pckg) { + this.module = module; + this.pckg = pckg; + } + + public String getModule() { + return module; + } + + public String getPackage() { + return pckg; + } +}
