[hotfix][docs] Refactor ConfigOptionsDocGenerator

Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/612e2d83
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/612e2d83
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/612e2d83

Branch: refs/heads/master
Commit: 612e2d83402ba2c3eb68d7a26f1b5f22b283583e
Parents: a926bc6
Author: zentol <ches...@apache.org>
Authored: Fri Feb 16 23:09:54 2018 +0100
Committer: zentol <ches...@apache.org>
Committed: Thu Apr 12 11:58:18 2018 +0200

----------------------------------------------------------------------
 flink-docs/README.md                            |  2 +-
 flink-docs/pom.xml                              | 62 ++++++------
 .../ConfigOptionsDocGenerator.java              | 99 ++++++++++++--------
 .../configuration/OptionsClassLocation.java     | 42 +++++++++
 4 files changed, 134 insertions(+), 71 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/612e2d83/flink-docs/README.md
----------------------------------------------------------------------
diff --git a/flink-docs/README.md b/flink-docs/README.md
index bad5444..61624fa 100644
--- a/flink-docs/README.md
+++ b/flink-docs/README.md
@@ -39,7 +39,7 @@ The documentation must be regenerated whenever
 
 The `ConfigOptionsDocGenerator` can be use to generate a reference of 
`ConfigOptions`. By default, a separate file is generated for each `*Options` 
class found in `org.apache.flink.configuration` and 
`org.apache.flink.yarn.configuration`. The `@ConfigGroups` annotation can be 
used to generate multiple files from a single class.
 
-To integrate an `*Options` class from another package, add another 
module-package argument pair to the `maven-antrun-plugin` configuration in the 
`generate-config-docs` profile.
+To integrate an `*Options` class from another package, add another 
module-package argument pair to `ConfigOptionsDocGenerator#LOCATIONS`.
 
 The files can be generated by running `mvn package -Dgenerate-config-docs -pl 
flink-docs -am -nsu`, and can be integrated into the documentation using `{% 
include generated/<file-name>.html %}`.
 

http://git-wip-us.apache.org/repos/asf/flink/blob/612e2d83/flink-docs/pom.xml
----------------------------------------------------------------------
diff --git a/flink-docs/pom.xml b/flink-docs/pom.xml
index b06e587..f98de57 100644
--- a/flink-docs/pom.xml
+++ b/flink-docs/pom.xml
@@ -118,6 +118,15 @@ under the License.
                                </executions>
                        </plugin>
                </plugins>
+
+               <pluginManagement>
+                       <plugins>
+                               <plugin>
+                                       
<artifactId>maven-antrun-plugin</artifactId>
+                                       <version>1.7</version>
+                               </plugin>
+                       </plugins>
+               </pluginManagement>
        </build>
 
        <profiles>
@@ -132,24 +141,24 @@ under the License.
                                <plugins>
                                        <plugin>
                                                
<artifactId>maven-antrun-plugin</artifactId>
-                                               <version>1.7</version>
                                                <executions>
                                                        <execution>
+                                                               
<id>generate-rest-docs</id>
                                                                
<phase>package</phase>
                                                                <goals>
                                                                        
<goal>run</goal>
                                                                </goals>
+                                                               <configuration>
+                                                                       <target>
+                                                                               
<mkdir dir="${rootDir}/${generated.docs.dir}"/>
+                                                                               
<java classname="org.apache.flink.docs.rest.RestAPIDocGenerator" fork="true" 
failonerror="true">
+                                                                               
        <classpath refid="maven.compile.classpath"/>
+                                                                               
        <arg value="${rootDir}/${generated.docs.dir}/"/>
+                                                                               
</java>
+                                                                       
</target>
+                                                               </configuration>
                                                        </execution>
                                                </executions>
-                                               <configuration>
-                                                       <target>
-                                                               <mkdir 
dir="${rootDir}/${generated.docs.dir}"/>
-                                                               <java 
classname="org.apache.flink.docs.rest.RestAPIDocGenerator" fork="true">
-                                                                       
<classpath refid="maven.compile.classpath"/>
-                                                                       <arg 
value="${rootDir}/${generated.docs.dir}/"/>
-                                                               </java>
-                                                       </target>
-                                               </configuration>
                                        </plugin>
                                </plugins>
                        </build>
@@ -165,36 +174,25 @@ under the License.
                                <plugins>
                                        <plugin>
                                                
<artifactId>maven-antrun-plugin</artifactId>
-                                               <version>1.7</version>
                                                <executions>
                                                        <execution>
+                                                               
<id>generate-config-docs</id>
                                                                
<phase>package</phase>
                                                                <goals>
                                                                        
<goal>run</goal>
                                                                </goals>
+                                                               <configuration>
+                                                                       <target>
+                                                                               
<mkdir dir="${rootDir}/${generated.docs.dir}"/>
+                                                                               
<java classname="org.apache.flink.docs.configuration.ConfigOptionsDocGenerator" 
fork="true" failonerror="true">
+                                                                               
        <classpath refid="maven.compile.classpath" />
+                                                                               
        <arg value="${rootDir}/${generated.docs.dir}/" />
+                                                                               
        <arg value="${rootDir}" />
+                                                                               
</java>
+                                                                       
</target>
+                                                               </configuration>
                                                        </execution>
                                                </executions>
-                                               <configuration>
-                                                       <target>
-                                                               <mkdir 
dir="${rootDir}/${generated.docs.dir}"/>
-                                                               <java 
classname="org.apache.flink.docs.configuration.ConfigOptionsDocGenerator" 
fork="true">
-                                                                       
<classpath refid="maven.compile.classpath" />
-                                                                       <arg 
value="${rootDir}/${generated.docs.dir}/" />
-                                                                       <arg 
value="${rootDir}" />
-                                                                       
<!--packages with configuration classes-->
-                                                                       <arg 
value="flink-core" />
-                                                                       <arg 
value="org.apache.flink.configuration" />
-                                                                       <arg 
value="flink-runtime" />
-                                                                       <arg 
value="org.apache.flink.runtime.io.network.netty" />
-                                                                       <arg 
value="flink-yarn" />
-                                                                       <arg 
value="org.apache.flink.yarn.configuration" />
-                                                                       <arg 
value="flink-mesos" />
-                                                                       <arg 
value="org.apache.flink.mesos.configuration" />
-                                                                       <arg 
value="flink-mesos" />
-                                                                       <arg 
value="org.apache.flink.mesos.runtime.clusterframework" />
-                                                               </java>
-                                                       </target>
-                                               </configuration>
                                        </plugin>
                                </plugins>
                        </build>

http://git-wip-us.apache.org/repos/asf/flink/blob/612e2d83/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java
----------------------------------------------------------------------
diff --git 
a/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java
 
b/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java
index 3bec09c..6256a67 100644
--- 
a/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java
+++ 
b/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java
@@ -25,6 +25,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.configuration.ConfigOption;
 import org.apache.flink.configuration.CoreOptions;
 import org.apache.flink.configuration.WebOptions;
+import org.apache.flink.util.function.ThrowingConsumer;
 
 import java.io.IOException;
 import java.lang.reflect.Field;
@@ -35,6 +36,7 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
@@ -47,6 +49,18 @@ import java.util.regex.Pattern;
  */
 public class ConfigOptionsDocGenerator {
 
+       private static final OptionsClassLocation[] LOCATIONS = new 
OptionsClassLocation[]{
+               new OptionsClassLocation("flink-core", 
"org.apache.flink.configuration"),
+               new OptionsClassLocation("flink-runtime", 
"org.apache.flink.runtime.io.network.netty"),
+               new OptionsClassLocation("flink-yarn", 
"org.apache.flink.yarn.configuration"),
+               new OptionsClassLocation("flink-mesos", 
"org.apache.flink.mesos.configuration"),
+               new OptionsClassLocation("flink-mesos", 
"org.apache.flink.mesos.runtime.clusterframework"),
+       };
+
+       private static final String CLASS_NAME_GROUP = "className";
+       private static final String CLASS_PREFIX_GROUP = "classPrefix";
+       private static final Pattern CLASS_NAME_PATTERN = Pattern.compile("(?<" 
+ CLASS_NAME_GROUP + ">(?<" + CLASS_PREFIX_GROUP + 
">[a-zA-Z]*)(?:Options|Config|Parameters))(?:\\.java)?");
+
        /**
         * This method generates html tables from set of classes containing 
{@link ConfigOption ConfigOptions}.
         *
@@ -57,39 +71,47 @@ public class ConfigOptionsDocGenerator {
         * @param args
         *  [0] output directory for the generated files
         *  [1] project root directory
-        *  [x] module containing an *Options class
-        *  [x+1] package to the * Options.classes
         */
        public static void main(String[] args) throws IOException, 
ClassNotFoundException {
                String outputDirectory = args[0];
                String rootDir = args[1];
-               for (int x = 2; x + 1 < args.length; x += 2) {
-                       createTable(rootDir, args[x], args[x + 1], 
outputDirectory);
+
+               for (OptionsClassLocation location : LOCATIONS) {
+                       createTable(rootDir, location.getModule(), 
location.getPackage(), outputDirectory);
                }
        }
 
        private static void createTable(String rootDir, String module, String 
packageName, String outputDirectory) throws IOException, ClassNotFoundException 
{
+               processConfigOptions(rootDir, module, packageName, optionsClass 
-> {
+                       List<Tuple2<ConfigGroup, String>> tables = 
generateTablesForClass(optionsClass);
+                       for (Tuple2<ConfigGroup, String> group : tables) {
+                               String name;
+                               if (group.f0 == null) {
+                                       Matcher matcher = 
CLASS_NAME_PATTERN.matcher(optionsClass.getSimpleName());
+                                       if (!matcher.matches()) {
+                                               throw new 
RuntimeException("Pattern did not match for " + optionsClass.getSimpleName() + 
'.');
+                                       }
+                                       name = 
matcher.group(CLASS_PREFIX_GROUP).replaceAll("(.)(\\p{Upper})", 
"$1_$2").toLowerCase();
+                               } else {
+                                       name = 
group.f0.name().replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase();
+                               }
+
+                               String outputFile = name + 
"_configuration.html";
+                               Files.write(Paths.get(outputDirectory, 
outputFile), group.f1.getBytes(StandardCharsets.UTF_8));
+                       }
+               });
+       }
+
+       private static void processConfigOptions(String rootDir, String module, 
String packageName, ThrowingConsumer<Class<?>, IOException> classConsumer) 
throws IOException, ClassNotFoundException {
                Path configDir = Paths.get(rootDir, module, "src/main/java", 
packageName.replaceAll("\\.", "/"));
 
-               Pattern p = 
Pattern.compile("(([a-zA-Z]*)(Options|Config|Parameters))\\.java");
                try (DirectoryStream<Path> stream = 
Files.newDirectoryStream(configDir)) {
                        for (Path entry : stream) {
                                String fileName = 
entry.getFileName().toString();
-                               Matcher matcher = p.matcher(fileName);
+                               Matcher matcher = 
CLASS_NAME_PATTERN.matcher(fileName);
                                if (!fileName.equals("ConfigOptions.java") && 
matcher.matches()) {
-                                       Class<?> optionsClass = 
Class.forName(packageName + "." + matcher.group(1));
-                                       List<Tuple2<ConfigGroup, String>> 
tables = generateTablesForClass(optionsClass);
-                                       if (tables.size() > 0) {
-                                               for (Tuple2<ConfigGroup, 
String> group : tables) {
-
-                                                       String name = group.f0 
== null
-                                                               ? 
matcher.group(2).replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase()
-                                                               : 
group.f0.name().replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase();
-
-                                                       String outputFile = 
name + "_configuration.html";
-                                                       
Files.write(Paths.get(outputDirectory, outputFile), 
group.f1.getBytes(StandardCharsets.UTF_8));
-                                               }
-                                       }
+                                       Class<?> optionsClass = 
Class.forName(packageName + '.' + matcher.group(CLASS_NAME_GROUP));
+                                       classConsumer.accept(optionsClass);
                                }
                        }
                }
@@ -98,40 +120,41 @@ public class ConfigOptionsDocGenerator {
        @VisibleForTesting
        static List<Tuple2<ConfigGroup, String>> 
generateTablesForClass(Class<?> optionsClass) {
                ConfigGroups configGroups = 
optionsClass.getAnnotation(ConfigGroups.class);
-               List<Tuple2<ConfigGroup, String>> tables = new ArrayList<>();
-               List<ConfigOption> allOptions = 
extractConfigOptions(optionsClass);
+               List<ConfigOption<?>> allOptions = 
extractConfigOptions(optionsClass);
 
+               List<Tuple2<ConfigGroup, String>> tables;
                if (configGroups != null) {
+                       tables = new ArrayList<>(configGroups.groups().length + 
1);
                        Tree tree = new Tree(configGroups.groups(), allOptions);
 
                        for (ConfigGroup group : configGroups.groups()) {
-                               List<ConfigOption> configOptions = 
tree.findConfigOptions(group);
+                               List<ConfigOption<?>> configOptions = 
tree.findConfigOptions(group);
                                sortOptions(configOptions);
                                tables.add(Tuple2.of(group, 
toHtmlTable(configOptions)));
                        }
-                       List<ConfigOption> configOptions = 
tree.getDefaultOptions();
+                       List<ConfigOption<?>> configOptions = 
tree.getDefaultOptions();
                        sortOptions(configOptions);
                        tables.add(Tuple2.of(null, toHtmlTable(configOptions)));
                } else {
                        sortOptions(allOptions);
-                       tables.add(Tuple2.of(null, toHtmlTable(allOptions)));
+                       tables = Collections.singletonList(Tuple2.of(null, 
toHtmlTable(allOptions)));
                }
                return tables;
        }
 
-       private static List<ConfigOption> extractConfigOptions(Class<?> clazz) {
+       private static List<ConfigOption<?>> extractConfigOptions(Class<?> 
clazz) {
                try {
-                       List<ConfigOption> configOptions = new ArrayList<>();
+                       List<ConfigOption<?>> configOptions = new 
ArrayList<>(8);
                        Field[] fields = clazz.getFields();
                        for (Field field : fields) {
                                if (field.getType().equals(ConfigOption.class) 
&& field.getAnnotation(Deprecated.class) == null) {
-                                       configOptions.add((ConfigOption) 
field.get(null));
+                                       configOptions.add((ConfigOption<?>) 
field.get(null));
                                }
                        }
 
                        return configOptions;
                } catch (Exception e) {
-                       throw new RuntimeException("Failed to extract config 
options from class " + clazz + ".", e);
+                       throw new RuntimeException("Failed to extract config 
options from class " + clazz + '.', e);
                }
        }
 
@@ -142,7 +165,7 @@ public class ConfigOptionsDocGenerator {
         * @param options list of options to include in this group
         * @return string containing HTML formatted table
         */
-       private static String toHtmlTable(final List<ConfigOption> options) {
+       private static String toHtmlTable(final List<ConfigOption<?>> options) {
                StringBuilder htmlTable = new StringBuilder();
                htmlTable.append("<table class=\"table table-bordered\">\n");
                htmlTable.append("    <thead>\n");
@@ -154,7 +177,7 @@ public class ConfigOptionsDocGenerator {
                htmlTable.append("    </thead>\n");
                htmlTable.append("    <tbody>\n");
 
-               for (ConfigOption option : options) {
+               for (ConfigOption<?> option : options) {
                        htmlTable.append(toHtmlString(option));
                }
 
@@ -203,7 +226,7 @@ public class ConfigOptionsDocGenerator {
                        .replaceAll(">", "&#62;");
        }
 
-       private static void sortOptions(List<ConfigOption> configOptions) {
+       private static void sortOptions(List<ConfigOption<?>> configOptions) {
                configOptions.sort(Comparator.comparing(ConfigOption::key));
        }
 
@@ -214,7 +237,7 @@ public class ConfigOptionsDocGenerator {
        private static class Tree {
                private final Node root = new Node();
 
-               Tree(ConfigGroup[] groups, Collection<ConfigOption> options) {
+               Tree(ConfigGroup[] groups, Collection<ConfigOption<?>> options) 
{
                        // generate a tree based on all key prefixes
                        for (ConfigGroup group : groups) {
                                String[] keyComponents = 
group.keyPrefix().split("\\.");
@@ -232,12 +255,12 @@ public class ConfigOptionsDocGenerator {
                        }
                }
 
-               List<ConfigOption> findConfigOptions(ConfigGroup configGroup) {
+               List<ConfigOption<?>> findConfigOptions(ConfigGroup 
configGroup) {
                        Node groupRoot = findGroupRoot(configGroup.keyPrefix());
                        return groupRoot.getConfigOptions();
                }
 
-               List<ConfigOption> getDefaultOptions() {
+               List<ConfigOption<?>> getDefaultOptions() {
                        return root.getConfigOptions();
                }
 
@@ -251,8 +274,8 @@ public class ConfigOptionsDocGenerator {
                }
 
                private static class Node {
-                       private final List<ConfigOption> configOptions = new 
ArrayList<>();
-                       private final Map<String, Node> children = new 
HashMap<>();
+                       private final List<ConfigOption<?>> configOptions = new 
ArrayList<>(8);
+                       private final Map<String, Node> children = new 
HashMap<>(8);
                        private boolean isGroupRoot = false;
 
                        private Node addChild(String keyComponent) {
@@ -272,7 +295,7 @@ public class ConfigOptionsDocGenerator {
                                return child;
                        }
 
-                       private void assignOption(ConfigOption option) {
+                       private void assignOption(ConfigOption<?> option) {
                                configOptions.add(option);
                        }
 
@@ -284,7 +307,7 @@ public class ConfigOptionsDocGenerator {
                                this.isGroupRoot = true;
                        }
 
-                       private List<ConfigOption> getConfigOptions() {
+                       private List<ConfigOption<?>> getConfigOptions() {
                                return configOptions;
                        }
                }

http://git-wip-us.apache.org/repos/asf/flink/blob/612e2d83/flink-docs/src/main/java/org/apache/flink/docs/configuration/OptionsClassLocation.java
----------------------------------------------------------------------
diff --git 
a/flink-docs/src/main/java/org/apache/flink/docs/configuration/OptionsClassLocation.java
 
b/flink-docs/src/main/java/org/apache/flink/docs/configuration/OptionsClassLocation.java
new file mode 100644
index 0000000..2fc11a9
--- /dev/null
+++ 
b/flink-docs/src/main/java/org/apache/flink/docs/configuration/OptionsClassLocation.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.docs.configuration;
+
+import org.apache.flink.configuration.ConfigOption;
+
+/**
+ * Simple descriptor for the location of a class containing {@link 
ConfigOption ConfigOptions}.
+ */
+class OptionsClassLocation {
+       private final String module;
+       private final String pckg;
+
+       OptionsClassLocation(String module, String pckg) {
+               this.module = module;
+               this.pckg = pckg;
+       }
+
+       public String getModule() {
+               return module;
+       }
+
+       public String getPackage() {
+               return pckg;
+       }
+}

Reply via email to