TEZ-2294. Add tez-site-template.xml with description of config properties. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/50df8651 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/50df8651 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/50df8651 Branch: refs/heads/master Commit: 50df8651f27d8bc8d463cf8c8cd70ceea3e6af4f Parents: 5cb0079 Author: Hitesh Shah <[email protected]> Authored: Tue Sep 1 13:55:16 2015 -0700 Committer: Hitesh Shah <[email protected]> Committed: Tue Sep 1 13:55:41 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 1 + docs/pom.xml | 1 + pom.xml | 61 +++-- tez-api/pom.xml | 33 ++- .../common/annotation/ConfigurationClass.java | 31 +++ .../annotation/ConfigurationProperty.java | 36 +++ .../apache/tez/dag/api/TezConfiguration.java | 126 +++++++++- tez-dist/pom.xml | 5 + tez-runtime-library/pom.xml | 32 +++ .../library/api/TezRuntimeConfiguration.java | 67 ++++-- tez-tools/pom.xml | 2 + .../tez-javadoc-tools/findbugs-exclude.xml | 22 ++ tez-tools/tez-javadoc-tools/pom.xml | 92 ++++++++ .../javadoc/doclet/ConfigStandardDoclet.java | 233 +++++++++++++++++++ .../apache/tez/tools/javadoc/model/Config.java | 40 ++++ .../tez/tools/javadoc/model/ConfigProperty.java | 46 ++++ .../tez/tools/javadoc/util/HtmlWriter.java | 161 +++++++++++++ .../apache/tez/tools/javadoc/util/Writer.java | 39 ++++ .../tez/tools/javadoc/util/XmlWriter.java | 96 ++++++++ 19 files changed, 1083 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 66fe000..3c7d0bd 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ Release 0.8.1: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-2294. Add tez-site-template.xml with description of config properties. TEZ-2757. Fix download links for Tez releases. TEZ-2742. VertexImpl.finished() terminationCause hides member var of the same name http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/docs/pom.xml ---------------------------------------------------------------------- diff --git a/docs/pom.xml b/docs/pom.xml index b76f9ba..df6f3ef 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -444,6 +444,7 @@ <artifactId>maven-project-info-reports-plugin</artifactId> <configuration> <customBundle>${project.basedir}/src/site/custom/project-info-report.properties</customBundle> + <dependencyLocationsEnabled>false</dependencyLocationsEnabled> </configuration> <reportSets> <reportSet> http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index fe996d0..d1d5fe6 100644 --- a/pom.xml +++ b/pom.xml @@ -160,6 +160,11 @@ </dependency> <dependency> <groupId>org.apache.tez</groupId> + <artifactId>tez-tools</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> <artifactId>tez-plugins</artifactId> <version>${project.version}</version> </dependency> @@ -885,6 +890,14 @@ </lifecycleMappingMetadata> </configuration> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-project-info-reports-plugin</artifactId> + <version>2.7</version> + <configuration> + <dependencyLocationsEnabled>false</dependencyLocationsEnabled> + </configuration> + </plugin> </plugins> </pluginManagement> @@ -1134,6 +1147,37 @@ </build> </profile> <profile> + <id>jdk1.7</id> + <activation> + <jdk>1.7</jdk> + </activation> + <dependencies> + <dependency> + <groupId>jdk.tools</groupId> + <artifactId>jdk.tools</artifactId> + <version>1.7</version> + <scope>system</scope> + <systemPath>${java.home}/../lib/tools.jar</systemPath> + </dependency> + </dependencies> + </profile> + <profile> + <id>jdk1.8</id> + <activation> + <jdk>1.8</jdk> + </activation> + <dependencies> + <dependency> + <groupId>jdk.tools</groupId> + <artifactId>jdk.tools</artifactId> + <version>1.8</version> + <scope>system</scope> + <systemPath>${java.home}/../lib/tools.jar</systemPath> + </dependency> + </dependencies> + </profile> + + <profile> <activation> <activeByDefault>false</activeByDefault> </activation> @@ -1187,21 +1231,12 @@ <frontend-maven-plugin.version>0.0.22</frontend-maven-plugin.version> </properties> </profile> - </profiles> <reporting> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-project-info-reports-plugin</artifactId> - <version>2.7</version> - <configuration> - <dependencyLocationsEnabled>false</dependencyLocationsEnabled> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-javadoc-plugin</artifactId> <version>${javadoc-maven-plugin.version}</version> <reportSets> @@ -1224,14 +1259,6 @@ </reportSet> </reportSets> </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>findbugs-maven-plugin</artifactId> - <version>${findbugs-maven-plugin.version}</version> - <configuration> - <excludeFilterFile>${basedir}/findbugs-exclude.xml</excludeFilterFile> - </configuration> - </plugin> </plugins> </reporting> http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-api/pom.xml ---------------------------------------------------------------------- diff --git a/tez-api/pom.xml b/tez-api/pom.xml index 1baec8f..b3eb97c 100644 --- a/tez-api/pom.xml +++ b/tez-api/pom.xml @@ -27,7 +27,7 @@ <dependencies> <dependency> <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> + <artifactId>slf4j-api</artifactId> </dependency> <dependency> <groupId>com.google.guava</groupId> @@ -177,4 +177,35 @@ </plugins> </build> + <reporting> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <reportSets> + <reportSet> + <id>configgen</id> + <configuration> + <doclet>org.apache.tez.tools.javadoc.doclet.ConfigStandardDoclet</doclet> + <docletArtifacts> + <docletArtifact> + <groupId>org.apache.tez</groupId> + <artifactId>tez-javadoc-tools</artifactId> + <version>${project.version}</version> + </docletArtifact> + </docletArtifacts> + <destDir>apidocs/configs</destDir> + <name>TezConfigGenerator</name> + <description>Tez Configuration documentation</description> + </configuration> + <reports> + <report>javadoc</report> + <report>aggregate</report> + </reports> + </reportSet> + </reportSets> + </plugin> + </plugins> + </reporting> + </project> http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationClass.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationClass.java b/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationClass.java new file mode 100644 index 0000000..6b8fdaf --- /dev/null +++ b/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationClass.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.common.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Target; + +import org.apache.hadoop.classification.InterfaceAudience.Private; + +@Private +@Target(ElementType.TYPE) +public @interface ConfigurationClass { + + public String templateFileName(); +} http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationProperty.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationProperty.java b/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationProperty.java new file mode 100644 index 0000000..f7d37e2 --- /dev/null +++ b/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationProperty.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.common.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.apache.hadoop.classification.InterfaceAudience.Private; + +@Private +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.RUNTIME) +public @interface ConfigurationProperty { + + public boolean mandatory() default false; + public String type() default "string"; + public String[] validValues() default {}; +} http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 54d490b..4d12740 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -23,6 +23,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; +import org.apache.tez.common.annotation.ConfigurationClass; +import org.apache.tez.common.annotation.ConfigurationProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -39,8 +41,11 @@ import com.google.common.annotations.VisibleForTesting; * Defines the configurations for Tez. These configurations are typically specified in * tez-site.xml on the client machine where TezClient is used to launch the Tez application. * tez-site.xml is expected to be picked up from the classpath of the client process. + * @see <a href="../../../../../configs/TezConfiguration.html">Detailed Configuration Information</a> + * @see <a href="../../../../../configs/tez-default-template.xml">XML-based Config Template</a> */ @Public +@ConfigurationClass(templateFileName = "tez-default-template.xml") public class TezConfiguration extends Configuration { public final static String TEZ_SITE_XML = "tez-site.xml"; @@ -122,6 +127,7 @@ public class TezConfiguration extends Configuration { * artifacts that it creates within the specified staging dir. Does not affect any user data. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_STAGING_SCRATCH_DATA_AUTO_DELETE = TEZ_AM_PREFIX + "staging.scratch-data.auto-delete"; public static final boolean TEZ_AM_STAGING_SCRATCH_DATA_AUTO_DELETE_DEFAULT = true; @@ -130,6 +136,7 @@ public class TezConfiguration extends Configuration { * String value. Specifies a directory where Tez can create temporary job artifacts. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_AM_STAGING_DIR = TEZ_PREFIX + "staging-dir"; public static final String TEZ_AM_STAGING_DIR_DEFAULT = "/tmp/" + System.getProperty("user.name") + "/tez/staging"; @@ -139,6 +146,7 @@ public class TezConfiguration extends Configuration { * Path to a credentials file (with serialized credentials) located on the local file system. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_CREDENTIALS_PATH = TEZ_PREFIX + "credentials.path"; @Private @@ -164,6 +172,7 @@ public class TezConfiguration extends Configuration { * mode is recommended. If session mode is enabled then container reuse is recommended. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="boolean") public static final String TEZ_AM_SESSION_MODE = TEZ_AM_PREFIX + "mode.session"; public static final boolean TEZ_AM_SESSION_MODE_DEFAULT = false; @@ -184,6 +193,7 @@ public class TezConfiguration extends Configuration { * org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO is not valid * */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_AM_LOG_LEVEL = TEZ_AM_PREFIX + "log.level"; public static final String TEZ_AM_LOG_LEVEL_DEFAULT = "INFO"; @@ -204,6 +214,7 @@ public class TezConfiguration extends Configuration { * org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO is not valid * */ @ConfigurationScope(Scope.VERTEX) + @ConfigurationProperty public static final String TEZ_TASK_LOG_LEVEL = TEZ_TASK_PREFIX + "log.level"; public static final String TEZ_TASK_LOG_LEVEL_DEFAULT = "INFO"; @@ -240,6 +251,7 @@ public class TezConfiguration extends Configuration { * this value must be appropriately chosen. Defaults to the safe choice of true. */ @ConfigurationScope(Scope.DAG) + @ConfigurationProperty(type="boolean") public static final String TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS = TEZ_AM_PREFIX + "commit-all-outputs-on-dag-success"; public static final boolean TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS_DEFAULT = true; @@ -260,6 +272,7 @@ public class TezConfiguration extends Configuration { * be overridden per job. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS = TEZ_AM_PREFIX + "launch.cluster-default.cmd-opts"; public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT = @@ -271,6 +284,7 @@ public class TezConfiguration extends Configuration { * Tez can determine them automatically. * */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_AM_LAUNCH_CMD_OPTS = TEZ_AM_PREFIX + "launch.cmd-opts"; public static final String TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT = "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC"; @@ -282,6 +296,7 @@ public class TezConfiguration extends Configuration { * be overridden per job. */ @ConfigurationScope(Scope.AM) // TODO DAG/Vertex level + @ConfigurationProperty public static final String TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS = TEZ_TASK_PREFIX + "launch.cluster-default.cmd-opts"; public static final String TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT = @@ -293,6 +308,7 @@ public class TezConfiguration extends Configuration { * so that Tez can determine them automatically. */ @ConfigurationScope(Scope.AM) // TODO DAG/Vertex level + @ConfigurationProperty public static final String TEZ_TASK_LAUNCH_CMD_OPTS = TEZ_TASK_PREFIX + "launch.cmd-opts"; public static final String TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT = @@ -308,6 +324,7 @@ public class TezConfiguration extends Configuration { * on container memory size. Value should be greater than 0 and less than 1. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="float") public static final String TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION = TEZ_PREFIX + "container.max.java.heap.fraction"; public static final double TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT = 0.8; @@ -324,6 +341,7 @@ public class TezConfiguration extends Configuration { * These get added to the app master environment prior to launching it. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_AM_LAUNCH_ENV = TEZ_AM_PREFIX + "launch.env"; public static final String TEZ_AM_LAUNCH_ENV_DEFAULT = NATIVE_LIB_PARAM_DEFAULT; @@ -336,12 +354,14 @@ public class TezConfiguration extends Configuration { * These get added to the task environment prior to launching it. */ @ConfigurationScope(Scope.VERTEX) + @ConfigurationProperty public static final String TEZ_TASK_LAUNCH_ENV = TEZ_TASK_PREFIX + "launch.env"; public static final String TEZ_TASK_LAUNCH_ENV_DEFAULT = NATIVE_LIB_PARAM_DEFAULT; @Private @ConfigurationScope(Scope.DAG) + @ConfigurationProperty(type="boolean") public static final String TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION = TEZ_PREFIX + "cancel.delegation.tokens.on.completion"; public static final boolean TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION_DEFAULT = true; @@ -358,6 +378,7 @@ public class TezConfiguration extends Configuration { * Expert level setting. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_AM_TASK_LISTENER_THREAD_COUNT = TEZ_AM_PREFIX + "task.listener.thread-count"; public static final int TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT = 30; @@ -370,6 +391,7 @@ public class TezConfiguration extends Configuration { */ @Unstable @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_COUNTERS_MAX = TEZ_PREFIX + "counters.max"; public static final int TEZ_COUNTERS_MAX_DEFAULT = 1200; @@ -380,6 +402,7 @@ public class TezConfiguration extends Configuration { */ @Unstable @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_COUNTERS_MAX_GROUPS = TEZ_PREFIX + "counters.max.groups"; public static final int TEZ_COUNTERS_MAX_GROUPS_DEFAULT = 500; @@ -390,6 +413,7 @@ public class TezConfiguration extends Configuration { */ @Unstable @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_COUNTERS_COUNTER_NAME_MAX_LENGTH = TEZ_PREFIX + "counters.counter-name.max-length"; public static final int TEZ_COUNTERS_COUNTER_NAME_MAX_LENGTH_DEFAULT = 64; @@ -401,6 +425,7 @@ public class TezConfiguration extends Configuration { */ @Unstable @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH = TEZ_PREFIX + "counters.group-name.max-length"; public static final int TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH_DEFAULT = 256; @@ -411,6 +436,7 @@ public class TezConfiguration extends Configuration { * when some tasks are running slower due bad/slow machines */ @ConfigurationScope(Scope.VERTEX) // TODO Verify the vertex speculation, TEZ-1788 + @ConfigurationProperty(type="boolean") public static final String TEZ_AM_SPECULATION_ENABLED = TEZ_AM_PREFIX + "speculation.enabled"; public static final boolean TEZ_AM_SPECULATION_ENABLED_DEFAULT = false; @@ -420,7 +446,8 @@ public class TezConfiguration extends Configuration { */ @Unstable @ConfigurationScope(Scope.AM) - public static final String TEZ_AM_LEGACY_SPECULATIVE_SLOWTASK_THRESHOLD = + @ConfigurationProperty(type="float") + public static final String TEZ_AM_LEGACY_SPECULATIVE_SLOWTASK_THRESHOLD = TEZ_AM_PREFIX + "legacy.speculative.slowtask.threshold"; /** @@ -428,6 +455,7 @@ public class TezConfiguration extends Configuration { * master. Expert level setting. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT = TEZ_AM_PREFIX + "containerlauncher.thread-count-limit"; @@ -439,6 +467,7 @@ public class TezConfiguration extends Configuration { * Int value. Specifies the number of task failures on a node before the node is considered faulty. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_AM_MAX_TASK_FAILURES_PER_NODE = TEZ_AM_PREFIX + "maxtaskfailures.per.node"; public static final int TEZ_AM_MAX_TASK_FAILURES_PER_NODE_DEFAULT = 10; @@ -450,7 +479,8 @@ public class TezConfiguration extends Configuration { * Expert level setting. */ @ConfigurationScope(Scope.AM) - public static final String TEZ_AM_MAX_APP_ATTEMPTS = TEZ_AM_PREFIX + + @ConfigurationProperty(type="integer") + public static final String TEZ_AM_MAX_APP_ATTEMPTS = TEZ_AM_PREFIX + "max.app.attempts"; public static final int TEZ_AM_MAX_APP_ATTEMPTS_DEFAULT = 2; @@ -459,6 +489,7 @@ public class TezConfiguration extends Configuration { * This does not count killed attempts. Task failure results in DAG failure. */ @ConfigurationScope(Scope.VERTEX) + @ConfigurationProperty(type="integer") public static final String TEZ_AM_TASK_MAX_FAILED_ATTEMPTS = TEZ_AM_PREFIX + "task.max.failed.attempts"; public static final int TEZ_AM_TASK_MAX_FAILED_ATTEMPTS_DEFAULT = 4; @@ -468,6 +499,7 @@ public class TezConfiguration extends Configuration { * will not be used to execute tasks. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="boolean") public static final String TEZ_AM_NODE_BLACKLISTING_ENABLED = TEZ_AM_PREFIX + "node-blacklisting.enabled"; public static final boolean TEZ_AM_NODE_BLACKLISTING_ENABLED_DEFAULT = true; @@ -478,12 +510,14 @@ public class TezConfiguration extends Configuration { * temporary surges in failures (e.g. due to network outages). */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_AM_NODE_BLACKLISTING_IGNORE_THRESHOLD = TEZ_AM_PREFIX + "node-blacklisting.ignore-threshold-node-percent"; public static final int TEZ_AM_NODE_BLACKLISTING_IGNORE_THRESHOLD_DEFAULT = 33; /** Int value. Number of threads to handle client RPC requests. Expert level setting.*/ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_AM_CLIENT_THREAD_COUNT = TEZ_AM_PREFIX + "client.am.thread-count"; public static final int TEZ_AM_CLIENT_THREAD_COUNT_DEFAULT = 1; @@ -494,6 +528,7 @@ public class TezConfiguration extends Configuration { * For example 50000-50050,50100-50200 */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_AM_CLIENT_AM_PORT_RANGE = TEZ_AM_PREFIX + "client.am.port-range"; @@ -510,18 +545,21 @@ public class TezConfiguration extends Configuration { * String value. The class to be used for DAG Scheduling. Expert level setting. */ @ConfigurationScope(Scope.DAG) + @ConfigurationProperty public static final String TEZ_AM_DAG_SCHEDULER_CLASS = TEZ_AM_PREFIX + "dag.scheduler.class"; public static final String TEZ_AM_DAG_SCHEDULER_CLASS_DEFAULT = "org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrder"; /** Int value. The amount of memory in MB to be used by the AppMaster */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_AM_RESOURCE_MEMORY_MB = TEZ_AM_PREFIX + "resource.memory.mb"; public static final int TEZ_AM_RESOURCE_MEMORY_MB_DEFAULT = 1024; /** Int value. The number of virtual cores to be used by the app master */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_AM_RESOURCE_CPU_VCORES = TEZ_AM_PREFIX + "resource.cpu.vcores"; public static final int TEZ_AM_RESOURCE_CPU_VCORES_DEFAULT = 1; @@ -530,6 +568,7 @@ public class TezConfiguration extends Configuration { * all vertices. Setting it to the same value for all tasks is helpful for container reuse and * thus good for performance typically. */ @ConfigurationScope(Scope.DAG) // TODO vertex level + @ConfigurationProperty(type="integer") public static final String TEZ_TASK_RESOURCE_MEMORY_MB = TEZ_TASK_PREFIX + "resource.memory.mb"; public static final int TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT = 1024; @@ -538,6 +577,7 @@ public class TezConfiguration extends Configuration { * Int value. The number of virtual cores to be used by tasks. */ @ConfigurationScope(Scope.DAG) // TODO vertex level + @ConfigurationProperty(type="integer") public static final String TEZ_TASK_RESOURCE_CPU_VCORES = TEZ_TASK_PREFIX + "resource.cpu.vcores"; public static final int TEZ_TASK_RESOURCE_CPU_VCORES_DEFAULT = 1; @@ -548,6 +588,7 @@ public class TezConfiguration extends Configuration { * help in scaling up. Expert level setting. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX = TEZ_AM_PREFIX + "am-rm.heartbeat.interval-ms.max"; public static final int TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX_DEFAULT = 1000; @@ -558,6 +599,7 @@ public class TezConfiguration extends Configuration { * number of concurrent tasks. Expert level setting. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX = TEZ_TASK_PREFIX + "get-task.sleep.interval-ms.max"; public static final int TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX_DEFAULT = 200; @@ -568,6 +610,7 @@ public class TezConfiguration extends Configuration { * Expert level setting. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS = TEZ_TASK_PREFIX + "am.heartbeat.interval-ms.max"; public static final int TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS_DEFAULT = 100; @@ -578,6 +621,7 @@ public class TezConfiguration extends Configuration { * counters. Improves AM scalability. Expert level setting. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS = TEZ_TASK_PREFIX + "am.heartbeat.counter.interval-ms.max"; public static final int TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS_DEFAULT = @@ -588,6 +632,7 @@ public class TezConfiguration extends Configuration { * Expert level setting. Expert level setting. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT = TEZ_TASK_PREFIX + "max-events-per-heartbeat"; public static final int TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT_DEFAULT = 500; @@ -600,6 +645,7 @@ public class TezConfiguration extends Configuration { @Unstable @Private @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="boolean") public static final String TEZ_TASK_GENERATE_COUNTERS_PER_IO = TEZ_TASK_PREFIX + "generate.counters.per.io"; @Private @@ -611,6 +657,7 @@ public class TezConfiguration extends Configuration { * Expert level setting. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TASK_HEARTBEAT_TIMEOUT_MS = TEZ_TASK_PREFIX + "timeout-ms"; public static final int TASK_HEARTBEAT_TIMEOUT_MS_DEFAULT = 5 * 60 * 1000; @@ -620,6 +667,7 @@ public class TezConfiguration extends Configuration { * Expert level setting. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TASK_HEARTBEAT_TIMEOUT_CHECK_MS = TEZ_TASK_PREFIX + "heartbeat.timeout.check-ms"; public static final int TASK_HEARTBEAT_TIMEOUT_CHECK_MS_DEFAULT = 30 * 1000; @@ -631,6 +679,7 @@ public class TezConfiguration extends Configuration { @Private @Unstable @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="boolean") public static final String TEZ_TASK_SCALE_MEMORY_ENABLED = TEZ_TASK_PREFIX + "scale.memory.enabled"; @Private @@ -642,6 +691,7 @@ public class TezConfiguration extends Configuration { @Private @Unstable @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_TASK_SCALE_MEMORY_ALLOCATOR_CLASS = TEZ_TASK_PREFIX + "scale.memory.allocator.class"; @Private @@ -655,6 +705,7 @@ public class TezConfiguration extends Configuration { @Private @Unstable @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="double") public static final String TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION = TEZ_TASK_PREFIX + "scale.memory.reserve-fraction"; @Private @@ -667,6 +718,7 @@ public class TezConfiguration extends Configuration { @Private @Unstable @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="float") public static final String TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_PER_IO = TEZ_TASK_PREFIX + "scale.memory.additional-reservation.fraction.per-io"; @@ -676,6 +728,7 @@ public class TezConfiguration extends Configuration { * Max cumulative total reservation for additional IOs. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="float") public static final String TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_MAX = TEZ_TASK_PREFIX + "scale.memory.additional-reservation.fraction.max"; /* @@ -686,6 +739,7 @@ public class TezConfiguration extends Configuration { @Private @Unstable @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS = TEZ_TASK_PREFIX + "scale.memory.ratios"; @@ -695,6 +749,7 @@ public class TezConfiguration extends Configuration { * Defines the ProcessTree implementation which will be used to collect resource utilization. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS = TEZ_TASK_PREFIX + "resource.calculator.process-tree.class"; @@ -704,6 +759,7 @@ public class TezConfiguration extends Configuration { * This improves performance by not incurring recurring launch overheads. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="boolean") public static final String TEZ_AM_CONTAINER_REUSE_ENABLED = TEZ_AM_PREFIX + "container.reuse.enabled"; public static final boolean TEZ_AM_CONTAINER_REUSE_ENABLED_DEFAULT = true; @@ -713,6 +769,7 @@ public class TezConfiguration extends Configuration { * enabled. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="boolean") public static final String TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED = TEZ_AM_PREFIX + "container.reuse.rack-fallback.enabled"; public static final boolean @@ -724,6 +781,7 @@ public class TezConfiguration extends Configuration { * volume being read from the primary data sources. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="boolean") public static final String TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED = TEZ_AM_PREFIX + "container.reuse.non-local-fallback.enabled"; public static final boolean @@ -734,6 +792,7 @@ public class TezConfiguration extends Configuration { * of locality. NODE -> RACK -> NON_LOCAL. Delay scheduling parameter. Expert level setting. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="long") public static final String TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS = TEZ_AM_PREFIX + "container.reuse.locality.delay-allocation-millis"; @@ -745,7 +804,8 @@ public class TezConfiguration extends Configuration { * reuse is enabled. Set to -1 to never release idle containers (not recommended). */ @ConfigurationScope(Scope.AM) - public static final String TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS = + @ConfigurationProperty(type="integer") + public static final String TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS = TEZ_AM_PREFIX + "container.idle.release-timeout-min.millis"; public static final long TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS_DEFAULT = 5000l; @@ -761,7 +821,8 @@ public class TezConfiguration extends Configuration { * creates a graceful reduction in the amount of idle resources held */ @ConfigurationScope(Scope.AM) - public static final String TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS = + @ConfigurationProperty(type="long") + public static final String TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS = TEZ_AM_PREFIX + "container.idle.release-timeout-max.millis"; public static final long TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS_DEFAULT = 10000l; @@ -772,7 +833,8 @@ public class TezConfiguration extends Configuration { * of containers to provide fast response times for the next DAG. */ @ConfigurationScope(Scope.AM) - public static final String TEZ_AM_SESSION_MIN_HELD_CONTAINERS = + @ConfigurationProperty(type="integer") + public static final String TEZ_AM_SESSION_MIN_HELD_CONTAINERS = TEZ_AM_PREFIX + "session.min.held-containers"; public static final int TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT = 0; @@ -795,7 +857,8 @@ public class TezConfiguration extends Configuration { * setting. */ @ConfigurationScope(Scope.AM) - public static final String TEZ_AM_PREEMPTION_PERCENTAGE = + @ConfigurationProperty(type="integer") + public static final String TEZ_AM_PREEMPTION_PERCENTAGE = TEZ_AM_PREFIX + "preemption.percentage"; public static final int TEZ_AM_PREEMPTION_PERCENTAGE_DEFAULT = 10; @@ -805,7 +868,8 @@ public class TezConfiguration extends Configuration { * RM can act on the released resources and assign new ones to us. Expert level setting. */ @ConfigurationScope(Scope.AM) - public static final String TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS = + @ConfigurationProperty(type="integer") + public static final String TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS = TEZ_AM_PREFIX + "preemption.heartbeats-between-preemptions"; public static final int TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS_DEFAULT = 3; @@ -831,6 +895,7 @@ public class TezConfiguration extends Configuration { * </ol> */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_LIB_URIS = TEZ_PREFIX + "lib.uris"; /** @@ -846,6 +911,7 @@ public class TezConfiguration extends Configuration { * */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_AUX_URIS = TEZ_PREFIX + "aux.uris"; /** @@ -853,7 +919,9 @@ public class TezConfiguration extends Configuration { * raw Tez application where classpath is propagated with application * via {@link LocalResource}s. This is mainly useful for developer/debugger scenarios. */ + @Unstable @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="boolean") public static final String TEZ_IGNORE_LIB_URIS = TEZ_PREFIX + "ignore.lib.uris"; /** @@ -863,6 +931,7 @@ public class TezConfiguration extends Configuration { * tez-deployment which contains the hadoop libraries. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="boolean") public static final String TEZ_USE_CLUSTER_HADOOP_LIBS = TEZ_PREFIX + "use.cluster.hadoop-libs"; public static final boolean TEZ_USE_CLUSTER_HADOOP_LIBS_DEFAULT = false; @@ -874,6 +943,7 @@ public class TezConfiguration extends Configuration { * specified. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX = TEZ_PREFIX + "cluster.additional.classpath.prefix"; @@ -881,6 +951,7 @@ public class TezConfiguration extends Configuration { * Session-related properties */ @Private + @ConfigurationProperty public static final String TEZ_SESSION_PREFIX = TEZ_PREFIX + "session."; @@ -891,6 +962,7 @@ public class TezConfiguration extends Configuration { * applicable. Otherwise increase the timeout (set to -1 for infinity. Not recommended) */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_SESSION_CLIENT_TIMEOUT_SECS = TEZ_SESSION_PREFIX + "client.timeout.secs"; public static final int TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT = @@ -901,6 +973,7 @@ public class TezConfiguration extends Configuration { * shutting down. Only relevant in session mode. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS = TEZ_SESSION_PREFIX + "am.dag.submit.timeout.secs"; public static final int TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT = @@ -910,6 +983,7 @@ public class TezConfiguration extends Configuration { * String value. The queue name for all jobs being submitted from a given client. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_QUEUE_NAME = TEZ_PREFIX + "queue.name"; /** @@ -923,6 +997,7 @@ public class TezConfiguration extends Configuration { * Boolean value. Generate debug artifacts like DAG plan in text format. */ @ConfigurationScope(Scope.DAG) + @ConfigurationProperty(type="boolean") public static final String TEZ_GENERATE_DEBUG_ARTIFACTS = TEZ_PREFIX + "generate.debug.artifacts"; public static final boolean TEZ_GENERATE_DEBUG_ARTIFACTS_DEFAULT = false; @@ -940,6 +1015,7 @@ public class TezConfiguration extends Configuration { */ @Unstable @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST = TEZ_PREFIX + "task-specific" + ".launch.cmd-opts.list"; @@ -952,6 +1028,7 @@ public class TezConfiguration extends Configuration { */ @Unstable @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS = TEZ_PREFIX + "task-specific" + ".launch.cmd-opts"; @@ -973,6 +1050,7 @@ public class TezConfiguration extends Configuration { * */ @Unstable @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_TASK_SPECIFIC_LOG_LEVEL = TEZ_PREFIX + "task-specific" + ".log.level"; /** @@ -980,6 +1058,7 @@ public class TezConfiguration extends Configuration { * Specify the class to use for logging history data */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_HISTORY_LOGGING_SERVICE_CLASS = TEZ_PREFIX + "history.logging.service.class"; @@ -992,6 +1071,7 @@ public class TezConfiguration extends Configuration { * used for {@link TezConfiguration#TEZ_HISTORY_LOGGING_SERVICE_CLASS} */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_SIMPLE_HISTORY_LOGGING_DIR = TEZ_PREFIX + "simple.history.logging.dir"; @@ -1000,6 +1080,7 @@ public class TezConfiguration extends Configuration { * logging gets disabled. The job continues to run after this. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_SIMPLE_HISTORY_LOGGING_MAX_ERRORS = TEZ_PREFIX + "simple.history.max.errors"; public static final int TEZ_SIMPLE_HISTORY_LOGGING_MAX_ERRORS_DEFAULT = 10; @@ -1009,6 +1090,7 @@ public class TezConfiguration extends Configuration { * Expert level setting. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="long") public static final String YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS = TEZ_PREFIX + "yarn.ats.event.flush.timeout.millis"; public static final long YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS_DEFAULT = @@ -1019,6 +1101,7 @@ public class TezConfiguration extends Configuration { * Expert level setting. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String YARN_ATS_MAX_EVENTS_PER_BATCH = TEZ_PREFIX + "yarn.ats.max.events.per.batch"; public static final int YARN_ATS_MAX_EVENTS_PER_BATCH_DEFAULT = 5; @@ -1029,11 +1112,13 @@ public class TezConfiguration extends Configuration { * Expert level setting. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String YARN_ATS_MAX_POLLING_TIME_PER_EVENT = TEZ_PREFIX + "yarn.ats.max.polling.time.per.event.millis"; public static final int YARN_ATS_MAX_POLLING_TIME_PER_EVENT_DEFAULT = 10; @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="boolean") public static final String YARN_ATS_ACL_DOMAINS_AUTO_CREATE = TEZ_PREFIX + "yarn.ats.acl.domains.auto-create"; public static final boolean YARN_ATS_ACL_DOMAINS_AUTO_CREATE_DEFAULT = true; @@ -1050,6 +1135,7 @@ public class TezConfiguration extends Configuration { * incomplete DAGs from the previous instance of the app master. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="boolean") public static final String DAG_RECOVERY_ENABLED = TEZ_PREFIX + "dag.recovery.enabled"; public static final boolean DAG_RECOVERY_ENABLED_DEFAULT = true; @@ -1059,6 +1145,7 @@ public class TezConfiguration extends Configuration { * Expert level setting. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String DAG_RECOVERY_FILE_IO_BUFFER_SIZE = TEZ_PREFIX + "dag.recovery.io.buffer.size"; public static final int DAG_RECOVERY_FILE_IO_BUFFER_SIZE_DEFAULT = 8192; @@ -1067,6 +1154,7 @@ public class TezConfiguration extends Configuration { * Int value. Number of recovery events to buffer before flushing them to the recovery log. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String DAG_RECOVERY_MAX_UNFLUSHED_EVENTS = TEZ_PREFIX + "dag.recovery.max.unflushed.events"; public static final int DAG_RECOVERY_MAX_UNFLUSHED_EVENTS_DEFAULT = 100; @@ -1075,6 +1163,7 @@ public class TezConfiguration extends Configuration { * Int value. Interval, in seconds, between flushing recovery data to the recovery log. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String DAG_RECOVERY_FLUSH_INTERVAL_SECS = TEZ_PREFIX + "dag.recovery.flush.interval.secs"; public static final int DAG_RECOVERY_FLUSH_INTERVAL_SECS_DEFAULT = 30; @@ -1084,6 +1173,7 @@ public class TezConfiguration extends Configuration { * the app master. Primarily used for debugging. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="boolean") public static final String TEZ_LOCAL_MODE = TEZ_PREFIX + "local.mode"; @@ -1094,6 +1184,7 @@ public class TezConfiguration extends Configuration { */ @Private @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="boolean") public static final String TEZ_AM_INLINE_TASK_EXECUTION_ENABLED = TEZ_AM_PREFIX + "inline.task.execution.enabled"; @@ -1108,6 +1199,7 @@ public class TezConfiguration extends Configuration { * The maximium number of tasks running in parallel within the app master process. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="integer") public static final String TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS = TEZ_AM_PREFIX + "inline.task.execution.max-tasks"; @@ -1133,6 +1225,7 @@ public class TezConfiguration extends Configuration { * Boolean value. Configuration to enable/disable ACL checks. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="boolean") public static final String TEZ_AM_ACLS_ENABLED = TEZ_AM_PREFIX + "acls.enabled"; public static final boolean TEZ_AM_ACLS_ENABLED_DEFAULT = true; @@ -1144,6 +1237,7 @@ public class TezConfiguration extends Configuration { * groups */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_AM_VIEW_ACLS = TEZ_AM_PREFIX + "view-acls"; /** @@ -1154,13 +1248,16 @@ public class TezConfiguration extends Configuration { * groups */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_AM_MODIFY_ACLS = TEZ_AM_PREFIX + "modify-acls"; /** * Boolean value. * Disable version check between client and AM/DAG. Default false. */ + @Private @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="boolean") public static final String TEZ_AM_DISABLE_CLIENT_VERSION_CHECK = TEZ_AM_PREFIX + "disable.client-version-check"; public static final boolean TEZ_AM_DISABLE_CLIENT_VERSION_CHECK_DEFAULT = false; @@ -1169,7 +1266,9 @@ public class TezConfiguration extends Configuration { * Boolean value. * Allow disabling of Timeline Domains even if Timeline is being used. */ + @Private @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="boolean") public static final String TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS = TEZ_PREFIX + "allow.disabled.timeline-domains"; public static final boolean TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS_DEFAULT = false; @@ -1189,6 +1288,7 @@ public class TezConfiguration extends Configuration { * http://uihost:9001/#/tez-app/application_1421880306565_0001/ */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE = TEZ_AM_PREFIX + "tez-ui.history-url.template"; public static final String TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE_DEFAULT = @@ -1202,6 +1302,7 @@ public class TezConfiguration extends Configuration { * be specified. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty public static final String TEZ_HISTORY_URL_BASE = TEZ_PREFIX + "tez-ui.history-url.base"; @@ -1211,6 +1312,7 @@ public class TezConfiguration extends Configuration { * updates for running application. */ @ConfigurationScope(Scope.AM) + @ConfigurationProperty(type="boolean") public static final String TEZ_AM_WEBSERVICE_ENABLE = TEZ_AM_PREFIX + "tez-ui.webservice.enable"; public static final boolean TEZ_AM_WEBSERVICE_ENABLE_DEFAULT = true; @@ -1240,6 +1342,7 @@ public class TezConfiguration extends Configuration { * Status Poll interval in Milliseconds used when getting DAG status with timeout. */ @ConfigurationScope(Scope.DAG) + @ConfigurationProperty(type="long") public static final String TEZ_DAG_STATUS_POLLINTERVAL_MS = TEZ_PREFIX + "dag.status.pollinterval-ms"; public static final long TEZ_DAG_STATUS_POLLINTERVAL_MS_DEFAULT = 500; @@ -1248,7 +1351,9 @@ public class TezConfiguration extends Configuration { * Long value. * Time to wait (in seconds) for apps to complete on MiniTezCluster shutdown. */ + @Private @ConfigurationScope(Scope.TEST) + @ConfigurationProperty(type="long") public static final String TEZ_TEST_MINI_CLUSTER_APP_WAIT_ON_SHUTDOWN_SECS = TEZ_PREFIX + "test.minicluster.app.wait.on.shutdown.secs"; public static final long TEZ_TEST_MINI_CLUSTER_APP_WAIT_ON_SHUTDOWN_SECS_DEFAULT = 30; @@ -1260,6 +1365,7 @@ public class TezConfiguration extends Configuration { */ @Private @ConfigurationScope(Scope.CLIENT) + @ConfigurationProperty(type="long") public static final String TEZ_CLIENT_DIAGNOSTICS_WAIT_TIMEOUT_MS = TEZ_PREFIX + "client.diagnostics.wait.timeout-ms"; @Private @@ -1273,6 +1379,7 @@ public class TezConfiguration extends Configuration { */ @Private @ConfigurationScope(Scope.CLIENT) + @ConfigurationProperty public static final String TEZ_CLIENT_JAVA_OPTS_CHECKER_CLASS = TEZ_PREFIX + "java.opts.checker.class"; @@ -1282,6 +1389,7 @@ public class TezConfiguration extends Configuration { */ @Private @ConfigurationScope(Scope.CLIENT) + @ConfigurationProperty(type="boolean") public static final String TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED = TEZ_PREFIX + "java.opts.checker.enabled"; public static final boolean TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED_DEFAULT = true; @@ -1292,8 +1400,8 @@ public class TezConfiguration extends Configuration { * Expert level setting. */ @ConfigurationScope(Scope.CLIENT) + @ConfigurationProperty(type="long") public static final String TEZ_CLIENT_HARD_KILL_TIMEOUT_MS = TEZ_PREFIX + "client.timeout-ms"; - public static final long TEZ_CLIENT_HARD_KILL_TIMEOUT_MS_DEFAULT = 30 * 1000L; /** @@ -1302,7 +1410,7 @@ public class TezConfiguration extends Configuration { * Expert level setting. */ @ConfigurationScope(Scope.CLIENT) + @ConfigurationProperty(type="boolean") public static final String TEZ_CLIENT_ASYNCHRONOUS_STOP = TEZ_PREFIX + "client.asynchronous-stop"; - public static final boolean TEZ_CLIENT_ASYNCHRONOUS_STOP_DEFAULT = true; } http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-dist/pom.xml ---------------------------------------------------------------------- diff --git a/tez-dist/pom.xml b/tez-dist/pom.xml index 9c7fa61..0605c63 100644 --- a/tez-dist/pom.xml +++ b/tez-dist/pom.xml @@ -30,6 +30,11 @@ <dependencies> <dependency> <groupId>org.apache.tez</groupId> + <artifactId>tez-api</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> <artifactId>tez-dag</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-runtime-library/pom.xml ---------------------------------------------------------------------- diff --git a/tez-runtime-library/pom.xml b/tez-runtime-library/pom.xml index 239d62d..efb9c12 100644 --- a/tez-runtime-library/pom.xml +++ b/tez-runtime-library/pom.xml @@ -138,4 +138,36 @@ </plugin> </plugins> </build> + + <reporting> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <reportSets> + <reportSet> + <id>configgen</id> + <configuration> + <doclet>org.apache.tez.tools.javadoc.doclet.ConfigStandardDoclet</doclet> + <docletArtifacts> + <docletArtifact> + <groupId>org.apache.tez</groupId> + <artifactId>tez-javadoc-tools</artifactId> + <version>${project.version}</version> + </docletArtifact> + </docletArtifacts> + <destDir>apidocs/configs</destDir> + <name>TezConfigGenerator</name> + <description>Tez Configuration documentation</description> + </configuration> + <reports> + <report>javadoc</report> + <report>aggregate</report> + </reports> + </reportSet> + </reportSets> + </plugin> + </plugins> + </reporting> + </project> http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java index 5d4ed88..a84448f 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java @@ -30,6 +30,8 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; +import org.apache.tez.common.annotation.ConfigurationClass; +import org.apache.tez.common.annotation.ConfigurationProperty; import org.apache.tez.runtime.library.conf.OrderedPartitionedKVOutputConfig.SorterImpl; /** @@ -37,11 +39,14 @@ import org.apache.tez.runtime.library.conf.OrderedPartitionedKVOutputConfig.Sort * <p/> * Note for developers: Whenever a new key is added to this file, it must also be added to the set of * known tezRuntimeKeys. + * @see <a href="../../../../../../configs/TezRuntimeConfiguration.html">Detailed Configuration Information</a> + * @see <a href="../../../../../configs/tez-runtime-default-template.xml">XML-based Config Template</a> */ // TODO EVENTUALLY A description for each property. @Public @Evolving +@ConfigurationClass(templateFileName = "tez-runtime-default-template.xml") public class TezRuntimeConfiguration { private static final String TEZ_RUNTIME_PREFIX = "tez.runtime."; @@ -69,6 +74,7 @@ public class TezRuntimeConfiguration { /** * Configuration key to enable/disable IFile readahead. */ + @ConfigurationProperty(type = "boolean") public static final String TEZ_RUNTIME_IFILE_READAHEAD = TEZ_RUNTIME_PREFIX + "ifile.readahead"; public static final boolean TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT = true; @@ -76,6 +82,7 @@ public class TezRuntimeConfiguration { /** * Configuration key to set the IFile readahead length in bytes. */ + @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_IFILE_READAHEAD_BYTES = TEZ_RUNTIME_PREFIX + "ifile.readahead.bytes"; public static final int TEZ_RUNTIME_IFILE_READAHEAD_BYTES_DEFAULT = @@ -91,20 +98,24 @@ public class TezRuntimeConfiguration { public static final String TEZ_RUNTIME_IO_FILE_BUFFER_SIZE = "io.file.buffer.size"; + @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_IO_SORT_FACTOR = TEZ_RUNTIME_PREFIX + "io.sort.factor"; public static final int TEZ_RUNTIME_IO_SORT_FACTOR_DEFAULT = 100; + @ConfigurationProperty(type = "float") public static final String TEZ_RUNTIME_SORT_SPILL_PERCENT = TEZ_RUNTIME_PREFIX + "sort.spill.percent"; public static final float TEZ_RUNTIME_SORT_SPILL_PERCENT_DEFAULT = 0.8f; + @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_IO_SORT_MB = TEZ_RUNTIME_PREFIX + "io.sort.mb"; public static final int TEZ_RUNTIME_IO_SORT_MB_DEFAULT = 100; + @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES = TEZ_RUNTIME_PREFIX + "index.cache.memory.limit.bytes"; public static final int TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES_DEFAULT = @@ -112,6 +123,7 @@ public class TezRuntimeConfiguration { // TODO Use the default value + @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_COMBINE_MIN_SPILLS = TEZ_RUNTIME_PREFIX + "combine.min.spills"; public static final int TEZ_RUNTIME_COMBINE_MIN_SPILLS_DEFAULT = 3; @@ -125,10 +137,12 @@ public class TezRuntimeConfiguration { * - PIPELINED ( default ) * {@link org.apache.tez.runtime.library.conf.OrderedPartitionedKVOutputConfig.SorterImpl} */ + @ConfigurationProperty public static final String TEZ_RUNTIME_SORTER_CLASS = TEZ_RUNTIME_PREFIX + "sorter.class"; public static final String TEZ_RUNTIME_SORTER_CLASS_DEFAULT = SorterImpl.PIPELINED.name(); + @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS = TEZ_RUNTIME_PREFIX + "pipelined.sorter.sort.threads"; public static final int TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS_DEFAULT = 2; @@ -137,6 +151,7 @@ public class TezRuntimeConfiguration { * Report partition statistics (e.g better scheduling in ShuffleVertexManager). TEZ-2496 * This can be enabled/disabled at vertex level. */ + @ConfigurationProperty(type = "boolean") public static final String TEZ_RUNTIME_REPORT_PARTITION_STATS = TEZ_RUNTIME_PREFIX + "report.partition.stats"; public static final boolean TEZ_RUNTIME_REPORT_PARTITION_STATS_DEFAULT = true; @@ -144,6 +159,7 @@ public class TezRuntimeConfiguration { /** * Size of the buffer to use if not writing directly to disk. */ + @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB = TEZ_RUNTIME_PREFIX + "unordered.output.buffer.size-mb"; public static final int TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB_DEFAULT = 100; @@ -153,6 +169,7 @@ public class TezRuntimeConfiguration { * This is only meant to be used by unit tests for now. */ @Private + @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES = TEZ_RUNTIME_PREFIX + "unordered.output.max-per-buffer.size-bytes"; @@ -161,125 +178,140 @@ public class TezRuntimeConfiguration { * Specifies a partitioner class, which is used in Tez Runtime components * like OnFileSortedOutput */ + @ConfigurationProperty public static final String TEZ_RUNTIME_PARTITIONER_CLASS = TEZ_RUNTIME_PREFIX + "partitioner.class"; /** * Specifies a combiner class (primarily for Shuffle) */ + @ConfigurationProperty public static final String TEZ_RUNTIME_COMBINER_CLASS = TEZ_RUNTIME_PREFIX + "combiner.class"; + @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES = TEZ_RUNTIME_PREFIX + "shuffle.parallel.copies"; public static final int TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES_DEFAULT = 20; - + @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT = TEZ_RUNTIME_PREFIX + "shuffle.fetch.failures.limit"; public static final int TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT_DEFAULT = 5; - + @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_SHUFFLE_FETCH_MAX_TASK_OUTPUT_AT_ONCE = TEZ_RUNTIME_PREFIX + "shuffle.fetch.max.task.output.at.once"; public final static int TEZ_RUNTIME_SHUFFLE_FETCH_MAX_TASK_OUTPUT_AT_ONCE_DEFAULT = 20; - + @ConfigurationProperty(type = "boolean") public static final String TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR = TEZ_RUNTIME_PREFIX + "shuffle.notify.readerror"; public static final boolean TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR_DEFAULT = true; - + @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT = TEZ_RUNTIME_PREFIX + "shuffle.connect.timeout"; public static final int TEZ_RUNTIME_SHUFFLE_STALLED_COPY_TIMEOUT_DEFAULT = 3 * 60 * 1000; - + @ConfigurationProperty(type = "boolean") public static final String TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_ENABLED = TEZ_RUNTIME_PREFIX + "shuffle.keep-alive.enabled"; public static final boolean TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_ENABLED_DEFAULT = false; - + @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_MAX_CONNECTIONS = TEZ_RUNTIME_PREFIX + "shuffle.keep-alive.max.connections"; public static final int TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_MAX_CONNECTIONS_DEFAULT = 20; - + @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT = TEZ_RUNTIME_PREFIX + "shuffle.read.timeout"; public final static int TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT_DEFAULT = 3 * 60 * 1000; - + @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_SHUFFLE_BUFFER_SIZE = TEZ_RUNTIME_PREFIX + "shuffle.buffersize"; public final static int TEZ_RUNTIME_SHUFFLE_BUFFER_SIZE_DEFAULT = 8 * 1024; + @ConfigurationProperty(type = "boolean") public static final String TEZ_RUNTIME_SHUFFLE_USE_ASYNC_HTTP = TEZ_RUNTIME_PREFIX + "shuffle.use.async.http"; public static final boolean TEZ_RUNTIME_SHUFFLE_USE_ASYNC_HTTP_DEFAULT = false; + @ConfigurationProperty(type = "boolean") public static final String TEZ_RUNTIME_SHUFFLE_ENABLE_SSL = TEZ_RUNTIME_PREFIX + "shuffle.ssl.enable"; public static final boolean TEZ_RUNTIME_SHUFFLE_ENABLE_SSL_DEFAULT = false; - + @ConfigurationProperty(type = "float") public static final String TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT = TEZ_RUNTIME_PREFIX + "shuffle.fetch.buffer.percent"; public static final float TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT_DEFAULT = 0.90f; - + @ConfigurationProperty(type = "float") public static final String TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT = TEZ_RUNTIME_PREFIX + "shuffle.memory.limit.percent"; public static final float TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT_DEFAULT = 0.25f; // Rename to fraction + @ConfigurationProperty(type = "float") public static final String TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT = TEZ_RUNTIME_PREFIX + "shuffle.merge.percent"; public static final float TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT_DEFAULT = 0.90f; - + @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS = TEZ_RUNTIME_PREFIX + "shuffle.memory-to-memory.segments"; - + @ConfigurationProperty(type = "boolean") public static final String TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM = TEZ_RUNTIME_PREFIX + "shuffle.memory-to-memory.enable"; public static final boolean TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM_DEFAULT = false; - + @ConfigurationProperty(type = "float") public static final String TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT = TEZ_RUNTIME_PREFIX + "task.input.post-merge.buffer.percent"; public static final float TEZ_RUNTIME_INPUT_BUFFER_PERCENT_DEFAULT = 0.0f; + @ConfigurationProperty public static final String TEZ_RUNTIME_GROUP_COMPARATOR_CLASS = TEZ_RUNTIME_PREFIX + "group.comparator.class"; + @ConfigurationProperty public static final String TEZ_RUNTIME_INTERNAL_SORTER_CLASS = TEZ_RUNTIME_PREFIX + "internal.sorter.class"; + @ConfigurationProperty public static final String TEZ_RUNTIME_KEY_COMPARATOR_CLASS = TEZ_RUNTIME_PREFIX + "key.comparator.class"; + @ConfigurationProperty public static final String TEZ_RUNTIME_KEY_CLASS = TEZ_RUNTIME_PREFIX + "key.class"; + @ConfigurationProperty public static final String TEZ_RUNTIME_VALUE_CLASS = TEZ_RUNTIME_PREFIX + "value.class"; + @ConfigurationProperty(type = "boolean") public static final String TEZ_RUNTIME_COMPRESS = TEZ_RUNTIME_PREFIX + "compress"; + @ConfigurationProperty public static final String TEZ_RUNTIME_COMPRESS_CODEC = TEZ_RUNTIME_PREFIX + "compress.codec"; // TODO Move this key to MapReduce + @ConfigurationProperty public static final String TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS = TEZ_RUNTIME_PREFIX + "key.secondary.comparator.class"; + @ConfigurationProperty(type = "boolean") public static final String TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED = TEZ_RUNTIME_PREFIX + "empty.partitions.info-via-events.enabled"; @@ -288,6 +320,7 @@ public class TezRuntimeConfiguration { /** * If the shuffle input is on the local host bypass the http fetch and access the files directly */ + @ConfigurationProperty(type = "boolean") public static final String TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH = TEZ_RUNTIME_PREFIX + "optimize.local.fetch"; public static final boolean TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT = true; @@ -298,6 +331,7 @@ public class TezRuntimeConfiguration { * Ensure to set tez.runtime.enable.final-merge.in.output=false. * Speculative execution needs to be turned off when using this parameter. //TODO: TEZ-2132 */ + @ConfigurationProperty(type = "boolean") public static final String TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED = TEZ_RUNTIME_PREFIX + "pipelined-shuffle.enabled"; public static final boolean TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED_DEFAULT = false; @@ -306,6 +340,7 @@ public class TezRuntimeConfiguration { * Expert level setting. Enable final merge in ordered (defaultsorter/pipelinedsorter) outputs. * Speculative execution needs to be turned off when disabling this parameter. //TODO: TEZ-2132 */ + @ConfigurationProperty(type = "boolean") public static final String TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT = TEZ_RUNTIME_PREFIX + "enable.final-merge.in.output"; public static final boolean TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT_DEFAULT = true; @@ -314,6 +349,7 @@ public class TezRuntimeConfiguration { /** * Share data fetched between tasks running on the same host if applicable */ + @ConfigurationProperty(type = "boolean") public static final String TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH = TEZ_RUNTIME_PREFIX + "optimize.shared.fetch"; @@ -328,6 +364,7 @@ public class TezRuntimeConfiguration { */ @Unstable @Private + @ConfigurationProperty(type = "boolean") public static final String TEZ_RUNTIME_CLEANUP_FILES_ON_INTERRUPT = TEZ_RUNTIME_PREFIX + "cleanup.files.on.interrupt"; public static final boolean TEZ_RUNTIME_CLEANUP_FILES_ON_INTERRUPT_DEFAULT = false; @@ -339,12 +376,14 @@ public class TezRuntimeConfiguration { * Value: Boolean * Whether to publish configuration information to History logger. Default false. */ + @ConfigurationProperty public static final String TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT = TEZ_RUNTIME_PREFIX + "convert.user-payload.to.history-text"; public static final boolean TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT = false; @Unstable @Private + @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_RECORDS_BEFORE_PROGRESS = TEZ_RUNTIME_PREFIX + "merge.progress.records"; public static final long TEZ_RUNTIME_RECORDS_BEFORE_PROGRESS_DEFAULT = 10000; @@ -448,4 +487,4 @@ public class TezRuntimeConfiguration { public static Map<String, String> getOtherConfigDefaults() { return Collections.unmodifiableMap(otherConfMap); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-tools/pom.xml ---------------------------------------------------------------------- diff --git a/tez-tools/pom.xml b/tez-tools/pom.xml index af1d304..bf3053e 100644 --- a/tez-tools/pom.xml +++ b/tez-tools/pom.xml @@ -28,7 +28,9 @@ <modules> <module>analyzers</module> + <module>tez-javadoc-tools</module> </modules> + <build> <plugins> <plugin> http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-tools/tez-javadoc-tools/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/tez-tools/tez-javadoc-tools/findbugs-exclude.xml b/tez-tools/tez-javadoc-tools/findbugs-exclude.xml new file mode 100644 index 0000000..ad8f6e4 --- /dev/null +++ b/tez-tools/tez-javadoc-tools/findbugs-exclude.xml @@ -0,0 +1,22 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<FindBugsFilter> + + <Match> + <Class name="org.apache.tez.tools.javadoc.model.ConfigProperty"/> + <Field name="validValues"/> + <Bug pattern="UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD"/> + </Match> + +</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-tools/tez-javadoc-tools/pom.xml ---------------------------------------------------------------------- diff --git a/tez-tools/tez-javadoc-tools/pom.xml b/tez-tools/tez-javadoc-tools/pom.xml new file mode 100644 index 0000000..11290cc --- /dev/null +++ b/tez-tools/tez-javadoc-tools/pom.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.tez</groupId> + <artifactId>tez-tools</artifactId> + <version>0.8.1-SNAPSHOT</version> + </parent> + <artifactId>tez-javadoc-tools</artifactId> + + <dependencies> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>tez-api</artifactId> + </dependency> + </dependencies> + + <profiles> + <profile> + <id>platform-macosx</id> + <activation> + <file> + <exists>${java.home}/../Classes/classes.jar</exists> + </file> + </activation> + <dependencies> + <dependency> + <groupId>com.sun</groupId> + <artifactId>tools</artifactId> + <version>${java.version}</version> + <scope>system</scope> + <systemPath>${java.home}/../Classes/classes.jar</systemPath> + </dependency> + </dependencies> + </profile> + <profile> + <id>other-os</id> + <activation> + <file> + <exists>${java.home}/../lib/tools.jar</exists> + </file> + </activation> + <dependencies> + <dependency> + <groupId>com.sun</groupId> + <artifactId>tools</artifactId> + <version>${java.version}</version> + <scope>system</scope> + <systemPath>${java.home}/../lib/tools.jar</systemPath> + </dependency> + </dependencies> + </profile> + </profiles> + + + <build> + <plugins> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-deploy-plugin</artifactId> + <configuration> + <skip>true</skip> + </configuration> + </plugin> + </plugins> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/doclet/ConfigStandardDoclet.java ---------------------------------------------------------------------- diff --git a/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/doclet/ConfigStandardDoclet.java b/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/doclet/ConfigStandardDoclet.java new file mode 100644 index 0000000..6cc9ced --- /dev/null +++ b/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/doclet/ConfigStandardDoclet.java @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.tools.javadoc.doclet; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.tez.common.annotation.ConfigurationClass; +import org.apache.tez.common.annotation.ConfigurationProperty; +import org.apache.tez.tools.javadoc.model.Config; +import org.apache.tez.tools.javadoc.model.ConfigProperty; +import org.apache.tez.tools.javadoc.util.HtmlWriter; +import org.apache.tez.tools.javadoc.util.XmlWriter; + +import com.sun.javadoc.AnnotationDesc; +import com.sun.javadoc.AnnotationDesc.ElementValuePair; +import com.sun.javadoc.ClassDoc; +import com.sun.javadoc.DocErrorReporter; +import com.sun.javadoc.FieldDoc; +import com.sun.javadoc.LanguageVersion; +import com.sun.javadoc.RootDoc; +import com.sun.tools.doclets.standard.Standard; + +public class ConfigStandardDoclet { + + private static final String DEBUG_SWITCH = "-debug"; + private static boolean debugMode = false; + + public static LanguageVersion languageVersion() { + return LanguageVersion.JAVA_1_5; + } + + private static void logMessage(String message) { + if (!debugMode) { + return; + } + System.out.println(message); + } + + public static boolean start(RootDoc root) { + //look for debug flag + for (String[] opts : root.options()) { + for (String opt : opts) { + if (opt.equals(DEBUG_SWITCH)) { + debugMode = true; + } + } + } + + logMessage("Running doclet " + ConfigStandardDoclet.class.getSimpleName()); + ClassDoc[] classes = root.classes(); + for (int i = 0; i < classes.length; ++i) { + processDoc(classes[i]); + } + + return true; + } + + private static void processDoc(ClassDoc doc) { + logMessage("Parsing : " + doc); + if (!doc.isClass()) { + logMessage("Ignoring non-class: " + doc); + return; + } + + AnnotationDesc[] annotations = doc.annotations(); + boolean isConfigClass = false; + String templateName = null; + for (AnnotationDesc annotation : annotations) { + logMessage("Checking annotation: " + annotation.annotationType()); + if (annotation.annotationType().qualifiedTypeName().equals( + ConfigurationClass.class.getName())) { + isConfigClass = true; + for (ElementValuePair element : annotation.elementValues()) { + if (element.element().name().equals("templateFileName")) { + templateName = stripQuotes(element.value().toString()); + } + } + break; + } + } + + if (!isConfigClass) { + logMessage("Ignoring non-config class: " + doc); + return; + } + + logMessage("Processing config class: " + doc); + Config config = new Config(doc.name(), templateName); + Map<String, ConfigProperty> configProperties = config.configProperties; + + FieldDoc[] fields = doc.fields(); + for (FieldDoc field : fields) { + if (field.isPrivate()) { + logMessage("Skipping private field: " + field); + continue; + } + if (!field.isStatic()) { + logMessage("Skipping non-static field: " + field); + continue; + } + + if (field.name().endsWith("_PREFIX")) { + logMessage("Skipping non-config prefix constant field: " + field); + continue; + } + if (field.name().equals("TEZ_SITE_XML")) { + logMessage("Skipping constant field: " + field); + continue; + } + + if (field.name().endsWith("_DEFAULT")) { + + String name = field.name().substring(0, + field.name().lastIndexOf("_DEFAULT")); + if (!configProperties.containsKey(name)) { + configProperties.put(name, new ConfigProperty()); + } + ConfigProperty configProperty = configProperties.get(name); + if (field.constantValue() == null) { + logMessage("Got null constant value" + + ", name=" + name + + ", field=" + field.name() + + ", val=" + field.constantValueExpression()); + configProperty.defaultValue = field.constantValueExpression(); + } else { + configProperty.defaultValue = field.constantValue().toString(); + } + configProperty.inferredType = field.type().simpleTypeName(); + + if (name.equals("TEZ_AM_STAGING_DIR") && configProperty.defaultValue != null) { + String defaultValue = configProperty.defaultValue; + defaultValue = defaultValue.replace(System.getProperty("user.name"), "${user.name}"); + configProperty.defaultValue = defaultValue; + } + + continue; + } + + String name = field.name(); + if (!configProperties.containsKey(name)) { + configProperties.put(name, new ConfigProperty()); + } + ConfigProperty configProperty = configProperties.get(name); + configProperty.propertyName = field.constantValue().toString(); + + AnnotationDesc[] annotationDescs = field.annotations(); + + for (AnnotationDesc annotationDesc : annotationDescs) { + + if (annotationDesc.annotationType().qualifiedTypeName().equals( + Private.class.getCanonicalName())) { + configProperty.isPrivate = true; + } + if (annotationDesc.annotationType().qualifiedTypeName().equals( + Unstable.class.getCanonicalName())) { + configProperty.isUnstable = true; + } + if (annotationDesc.annotationType().qualifiedTypeName().equals( + Evolving.class.getCanonicalName())) { + configProperty.isEvolving = true; + } + if (annotationDesc.annotationType().qualifiedTypeName().equals( + ConfigurationProperty.class.getCanonicalName())) { + configProperty.isValidConfigProp = true; + + boolean foundType = false; + for (ElementValuePair element : annotationDesc.elementValues()) { + if (element.element().name().equals("type")) { + configProperty.type = stripQuotes(element.value().toString()); + foundType = true; + } else { + logMessage("Unhandled annotation property: " + element.element().name()); + } + } + } + } + + configProperty.description = field.commentText(); + + } + + HtmlWriter writer = new HtmlWriter(); + try { + writer.write(config); + } catch (IOException e) { + throw new RuntimeException(e); + } + + XmlWriter xmlWriter = new XmlWriter(); + try { + xmlWriter.write(config); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + private static String stripQuotes(String s) { + if (s.charAt(0) == '"' && s.charAt(s.length()-1) == '"') { + return s.substring(1, s.length()-1); + } + return s; + } + + public static int optionLength(String option) { + return Standard.optionLength(option); + } + + public static boolean validOptions(String options[][], DocErrorReporter reporter) { + return true; + } +} http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/model/Config.java ---------------------------------------------------------------------- diff --git a/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/model/Config.java b/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/model/Config.java new file mode 100644 index 0000000..604d48a --- /dev/null +++ b/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/model/Config.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.tools.javadoc.model; + +import java.util.Map; +import java.util.TreeMap; + +public class Config { + + public final String templateName; + public final String configName; + public Map<String, ConfigProperty> configProperties; + + public Config(String configName, String templateName) { + this.configName = configName; + this.templateName = templateName; + this.configProperties = new TreeMap<String, ConfigProperty>(); + } + + public Config() { + this(null, null); + } + +} http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/model/ConfigProperty.java ---------------------------------------------------------------------- diff --git a/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/model/ConfigProperty.java b/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/model/ConfigProperty.java new file mode 100644 index 0000000..94c7e27 --- /dev/null +++ b/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/model/ConfigProperty.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.tools.javadoc.model; + +public class ConfigProperty { + + public String propertyName; + public String defaultValue; + public String description; + public String type = "string"; + public boolean isPrivate = false; + public boolean isUnstable = false; + public boolean isEvolving = false; + public boolean isValidConfigProp = false; + public String[] validValues; + public String inferredType; + + @Override + public String toString() { + return "name=" + propertyName + + ", defaultValue=" + defaultValue + + ", description=" + description + + ", type=" + type + + ", private=" + isPrivate + + ", validValues=" + (validValues == null ? "null" : validValues) + + ", isConfigProp=" + isValidConfigProp; + } +} + +
