This is an automated email from the ASF dual-hosted git repository.
guyuqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new c0207848 BIGTOP-3739: Add Tez support for Bigtop 3.1.0 Mpack (#948)
c0207848 is described below
commit c0207848af76e0eee06de93c4692dab41e22b808
Author: timyuer <[email protected]>
AuthorDate: Mon Jul 18 23:45:05 2022 +0800
BIGTOP-3739: Add Tez support for Bigtop 3.1.0 Mpack (#948)
* BIGTOP-3739: Add Tez support for Bigtop 3.1.0 Mpack
Co-authored-by: houyu <[email protected]>
---
.../1.0/services/TEZ/configuration/tez-env.xml | 97 ++++
.../1.0/services/TEZ/configuration/tez-site.xml | 534 +++++++++++++++++++++
.../stacks/BGTP/1.0/services/TEZ/kerberos.json | 25 +
.../stacks/BGTP/1.0/services/TEZ/metainfo.xml | 111 +++++
.../1.0/services/TEZ/package/scripts/params.py | 29 ++
.../services/TEZ/package/scripts/params_linux.py | 123 +++++
.../services/TEZ/package/scripts/params_windows.py | 54 +++
.../services/TEZ/package/scripts/pre_upgrade.py | 59 +++
.../services/TEZ/package/scripts/service_check.py | 126 +++++
.../BGTP/1.0/services/TEZ/package/scripts/tez.py | 83 ++++
.../1.0/services/TEZ/package/scripts/tez_client.py | 130 +++++
.../BGTP/1.0/services/TEZ/role_command_order.json | 6 +
.../BGTP/1.0/services/TEZ/service_advisor.py | 343 +++++++++++++
.../BGTP/1.0/services/TEZ/themes/directories.json | 177 +++++++
14 files changed, 1897 insertions(+)
diff --git
a/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/configuration/tez-env.xml
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/configuration/tez-env.xml
new file mode 100644
index 00000000..80e66a29
--- /dev/null
+++
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/configuration/tez-env.xml
@@ -0,0 +1,97 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+ <property>
+ <name>tez_user</name>
+ <display-name>Tez User</display-name>
+ <value>tez</value>
+ <property-type>USER</property-type>
+ <description/>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ <user-groups>
+ <property>
+ <type>hadoop-env</type>
+ <name>proxyuser_group</name>
+ </property>
+ <property>
+ <type>cluster-env</type>
+ <name>user_group</name>
+ </property>
+ </user-groups>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <!-- tez-env.sh -->
+ <property>
+ <name>content</name>
+ <display-name>tez-env template</display-name>
+ <description>This is the jinja template for tez-env.sh file</description>
+ <value>
+# Tez specific configuration
+export TEZ_CONF_DIR={{config_dir}}
+
+# Set HADOOP_HOME to point to a specific hadoop install directory
+export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
+
+# The java implementation to use.
+export JAVA_HOME={{java64_home}}
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>enable_heap_dump</name>
+ <value>false</value>
+ <description>Enable or disable taking Heap Dump. (true/false)</description>
+ <display-name>Enable heap dump</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <overridable>false</overridable>
+ <entries>
+ <entry>
+ <value>true</value>
+ <label>Enabled</label>
+ </entry>
+ <entry>
+ <value>false</value>
+ <label>Disabled</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>heap_dump_location</name>
+ <value>/tmp</value>
+ <description>Location for heap dump file</description>
+ <display-name>Heap dump location</display-name>
+ <on-ambari-upgrade add="false"/>
+ <value-attributes>
+ <empty-value-valid>false</empty-value-valid>
+ </value-attributes>
+ </property>
+</configuration>
diff --git
a/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/configuration/tez-site.xml
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/configuration/tez-site.xml
new file mode 100644
index 00000000..e1748fc0
--- /dev/null
+++
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/configuration/tez-site.xml
@@ -0,0 +1,534 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="true">
+ <property>
+ <name>tez.lib.uris.classpath</name>
+ <display-name>TEZ Lib URIs Classpath</display-name>
+
<value>{{yarn_application_classpath}},{{tez_home}}/*,{{tez_home}}/lib/*</value>
+ <description>Comma-delimited list of the location of the Tez libraries
Classpath which will be localized for DAGs.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.lib.uris</name>
+ <display-name>TEZ Lib URIs</display-name>
+ <value>{{tez_lib_uris}}</value>
+ <description>Comma-delimited list of the location of the Tez libraries
which will be localized for DAGs.
+ Specifying a single .tar.gz or .tgz assumes that a compressed version of
the tez libs is being used. This is uncompressed into a tezlibs directory when
running containers, and tezlibs/;tezlibs/lib/ are added to the classpath (after
. and .*).
+ If multiple files are specified - files are localized as regular files,
contents of directories are localized as regular files (non-recursive).
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.cluster.additional.classpath.prefix</name>
+ <value>/etc/hadoop/conf/secure</value>
+ <description/>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.am.log.level</name>
+ <value>INFO</value>
+ <description>Root Logging level passed to the Tez app master</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.generate.debug.artifacts</name>
+ <value>false</value>
+ <description>Generate debug artifacts such as a text representation of the
submitted DAG plan</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.staging-dir</name>
+ <display-name>TEZ Staging directory</display-name>
+ <value>/tmp/${user.name}/staging</value>
+ <description>The staging dir used while submitting DAGs</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.am.resource.memory.mb</name>
+ <value>2048</value>
+ <description>The amount of memory to be used by the AppMaster.
+ Used only if the value is not specified explicitly by the DAG definition.
+ </description>
+ <value-attributes>
+ <type>int</type>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.am.launch.cmd-opts</name>
+ <value>-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA
-XX:+UseG1GC -XX:+ResizeTLAB{{heap_dump_opts}}</value>
+ <description>Java options for the Tez AppMaster process. The Xmx value is
derived based on tez.am.resource.memory.mb and is 80% of the value by default.
+ Used only if the value is not specified explicitly by the DAG definition.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.am.launch.cluster-default.cmd-opts</name>
+ <value>-server -Djava.net.preferIPv4Stack=true</value>
+ <description>Cluster default Java options for the Tez AppMaster process.
These will be prepended to the properties specified via
tez.am.launch.cmd-opts</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.am.launch.env</name>
+ <value>LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native</value>
+ <description>
+ Additional execution environment entries for tez. This is not an
additive property. You must preserve the original value if
+ you want to have access to native libraries.
+ Used only if the value is not specified explicitly by the DAG definition.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.task.resource.memory.mb</name>
+ <value>1536</value>
+ <description>The amount of memory to be used by launched tasks.
+ Used only if the value is not specified explicitly by the DAG definition.
+ </description>
+ <depends-on>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ </property>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.scheduler.maximum-allocation-mb</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <type>int</type>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.task.launch.cmd-opts</name>
+ <value>-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA
-XX:+UseG1GC -XX:+ResizeTLAB{{heap_dump_opts}}</value>
+ <description>Java options for tasks. The Xmx value is derived based on
tez.task.resource.memory.mb and is 80% of this value by default.
+ Used only if the value is not specified explicitly by the DAG definition.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.task.launch.cluster-default.cmd-opts</name>
+ <value>-server -Djava.net.preferIPv4Stack=true</value>
+ <description>Cluster default Java options for tasks. These will be
prepended to the properties specified via tez.task.launch.cmd-opts</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.task.launch.env</name>
+ <value>LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native</value>
+ <description>
+ Additional execution environment entries for tez. This is not an
additive property. You must preserve the original value if
+ you want to have access to native libraries.
+ Used only if the value is not specified explicitly by the DAG definition.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.shuffle-vertex-manager.min-src-fraction</name>
+ <value>0.2</value>
+ <description>In case of a ScatterGather connection, the fraction of source
tasks which should
+ complete before tasks for the current vertex are schedule
+ </description>
+ <value-attributes>
+ <type>float</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.shuffle-vertex-manager.max-src-fraction</name>
+ <value>0.4</value>
+ <description>In case of a ScatterGather connection, once this fraction of
source tasks have
+ completed, all tasks on the current vertex can be scheduled. Number of
tasks ready for
+ scheduling on the current vertex scales linearly between min-fraction
and max-fraction
+ </description>
+ <value-attributes>
+ <type>float</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.am.am-rm.heartbeat.interval-ms.max</name>
+ <value>250</value>
+ <description>The maximum heartbeat interval between the AM and RM in
milliseconds</description>
+ <value-attributes>
+ <type>int</type>
+ <unit>ms</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.grouping.split-waves</name>
+ <value>1.7</value>
+ <description>The multiplier for available queue capacity when determining
number of tasks for
+ a Vertex. 1.7 with 100% queue available implies generating a number of
tasks roughly equal
+ to 170% of the available containers on the queue
+ </description>
+ <value-attributes>
+ <type>float</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.grouping.min-size</name>
+ <value>16777216</value>
+ <description>Lower bound on the size (in bytes) of a grouped split, to
avoid generating
+ too many splits
+ </description>
+ <value-attributes>
+ <type>int</type>
+ <unit>Bytes</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.grouping.max-size</name>
+ <value>1073741824</value>
+ <description>Upper bound on the size (in bytes) of a grouped split, to
avoid generating
+ excessively large split
+ </description>
+ <value-attributes>
+ <type>int</type>
+ <unit>Bytes</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.am.container.reuse.enabled</name>
+ <value>true</value>
+ <description>Configuration to specify whether container should be
reused</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.am.container.reuse.rack-fallback.enabled</name>
+ <value>true</value>
+ <description>Whether to reuse containers for rack local tasks. Active only
if reuse is enabled
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.am.container.reuse.non-local-fallback.enabled</name>
+ <value>false</value>
+ <description>Whether to reuse containers for non-local tasks. Active only
if reuse is enabled
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.am.container.idle.release-timeout-min.millis</name>
+ <value>10000</value>
+ <description>The minimum amount of time to hold on to a container that is
idle. Only active when reuse is enabled.</description>
+ <value-attributes>
+ <type>int</type>
+ <unit>ms</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.am.container.idle.release-timeout-max.millis</name>
+ <value>20000</value>
+ <description>The maximum amount of time to hold on to a container if no
task can be assigned to it immediately. Only active when reuse is
enabled.</description>
+ <value-attributes>
+ <type>int</type>
+ <unit>ms</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.am.container.reuse.locality.delay-allocation-millis</name>
+ <value>250</value>
+ <description>The amount of time to wait before assigning a container to
the next level of
+ locality. NODE -> RACK -> NON_LOCAL
+ </description>
+ <value-attributes>
+ <type>int</type>
+ <unit>ms</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.am.max.app.attempts</name>
+ <value>2</value>
+ <description>Specifies the total number of time the app master will run in
case recovery is triggered</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.am.maxtaskfailures.per.node</name>
+ <value>10</value>
+ <description>The maximum number of allowed task attempt failures on a node
before
+ it gets marked as blacklisted
+ </description>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.task.am.heartbeat.counter.interval-ms.max</name>
+ <value>4000</value>
+ <description>Time interval at which task counters are sent to the
AM</description>
+ <value-attributes>
+ <type>int</type>
+ <unit>ms</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.task.get-task.sleep.interval-ms.max</name>
+ <value>200</value>
+ <description>The maximum amount of time, in seconds, to wait before a task
asks an AM for
+ another task
+ </description>
+ <value-attributes>
+ <type>int</type>
+ <unit>ms</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.task.max-events-per-heartbeat</name>
+ <value>500</value>
+ <description>Maximum number of of events to fetch from the AM by the tasks
in a single heartbeat.</description>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.session.client.timeout.secs</name>
+ <value>-1</value>
+ <description>Time (in seconds) to wait for AM to come up when trying to
submit a DAG from
+ the client
+ </description>
+ <value-attributes>
+ <type>int</type>
+ <unit>seconds</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.session.am.dag.submit.timeout.secs</name>
+ <value>300</value>
+ <description>Time (in seconds) for which the Tez AM should wait for a DAG
to be submitted
+ before shutting down
+ </description>
+ <value-attributes>
+ <type>int</type>
+ <unit>seconds</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.runtime.compress</name>
+ <value>true</value>
+ <description>Whether intermediate data should be compressed or
not</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.runtime.compress.codec</name>
+ <value>org.apache.hadoop.io.compress.SnappyCodec</value>
+ <description>The coded to be used if compressing intermediate data. Only
+ applicable if tez.runtime.compress is enabled
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.runtime.unordered.output.buffer.size-mb</name>
+ <value>100</value>
+ <description>The size of the buffer when output does not require to be
sorted</description>
+ <depends-on>
+ <property>
+ <type>tez-site</type>
+ <name>tez.task.resource.memory.mb</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <type>int</type>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.runtime.convert.user-payload.to.history-text</name>
+ <value>false</value>
+ <description>Whether to publish configuration information to History
logger</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.use.cluster.hadoop-libs</name>
+ <value>false</value>
+ <description>This being true implies that the deployment is relying on
hadoop jars being available on the cluster on all nodes.</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.am.tez-ui.history-url.template</name>
+
<value>__HISTORY_URL_BASE__?viewPath=%2F%23%2Ftez-app%2F__APPLICATION_ID__</value>
+ <description>Template to generate the History URL for a particular Tez
Application.
+ Template replaces __APPLICATION_ID__ with the actual applicationId and
+ __HISTORY_URL_BASE__ with the value from the
tez.tez-ui.history-url.base config property
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.tez-ui.history-url.base</name>
+ <description/>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.am.view-acls</name>
+ <value>*</value>
+ <description>AM view ACLs. This allows the specified users/groups to view
the status of the AM and all DAGs that run within this AM.
+ Value format: Comma separated list of users, followed by whitespace,
followed by a comma separated list of groups.
+ </description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.runtime.optimize.local.fetch</name>
+ <value>true</value>
+ <description>If the shuffle input is on the local host bypass the http
fetch and access the files directly.</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.task.generate.counters.per.io</name>
+ <value>true</value>
+ <description>Whether to generate counters on a per-edge basis for a Tez
DAG. Helpful for in-depth analysis.</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.runtime.sorter.class</name>
+ <value>PIPELINED</value>
+ <description>Which sorter implementation to use</description>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>PIPELINED</value>
+ <label>Pipelined Sorter</label>
+ </entry>
+ <entry>
+ <value>LEGACY</value>
+ <label>Legacy Sorter</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.runtime.pipelined.sorter.sort.threads</name>
+ <value>2</value>
+ <description>Tez runtime pipelined sorter sort threads</description>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>tez-site</type>
+ <name>tez.runtime.sorter.class</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.runtime.io.sort.mb</name>
+ <value>272</value>
+ <description>The size of the sort buffer when output needs to be
sorted</description>
+ <value-attributes>
+ <type>int</type>
+ <unit>MB</unit>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>tez-site</type>
+ <name>tez.runtime.sorter.class</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.counters.max</name>
+ <value>10000</value>
+ <description>The number of allowed counters for the executing
DAG</description>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.counters.max.groups</name>
+ <value>3000</value>
+ <description>The number of allowed counter groups for the executing
DAG</description>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.runtime.shuffle.fetch.buffer.percent</name>
+ <value>0.6</value>
+ <description>Fraction (0-1) of the available memory which can be used to
+ retain shuffled data</description>
+ <value-attributes>
+ <type>float</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.runtime.shuffle.memory.limit.percent</name>
+ <value>0.25</value>
+ <description>This property determines the maximum size of a shuffle segment
+ which can be fetched to memory. Fraction (0-1) of shuffle memory
+ (after applying tez.runtime.shuffle.fetch.buffer.percent)</description>
+ <value-attributes>
+ <type>float</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+
<name>tez.history.logging.timeline-cache-plugin.old-num-dags-per-group</name>
+ <value>5</value>
+ <description>DAGs per group</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.enabled</name>
+ <value>false</value>
+ <description>Timeline service version we’re currently
using.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
diff --git
a/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/kerberos.json
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/kerberos.json
new file mode 100644
index 00000000..6be8292d
--- /dev/null
+++
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/kerberos.json
@@ -0,0 +1,25 @@
+{
+ "services": [
+ {
+ "name": "TEZ",
+ "components": [
+ {
+ "name": "TEZ_CLIENT",
+ "identities": [
+ {
+ "name": "tez_tez_client_hdfs",
+ "reference": "/HDFS/NAMENODE/hdfs"
+ }
+ ],
+ "configurations": [
+ {
+ "tez-site": {
+ "tez.am.view-acls": ""
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git
a/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/metainfo.xml
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/metainfo.xml
new file mode 100644
index 00000000..17afc757
--- /dev/null
+++
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/metainfo.xml
@@ -0,0 +1,111 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>TEZ</name>
+ <displayName>Tez</displayName>
+ <comment>Tez is the next generation Hadoop Query Processing
framework written on top of YARN.</comment>
+ <version>Bigtop+3.1</version>
+ <components>
+ <component>
+ <name>TEZ_CLIENT</name>
+ <displayName>Tez Client</displayName>
+ <cardinality>1+</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <category>CLIENT</category>
+ <commandScript>
+ <script>scripts/tez_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ <configFiles>
+ <configFile>
+ <type>xml</type>
+ <fileName>tez-site.xml</fileName>
+ <dictionaryName>tez-site</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>tez-env.sh</fileName>
+ <dictionaryName>tez-env</dictionaryName>
+ </configFile>
+ </configFiles>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>YARN/YARN_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ </component>
+ </components>
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>tez</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>YARN</service>
+ </requiredServices>
+
+ <configuration-dependencies>
+ <config-type>tez-site</config-type>
+ <config-type>tez-env</config-type>
+ <config-type>hive-site</config-type>
+ </configuration-dependencies>
+
+ <themes>
+ <theme>
+ <fileName>directories.json</fileName>
+ <default>true</default>
+ </theme>
+ </themes>
+
+ </service>
+ </services>
+</metainfo>
diff --git
a/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/params.py
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/params.py
new file mode 100644
index 00000000..c01dc620
--- /dev/null
+++
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/params.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.copy_tarball import
get_sysprep_skip_copy_tarballs_hdfs
+
+if OSCheck.is_windows_family():
+ from params_windows import *
+else:
+ from params_linux import *
+
+sysprep_skip_copy_tarballs_hdfs = get_sysprep_skip_copy_tarballs_hdfs()
\ No newline at end of file
diff --git
a/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/params_linux.py
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/params_linux.py
new file mode 100644
index 00000000..94d6cb99
--- /dev/null
+++
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/params_linux.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os
+
+from resource_management.libraries.resources import HdfsResource
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.stack_features import
check_stack_feature
+from resource_management.libraries.functions.version import
format_stack_version
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions.get_not_managed_resources import
get_not_managed_resources
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions.get_architecture import
get_architecture
+
+# server configurations
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+architecture = get_architecture()
+
+stack_name = default("/clusterLevelParams/stack_name", None)
+stack_root = Script.get_stack_root()
+
+# This is expected to be of the form #.#.#.#
+stack_version_unformatted = config['clusterLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+# New Cluster Stack Version that is defined during the RESTART of a Rolling
Upgrade
+version = default("/commandParams/version", None)
+
+# default hadoop parameters
+hadoop_home = '/usr/lib/hadoop'
+tez_home = '/usr/lib/tez'
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+tez_etc_dir = "/etc/tez"
+config_dir = "/etc/tez/conf"
+tez_examples_jar = "/usr/lib/tez/tez-examples*.jar"
+
+
+# hadoop parameters for stacks that support rolling_upgrade
+if stack_version_formatted and
check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+ tez_examples_jar = "{stack_root}/current/tez-client/tez-examples*.jar"
+
+# tez only started linking <stack-root>/x.x.x.x/tez-client/conf in
config_versioning
+if stack_version_formatted and
check_stack_feature(StackFeature.CONFIG_VERSIONING, stack_version_formatted):
+ # !!! use realpath for now since the symlink exists but is broken and a
+ # broken symlink messes with the DirectoryProvider class
+ config_path = os.path.join(stack_root, "current/tez-client/conf")
+ config_dir = os.path.realpath(config_path)
+
+# Heap dump related
+heap_dump_enabled = default('/configurations/tez-env/enable_heap_dump', None)
+heap_dump_opts = "" # Empty if 'heap_dump_enabled' is False.
+if heap_dump_enabled:
+ heap_dump_path = default('/configurations/tez-env/heap_dump_location',
"/tmp")
+ heap_dump_opts = " -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath="+heap_dump_path
+
+kinit_path_local =
get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',
None))
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+smokeuser_principal =
config['configurations']['cluster-env']['smokeuser_principal_name']
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name =
config['configurations']['hadoop-env']['hdfs_principal_name']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+
+java64_home = config['ambariLevelParams']['java_home']
+
+tez_user = config['configurations']['tez-env']['tez_user']
+user_group = config['configurations']['cluster-env']['user_group']
+tez_env_sh_template = config['configurations']['tez-env']['content']
+
+tez_lib_base_dir_path = '/apps/tez'
+tez_lib_uris = os.path.join(tez_lib_base_dir_path, 'tez.tar.gz')
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+yarn_application_classpath =
config['configurations']['yarn-site']['yarn.application.classpath']
+
+dfs_type = default("/clusterLevelParams/dfs_type", "")
+
+import functools
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete/copyfromlocal hdfs directories/files we need to call
params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ hdfs_resource_ignore_file =
"/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ hadoop_bin_dir = hadoop_bin_dir,
+ hadoop_conf_dir = hadoop_conf_dir,
+ principal_name = hdfs_principal_name,
+ hdfs_site = hdfs_site,
+ default_fs = default_fs,
+ immutable_paths = get_not_managed_resources(),
+ dfs_type = dfs_type
+)
+
+tez_site_config = dict(config['configurations']['tez-site'])
+
+
diff --git
a/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/params_windows.py
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/params_windows.py
new file mode 100644
index 00000000..64c2a53c
--- /dev/null
+++
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/params_windows.py
@@ -0,0 +1,54 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+from resource_management.libraries.functions.get_stack_version import
get_stack_version
+from resource_management.libraries.script.script import Script
+
+config = Script.get_config()
+hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]
+user_group = config["configurations"]["cluster-env"]["user_group"]
+tez_user = hadoop_user
+tez_home_dir = None
+tez_conf_dir = "conf"
+
+try:
+ hadoop_classpath_prefix_template =
config["configurations"]["tez-site"]["tez.cluster.additional.classpath.prefix"]
+except KeyError:
+ hadoop_classpath_prefix_template = ""
+
+stack_version_formatted = ""
+
+stack_root = None
+try:
+ stack_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"], ".."))
+except:
+ pass
+
+def refresh_tez_state_dependent_params():
+ global tez_home_dir, tez_conf_dir, stack_version_formatted
+ tez_home_dir = os.environ["TEZ_HOME"]
+ tez_conf_dir = os.path.join(tez_home_dir, "conf")
+ # this is not available on INSTALL action because <stack-selector-tool> is
not available
+ stack_version_formatted = get_stack_version("tez")
+
+
+if os.environ.has_key("TEZ_HOME"):
+ refresh_tez_state_dependent_params()
diff --git
a/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/pre_upgrade.py
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/pre_upgrade.py
new file mode 100644
index 00000000..d182efee
--- /dev/null
+++
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/pre_upgrade.py
@@ -0,0 +1,59 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management.libraries.script import Script
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import
check_stack_feature
+from resource_management.libraries.functions.copy_tarball import copy_to_hdfs
+from resource_management.core.exceptions import Fail
+
+from resource_management.core.logger import Logger
+
+class TezPreUpgrade(Script):
+
+ def prepare(self, env):
+ """
+ During the "Upgrade" direction of a Stack Upgrade, it is necessary to
ensure that the older tez tarball
+ has been copied to HDFS. This is an additional check for added robustness.
+ """
+ import params
+ env.set_params(params)
+
+ Logger.info("Before starting Stack Upgrade, check if tez tarball has been
copied to HDFS.")
+
+ if params.stack_version_formatted and
check_stack_feature(StackFeature.ROLLING_UPGRADE,
params.stack_version_formatted):
+ Logger.info("Stack version {0} is sufficient to check if need to copy
tez.tar.gz to HDFS.".format(params.stack_version_formatted))
+
+ # Force it to copy the current version of the tez tarball, rather than
the version the RU will go to.
+ resource_created = copy_to_hdfs(
+ "tez",
+ params.user_group,
+ params.hdfs_user,
+ use_upgrading_version_during_upgrade=False,
+ skip=params.sysprep_skip_copy_tarballs_hdfs)
+ if resource_created:
+ params.HdfsResource(None, action="execute")
+ else:
+ raise Fail("Could not copy tez tarball to HDFS.")
+
+if __name__ == "__main__":
+ TezPreUpgrade().execute()
+
diff --git
a/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/service_check.py
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/service_check.py
new file mode 100644
index 00000000..3fe4e3fc
--- /dev/null
+++
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/service_check.py
@@ -0,0 +1,126 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+import os
+from resource_management.libraries.script import Script
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.resources.execute_hadoop import
ExecuteHadoop
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import
check_stack_feature
+from resource_management.libraries.functions.copy_tarball import copy_to_hdfs
+from resource_management.core.resources.system import File, Execute
+
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyImpl
+
+from resource_management.core.logger import Logger
+
+class TezServiceCheck(Script):
+ pass
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class TezServiceCheckLinux(TezServiceCheck):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+
+ path_to_tez_jar = format(params.tez_examples_jar)
+ wordcount_command = format("jar {path_to_tez_jar} orderedwordcount
/tmp/tezsmokeinput/sample-tez-test /tmp/tezsmokeoutput/")
+ test_command = format("fs -test -e /tmp/tezsmokeoutput/_SUCCESS")
+
+ File(format("{tmp_dir}/sample-tez-test"),
+ content = "foo\nbar\nfoo\nbar\nfoo",
+ mode = 0755
+ )
+
+ params.HdfsResource("/tmp/tezsmokeoutput",
+ action = "delete_on_execute",
+ type = "directory"
+ )
+
+ params.HdfsResource("/tmp/tezsmokeinput",
+ action = "create_on_execute",
+ type = "directory",
+ owner = params.smokeuser,
+ )
+ params.HdfsResource("/tmp/tezsmokeinput/sample-tez-test",
+ action = "create_on_execute",
+ type = "file",
+ owner = params.smokeuser,
+ source = format("{tmp_dir}/sample-tez-test"),
+ )
+
+ if params.stack_version_formatted and
check_stack_feature(StackFeature.ROLLING_UPGRADE,
params.stack_version_formatted):
+ copy_to_hdfs("tez", params.user_group, params.hdfs_user,
skip=params.sysprep_skip_copy_tarballs_hdfs)
+ else:
+ # If the directory already exists, it is a NO-OP
+ params.HdfsResource(params.tez_lib_base_dir_path,
+ type="directory",
+ action="create_on_execute",
+ owner=params.smokeuser,
+ )
+ # If the file already exists, it is a NO-OP
+ params.HdfsResource(params.tez_lib_uris,
+ action = "create_on_execute",
+ type = "file",
+ owner = params.smokeuser,
+ source = format("{tez_home}/lib/tez.tar.gz"),
+ )
+
+
+ params.HdfsResource(None, action = "execute")
+
+ if params.security_enabled:
+ kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab}
{smokeuser_principal};")
+ Execute(kinit_cmd,
+ user=params.smokeuser
+ )
+
+ ExecuteHadoop(wordcount_command,
+ tries = 3,
+ try_sleep = 5,
+ user = params.smokeuser,
+ conf_dir = params.hadoop_conf_dir,
+ bin_dir = params.hadoop_bin_dir
+ )
+
+ ExecuteHadoop(test_command,
+ tries = 10,
+ try_sleep = 6,
+ user = params.smokeuser,
+ conf_dir = params.hadoop_conf_dir,
+ bin_dir = params.hadoop_bin_dir
+ )
+
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class TezServiceCheckWindows(TezServiceCheck):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+ smoke_cmd = os.path.join(params.stack_root,"Run-SmokeTests.cmd")
+ service = "TEZ"
+ Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True,
user=params.tez_user)
+
+
+if __name__ == "__main__":
+ TezServiceCheck().execute()
+
diff --git
a/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/tez.py
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/tez.py
new file mode 100644
index 00000000..dfa40821
--- /dev/null
+++
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/tez.py
@@ -0,0 +1,83 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+# Python Imports
+import os
+
+# Local Imports
+from resource_management.core.resources.system import Directory, File
+from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions import lzo_utils
+from resource_management.core.source import InlineTemplate
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def tez(config_dir):
+ """
+ Write out tez-site.xml and tez-env.sh to the config directory.
+ :param config_dir: Which config directory to save configs to, which is
different during rolling upgrade.
+ """
+ import params
+
+ # ensure that matching LZO libraries are installed for Tez
+ lzo_utils.install_lzo_if_needed()
+
+ if config_dir is None:
+ config_dir = params.config_dir
+
+ Directory(params.tez_etc_dir, mode=0755)
+
+ Directory(config_dir,
+ owner = params.tez_user,
+ group = params.user_group,
+ create_parents = True)
+
+ XmlConfig( "tez-site.xml",
+ conf_dir = config_dir,
+ configurations = params.tez_site_config,
+
configuration_attributes=params.config['configurationAttributes']['tez-site'],
+ owner = params.tez_user,
+ group = params.user_group,
+ mode = 0664)
+
+ tez_env_file_path = os.path.join(config_dir, "tez-env.sh")
+ File(tez_env_file_path,
+ owner=params.tez_user,
+ content=InlineTemplate(params.tez_env_sh_template),
+ mode=0555)
+
+
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def tez(config_dir):
+ """
+ Write out tez-site.xml and tez-env.sh to the config directory.
+ :param config_dir: Directory to write configs to.
+ """
+ import params
+ XmlConfig("tez-site.xml",
+ conf_dir=config_dir,
+ configurations=params.config['configurations']['tez-site'],
+ owner=params.tez_user,
+ mode="f",
+
configuration_attributes=params.config['configurationAttributes']['tez-site']
+ )
+
diff --git
a/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/tez_client.py
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/tez_client.py
new file mode 100644
index 00000000..b3e689a5
--- /dev/null
+++
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/package/scripts/tez_client.py
@@ -0,0 +1,130 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+import os
+import urlparse
+
+from ambari_commons import OSConst
+from ambari_commons.inet_utils import download_file
+from ambari_commons.os_family_impl import OsFamilyImpl
+from ambari_commons.os_utils import copy_file, extract_path_component
+
+from resource_management.core.exceptions import ClientComponentHasNoStatus
+from resource_management.core.source import InlineTemplate
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import
check_stack_feature
+from resource_management.libraries.functions.get_stack_version import
get_stack_version
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions.default import default
+from resource_management.core.logger import Logger
+
+from tez import tez
+
+class TezClient(Script):
+
+ def configure(self, env, config_dir=None, upgrade_type=None):
+ """
+ Write tez-site.xml and tez-env.sh to the config directory
+ :param env: Python Environment
+ :param config_dir: During rolling upgrade, which config directory to save
configs to.
+ E.g., /usr/$STACK/current/tez-client/conf
+ """
+ import params
+ env.set_params(params)
+ tez(config_dir)
+
+ def status(self, env):
+ raise ClientComponentHasNoStatus()
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class TezClientLinux(TezClient):
+
+ def stack_upgrade_save_new_config(self, env):
+ """
+ Because this gets called during a Rolling Upgrade, the new tez configs
have already been saved, so we must be
+ careful to only call configure() on the directory of the new version.
+ :param env:
+ """
+ import params
+ env.set_params(params)
+
+ conf_select_name = "tez"
+ base_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+ config_dir = self.get_config_dir_during_stack_upgrade(env, base_dir,
conf_select_name)
+
+ if config_dir:
+ Logger.info("stack_upgrade_save_new_config(): Calling conf-select on %s
using version %s" % (conf_select_name, str(params.version)))
+
+ # Because this script was called from ru_execute_tasks.py which already
enters an Environment with its own basedir,
+ # must change it now so this function can find the Jinja Templates for
the service.
+ env.config.basedir = base_dir
+ self.configure(env, config_dir=config_dir)
+
+ def pre_upgrade_restart(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE,
params.version):
+ stack_select.select_packages(params.version)
+
+ def install(self, env):
+ import params
+ self.install_packages(env)
+ self.configure(env, config_dir=params.config_dir)
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class TezClientWindows(TezClient):
+ def install(self, env):
+ import params
+ if params.tez_home_dir is None:
+ self.install_packages(env)
+ params.refresh_tez_state_dependent_params()
+ env.set_params(params)
+ self._install_lzo_support_if_needed(params)
+ self.configure(env, config_dir=params.tez_conf_dir)
+
+ def _install_lzo_support_if_needed(self, params):
+ hadoop_classpath_prefix =
self._expand_hadoop_classpath_prefix(params.hadoop_classpath_prefix_template,
params.tez_site_config)
+
+ hadoop_lzo_dest_path = extract_path_component(hadoop_classpath_prefix,
"hadoop-lzo-")
+ if hadoop_lzo_dest_path:
+ hadoop_lzo_file = os.path.split(hadoop_lzo_dest_path)[1]
+
+ config = Script.get_config()
+ file_url = urlparse.urljoin(config['ambariLevelParams']['jdk_location'],
hadoop_lzo_file)
+ hadoop_lzo_dl_path =
os.path.join(config["agentLevelParams"]["agentCacheDir"], hadoop_lzo_file)
+ download_file(file_url, hadoop_lzo_dl_path)
+ #This is for protection against configuration changes. It will infect
every new destination with the lzo jar,
+ # but since the classpath points to the jar directly we're getting away
with it.
+ if not os.path.exists(hadoop_lzo_dest_path):
+ copy_file(hadoop_lzo_dl_path, hadoop_lzo_dest_path)
+
+ def _expand_hadoop_classpath_prefix(self, hadoop_classpath_prefix_template,
configurations):
+ import resource_management
+
+ hadoop_classpath_prefix_obj =
InlineTemplate(hadoop_classpath_prefix_template,
configurations_dict=configurations,
+
extra_imports=[resource_management, resource_management.core,
+
resource_management.core.source])
+ hadoop_classpath_prefix = hadoop_classpath_prefix_obj.get_content()
+ return hadoop_classpath_prefix
+
+if __name__ == "__main__":
+ TezClient().execute()
diff --git
a/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/role_command_order.json
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/role_command_order.json
new file mode 100644
index 00000000..05db0923
--- /dev/null
+++
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/role_command_order.json
@@ -0,0 +1,6 @@
+{
+ "general_deps": {
+ "_comment": "dependencies for TEZ",
+ "TEZ_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START",
"RESOURCEMANAGER-START"]
+ }
+}
\ No newline at end of file
diff --git
a/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/service_advisor.py
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/service_advisor.py
new file mode 100644
index 00000000..684b6de5
--- /dev/null
+++
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/service_advisor.py
@@ -0,0 +1,343 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+# Python imports
+import imp
+import os
+import traceback
+import re
+import socket
+import fnmatch
+import subprocess
+
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../../../stacks/')
+PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
+
+try:
+ if "BASE_SERVICE_ADVISOR" in os.environ:
+ PARENT_FILE = os.environ["BASE_SERVICE_ADVISOR"]
+ with open(PARENT_FILE, 'rb') as fp:
+ service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE,
('.py', 'rb', imp.PY_SOURCE))
+except Exception as e:
+ traceback.print_exc()
+ print "Failed to load parent"
+
+class TezServiceAdvisor(service_advisor.ServiceAdvisor):
+
+ def __init__(self, *args, **kwargs):
+ self.as_super = super(TezServiceAdvisor, self)
+ self.as_super.__init__(*args, **kwargs)
+
+ self.initialize_logger("TezServiceAdvisor")
+
+ # Always call these methods
+ self.modifyMastersWithMultipleInstances()
+ self.modifyCardinalitiesDict()
+ self.modifyHeapSizeProperties()
+ self.modifyNotValuableComponents()
+ self.modifyComponentsNotPreferableOnServer()
+ self.modifyComponentLayoutSchemes()
+
+ def modifyMastersWithMultipleInstances(self):
+ """
+ Modify the set of masters with multiple instances.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyCardinalitiesDict(self):
+ """
+ Modify the dictionary of cardinalities.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyHeapSizeProperties(self):
+ """
+ Modify the dictionary of heap size properties.
+ Must be overriden in child class.
+ """
+ pass
+
+ def modifyNotValuableComponents(self):
+ """
+ Modify the set of components whose host assignment is based on other
services.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyComponentsNotPreferableOnServer(self):
+ """
+ Modify the set of components that are not preferable on the server.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyComponentLayoutSchemes(self):
+ """
+ Modify layout scheme dictionaries for components.
+ The scheme dictionary basically maps the number of hosts to
+ host index where component should exist.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def getServiceComponentLayoutValidations(self, services, hosts):
+ """
+ Get a list of errors.
+ Must be overriden in child class.
+ """
+
+ return self.getServiceComponentCardinalityValidations(services, hosts,
"TEZ")
+
+ def getServiceConfigurationRecommendations(self, configurations,
clusterData, services, hosts):
+ """
+ Entry point.
+ Must be overriden in child class.
+ """
+ # self.logger.info("Class: %s, Method: %s. Recommending Service
Configurations." %
+ # (self.__class__.__name__, inspect.stack()[0][3]))
+
+ recommender = TezRecommender()
+ recommender.recommendTezConfigurationsFromHDP21(configurations,
clusterData, services, hosts)
+ recommender.recommendTezConfigurationsFromHDP22(configurations,
clusterData, services, hosts)
+ recommender.recommendTezConfigurationsFromHDP23(configurations,
clusterData, services, hosts)
+ recommender.recommendTezConfigurationsFromHDP30(configurations,
clusterData, services, hosts)
+
+
+ def getServiceConfigurationsValidationItems(self, configurations,
recommendedDefaults, services, hosts):
+ """
+ Entry point.
+ Validate configurations for the service. Return a list of errors.
+ The code for this function should be the same for each Service Advisor.
+ """
+ # self.logger.info("Class: %s, Method: %s. Validating Configurations." %
+ # (self.__class__.__name__, inspect.stack()[0][3]))
+
+ validator = TezValidator()
+ # Calls the methods of the validator using arguments,
+ # method(siteProperties, siteRecommendations, configurations, services,
hosts)
+ return validator.validateListOfConfigUsingMethod(configurations,
recommendedDefaults, services, hosts, validator.validators)
+
+
+
+class TezRecommender(service_advisor.ServiceAdvisor):
+ """
+ Tez Recommender suggests properties when adding the service for the first
time or modifying configs via the UI.
+ """
+
+ def __init__(self, *args, **kwargs):
+ self.as_super = super(TezRecommender, self)
+ self.as_super.__init__(*args, **kwargs)
+
+ def recommendTezConfigurationsFromHDP21(self, configurations, clusterData,
services, hosts):
+ putTezProperty = self.putProperty(configurations, "tez-site")
+ putTezProperty("tez.am.resource.memory.mb", int(clusterData['amMemory']))
+ putTezProperty("tez.am.java.opts",
+ "-server -Xmx" + str(int(0.8 * clusterData["amMemory"]))
+ + "m -Djava.net.preferIPv4Stack=true")
+ recommended_tez_queue = self.recommendYarnQueue(services, "tez-site",
"tez.queue.name")
+ if recommended_tez_queue is not None:
+ putTezProperty("tez.queue.name", recommended_tez_queue)
+
+
+ def recommendTezConfigurationsFromHDP22(self, configurations, clusterData,
services, hosts):
+ if not "yarn-site" in configurations:
+ self.calculateYarnAllocationSizes(configurations, services, hosts)
+ #properties below should be always present as they are provided in
HDP206 stack advisor
+ yarnMaxAllocationSize = min(30 *
int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]),
int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
+
+ putTezProperty = self.putProperty(configurations, "tez-site", services)
+ putTezProperty("tez.am.resource.memory.mb",
min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]),
int(clusterData['amMemory']) * 2 if int(clusterData['amMemory']) < 3072 else
int(clusterData['amMemory'])))
+
+ taskResourceMemory = clusterData['mapMemory'] if clusterData['mapMemory']
> 2048 else int(clusterData['reduceMemory'])
+ taskResourceMemory = min(clusterData['containers'] *
clusterData['ramPerContainer'], taskResourceMemory, yarnMaxAllocationSize)
+ putTezProperty("tez.task.resource.memory.mb",
min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]),
taskResourceMemory))
+ taskResourceMemory =
int(configurations["tez-site"]["properties"]["tez.task.resource.memory.mb"])
+ putTezProperty("tez.runtime.io.sort.mb", min(int(taskResourceMemory *
0.4), 2047))
+ putTezProperty("tez.runtime.unordered.output.buffer.size-mb",
int(taskResourceMemory * 0.075))
+ putTezProperty("tez.session.am.dag.submit.timeout.secs", "600")
+
+ tez_queue = self.recommendYarnQueue(services, "tez-site", "tez.queue.name")
+ if tez_queue is not None:
+ putTezProperty("tez.queue.name", tez_queue)
+
+ serverProperties = services["ambari-server-properties"]
+ latest_tez_jar_version = None
+
+ server_host = socket.getfqdn()
+ for host in hosts["items"]:
+ if server_host == host["Hosts"]["host_name"]:
+ server_host = host["Hosts"]["public_host_name"]
+ server_port = '8080'
+ server_protocol = 'http'
+ views_dir = '/var/lib/ambari-server/resources/views/'
+
+ has_tez_view = False
+ if serverProperties:
+ if 'client.api.port' in serverProperties:
+ server_port = serverProperties['client.api.port']
+ if 'views.dir' in serverProperties:
+ views_dir = serverProperties['views.dir']
+ if 'api.ssl' in serverProperties:
+ if serverProperties['api.ssl'].lower() == 'true':
+ server_protocol = 'https'
+
+ views_work_dir = os.path.join(views_dir, 'work')
+
+ if os.path.exists(views_work_dir) and os.path.isdir(views_work_dir):
+ for file in os.listdir(views_work_dir):
+ if fnmatch.fnmatch(file, 'TEZ{*}'):
+ has_tez_view = True # now used just to verify if the tez view
exists
+ pass
+ pass
+ pass
+ pass
+
+ if has_tez_view:
+ tez_url =
'{0}://{1}:{2}/#/main/view/TEZ/tez_cluster_instance'.format(server_protocol,
server_host, server_port)
+ putTezProperty("tez.tez-ui.history-url.base", tez_url)
+ pass
+
+
+ def recommendTezConfigurationsFromHDP23(self, configurations, clusterData,
services, hosts):
+
+ putTezProperty = self.putProperty(configurations, "tez-site")
+
+ if "HIVE" in self.getServiceNames(services):
+ if not "hive-site" in configurations:
+ self.recommendHIVEConfigurations(configurations, clusterData,
services, hosts)
+
+ if "hive-site" in configurations and "hive.tez.container.size" in
configurations["hive-site"]["properties"]:
+ putTezProperty("tez.task.resource.memory.mb",
configurations["hive-site"]["properties"]["hive.tez.container.size"])
+
+ # remove 2gb limit for tez.runtime.io.sort.mb
+ # in HDP 2.3 "tez.runtime.sorter.class" is set by default to PIPELINED, in
other case comment calculation code below
+ taskResourceMemory =
int(configurations["tez-site"]["properties"]["tez.task.resource.memory.mb"])
+ # fit io.sort.mb into tenured regions
+ putTezProperty("tez.runtime.io.sort.mb", int(taskResourceMemory * 0.8 *
0.33))
+
+ if "tez-site" in services["configurations"] and "tez.runtime.sorter.class"
in services["configurations"]["tez-site"]["properties"]:
+ if
services["configurations"]["tez-site"]["properties"]["tez.runtime.sorter.class"]
== "LEGACY":
+ putTezAttribute = self.putPropertyAttribute(configurations, "tez-site")
+ putTezAttribute("tez.runtime.io.sort.mb", "maximum", 1800)
+ pass
+
+ serverProperties = services["ambari-server-properties"]
+ latest_tez_jar_version = None
+
+ server_host = socket.getfqdn()
+ for host in hosts["items"]:
+ if server_host == host["Hosts"]["host_name"]:
+ server_host = host["Hosts"]["public_host_name"]
+ server_port = '8080'
+ server_protocol = 'http'
+ views_dir = '/var/lib/ambari-server/resources/views/'
+
+ has_tez_view = False
+ if serverProperties:
+ if 'client.api.port' in serverProperties:
+ server_port = serverProperties['client.api.port']
+ if 'views.dir' in serverProperties:
+ views_dir = serverProperties['views.dir']
+ if 'api.ssl' in serverProperties:
+ if serverProperties['api.ssl'].lower() == 'true':
+ server_protocol = 'https'
+
+ views_work_dir = os.path.join(views_dir, 'work')
+
+ if os.path.exists(views_work_dir) and os.path.isdir(views_work_dir):
+ for file in os.listdir(views_work_dir):
+ if fnmatch.fnmatch(file, 'TEZ{*}'):
+ has_tez_view = True # now used just to verify if the tez view
exists
+ pass
+ pass
+ pass
+ pass
+
+ if has_tez_view:
+ tez_url =
'{0}://{1}:{2}/#/main/view/TEZ/tez_cluster_instance'.format(server_protocol,
server_host, server_port)
+ putTezProperty("tez.tez-ui.history-url.base", tez_url)
+ pass
+
+ def recommendTezConfigurationsFromHDP30(self, configurations, clusterData,
services, hosts):
+ putTezProperty = self.putProperty(configurations, "tez-site")
+ if "HIVE" in self.getServiceNames(services) and "hive-site" in
services["configurations"] and "hive.metastore.warehouse.external.dir" in
services["configurations"]["hive-site"]["properties"]:
+ hive_metastore_warehouse_external_dir =
services["configurations"]["hive-site"]["properties"]['hive.metastore.warehouse.external.dir']
+ putTezProperty("tez.history.logging.proto-base-dir",
"{0}/sys.db".format(hive_metastore_warehouse_external_dir))
+ putTezProperty("tez.history.logging.service.class",
"org.apache.tez.dag.history.logging.proto.ProtoHistoryLoggingService")
+ self.logger.info("Updated 'tez-site' config
'tez.history.logging.proto-base-dir' and 'tez.history.logging.service.class'")
+
+
+class TezValidator(service_advisor.ServiceAdvisor):
+ """
+ Tez Validator checks the correctness of properties whenever the service is
first added or the user attempts to
+ change configs via the UI.
+ """
+
+ def __init__(self, *args, **kwargs):
+ self.as_super = super(TezValidator, self)
+ self.as_super.__init__(*args, **kwargs)
+
+ self.validators = [("tez-site", self.validateTezConfigurationsFromHDP21),
+ ("tez-site", self.validateTezConfigurationsFromHDP22)]
+
+
+ def validateTezConfigurationsFromHDP21(self, properties,
recommendedDefaults, configurations, services, hosts):
+ validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item":
self.validatorLessThenDefaultValue(properties, recommendedDefaults,
'tez.am.resource.memory.mb')},
+ {"config-name": 'tez.am.java.opts', "item":
self.validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')},
+ {"config-name": 'tez.queue.name', "item":
self.validatorYarnQueue(properties, recommendedDefaults, 'tez.queue.name',
services)} ]
+ return self.toConfigurationValidationProblems(validationItems, "tez-site")
+
+
+ def validateTezConfigurationsFromHDP22(self, properties,
recommendedDefaults, configurations, services, hosts):
+ validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item":
self.validatorLessThenDefaultValue(properties, recommendedDefaults,
'tez.am.resource.memory.mb')},
+ {"config-name": 'tez.task.resource.memory.mb', "item":
self.validatorLessThenDefaultValue(properties, recommendedDefaults,
'tez.task.resource.memory.mb')},
+ {"config-name": 'tez.runtime.io.sort.mb', "item":
self.validatorLessThenDefaultValue(properties, recommendedDefaults,
'tez.runtime.io.sort.mb')},
+ {"config-name":
'tez.runtime.unordered.output.buffer.size-mb', "item":
self.validatorLessThenDefaultValue(properties, recommendedDefaults,
'tez.runtime.unordered.output.buffer.size-mb')},
+ {"config-name": 'tez.queue.name', "item":
self.validatorYarnQueue(properties, recommendedDefaults, 'tez.queue.name',
services)} ]
+ if "tez.tez-ui.history-url.base" in recommendedDefaults:
+ validationItems.append({"config-name": 'tez.tez-ui.history-url.base',
"item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults,
'tez.tez-ui.history-url.base')})
+
+ tez_site = properties
+ prop_name1 = 'tez.am.resource.memory.mb'
+ prop_name2 = 'tez.task.resource.memory.mb'
+ yarnSiteProperties = self.getSiteProperties(configurations, "yarn-site")
+ if yarnSiteProperties:
+ yarnMaxAllocationSize = min(30 *
int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]),int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
+ if int(tez_site[prop_name1]) > yarnMaxAllocationSize:
+ validationItems.append({"config-name": prop_name1,
+ "item": self.getWarnItem(
+ "{0} should be less than YARN max allocation
size ({1})".format(prop_name1, yarnMaxAllocationSize))})
+ if int(tez_site[prop_name2]) > yarnMaxAllocationSize:
+ validationItems.append({"config-name": prop_name2,
+ "item": self.getWarnItem(
+ "{0} should be less than YARN max allocation
size ({1})".format(prop_name2, yarnMaxAllocationSize))})
+
+ return self.toConfigurationValidationProblems(validationItems, "tez-site")
+
+
+
diff --git
a/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/themes/directories.json
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/themes/directories.json
new file mode 100644
index 00000000..629e82ec
--- /dev/null
+++
b/bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/TEZ/themes/directories.json
@@ -0,0 +1,177 @@
+{
+ "name": "directories",
+ "description": "Directories theme for YARN service",
+ "configuration": {
+ "layouts": [
+ {
+ "name": "directories",
+ "tabs": [
+ {
+ "name": "directories",
+ "display-name": "Directories",
+ "layout": {
+ "tab-columns": "1",
+ "tab-rows": "4",
+ "sections": [
+ {
+ "name": "subsection-data-dirs",
+ "display-name": "DATA DIRS",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-data-dirs",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ },
+ {
+ "name": "subsection-log-dirs",
+ "display-name": "LOG DIRS",
+ "row-index": "1",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-log-dirs",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ },
+ {
+ "name": "subsection-pid-dirs",
+ "display-name": "PID DIRS",
+ "row-index": "2",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-pid-dirs",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ],
+ "placement": {
+ "configuration-layout": "default",
+ "configs": [
+ {
+ "config": "yarn-site/yarn.nodemanager.local-dirs",
+ "subsection-name": "subsection-data-dirs"
+ },
+ {
+ "config": "yarn-site/yarn.nodemanager.log-dirs",
+ "subsection-name": "subsection-log-dirs"
+ },
+ {
+ "config": "yarn-site/yarn.nodemanager.remote-app-log-dir",
+ "subsection-name": "subsection-log-dirs"
+ },
+ {
+ "config": "yarn-env/yarn_pid_dir_prefix",
+ "subsection-name": "subsection-pid-dirs"
+ },
+ {
+ "config": "yarn-env/yarn_log_dir_prefix",
+ "subsection-name": "subsection-log-dirs"
+ },
+ {
+ "config":
"yarn-site/yarn.timeline-service.entity-group-fs-store.active-dir",
+ "subsection-name": "subsection-data-dirs"
+ },
+ {
+ "config": "yarn-site/yarn.node-labels.fs-store.root-dir",
+ "subsection-name": "subsection-data-dirs"
+ },
+ {
+ "config": "yarn-site/yarn.nodemanager.recovery.dir",
+ "subsection-name": "subsection-data-dirs"
+ },
+ {
+ "config":
"yarn-site/yarn.timeline-service.entity-group-fs-store.done-dir",
+ "subsection-name": "subsection-data-dirs"
+ }
+ ]
+ },
+ "widgets": [
+ {
+ "config": "yarn-site/yarn.nodemanager.local-dirs",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "yarn-site/yarn.nodemanager.log-dirs",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "yarn-site/yarn.nodemanager.remote-app-log-dir",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "yarn-env/yarn_pid_dir_prefix",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "yarn-env/yarn_log_dir_prefix",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config":
"yarn-site/yarn.timeline-service.entity-group-fs-store.active-dir",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "yarn-site/yarn.node-labels.fs-store.root-dir",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "yarn-site/yarn.nodemanager.recovery.dir",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config":
"yarn-site/yarn.timeline-service.entity-group-fs-store.done-dir",
+ "widget": {
+ "type": "text-field"
+ }
+ }
+ ]
+ }
+}
