Updated Branches: refs/heads/trunk 8e8a9a47f -> fd9078d8d
AMBARI-4377. Add Tez as a service to 2.1.1 stack. (swagle) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fd9078d8 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fd9078d8 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fd9078d8 Branch: refs/heads/trunk Commit: fd9078d8dcef135a4c02c77dcb2adad51577d6d8 Parents: 8e8a9a4 Author: Siddharth Wagle <[email protected]> Authored: Wed Jan 22 12:53:16 2014 -0800 Committer: Siddharth Wagle <[email protected]> Committed: Wed Jan 22 13:04:29 2014 -0800 ---------------------------------------------------------------------- .../2.1.1/hooks/before-START/scripts/params.py | 2 +- .../before-START/templates/hadoop-env.sh.j2 | 4 + .../PIG/package/templates/pig-env.sh.j2 | 4 + .../services/TEZ/configuration/tez-site.xml | 142 +++++++++++++++++++ .../stacks/HDP/2.1.1/services/TEZ/metainfo.xml | 61 ++++++++ .../services/TEZ/package/scripts/params.py | 34 +++++ .../2.1.1/services/TEZ/package/scripts/tez.py | 53 +++++++ .../services/TEZ/package/scripts/tez_client.py | 51 +++++++ .../TEZ/package/templates/tez-env.sh.j2 | 25 ++++ .../python/stacks/2.1.1/TEZ/test_tez_client.py | 51 +++++++ .../python/stacks/2.1.1/configs/default.json | 6 + 11 files changed, 432 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/fd9078d8/ambari-server/src/main/resources/stacks/HDP/2.1.1/hooks/before-START/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.1.1/hooks/before-START/scripts/params.py index d080a11..63810fa 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/hooks/before-START/scripts/params.py @@ -169,4 +169,4 @@ if dfs_ha_enabled: namenode_id = nn_id namenode_id = None -dfs_hosts = default('/configurations/hdfs-site/dfs.hosts', None) \ No newline at end of file +dfs_hosts = default('/configurations/hdfs-site/dfs.hosts', None) http://git-wip-us.apache.org/repos/asf/ambari/blob/fd9078d8/ambari-server/src/main/resources/stacks/HDP/2.1.1/hooks/before-START/templates/hadoop-env.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/hooks/before-START/templates/hadoop-env.sh.j2 b/ambari-server/src/main/resources/stacks/HDP/2.1.1/hooks/before-START/templates/hadoop-env.sh.j2 index 7d10cc3..1f596a3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/hooks/before-START/templates/hadoop-env.sh.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/hooks/before-START/templates/hadoop-env.sh.j2 @@ -114,6 +114,10 @@ done MAPREDUCE_LIBS={{mapreduce_libs_path}} export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS} +if [ -d "/usr/lib/tez" ]; then + export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/tez/*:/usr/lib/tez/lib/* +fi + # Setting path to hdfs command line export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd9078d8/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/PIG/package/templates/pig-env.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/PIG/package/templates/pig-env.sh.j2 b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/PIG/package/templates/pig-env.sh.j2 index b0e17d4..ad10c21 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/PIG/package/templates/pig-env.sh.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/PIG/package/templates/pig-env.sh.j2 @@ -15,3 +15,7 @@ JAVA_HOME={{java64_home}} HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + +if [ -d "/usr/lib/tez" ]; then + PIG_OPTS="-Dmapreduce.framework.name=yarn" +fi http://git-wip-us.apache.org/repos/asf/ambari/blob/fd9078d8/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml new file mode 100644 index 0000000..94642c7 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml @@ -0,0 +1,142 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + + <property> + <name>tez.lib.uris</name> + <value>file:///usr/lib/tez/,file:///usr/lib/tez/lib/</value> + </property> + + <property> + <name>tez.am.log.level</name> + <value>INFO</value> + </property> + + <property> + <name>tez.staging-dir</name> + <value>/tmp/${user.name}/staging</value> + </property> + + <property> + <name>tez.slowstart-vertex-scheduler.min-src-fraction</name> + <value>0.1</value> + </property> + + <property> + <name>tez.slowstart-vertex-scheduler.max-src-fraction</name> + <value>0.1</value> + </property> + + <property> + <name>tez.am.am-rm.heartbeat.interval-ms.max</name> + <value>250</value> + </property> + + <property> + <name>tez.runtime.broadcast.data-via-events.enabled</name> + <value>true</value> + </property> + + <property> + <name>tez.runtime.broadcast.data-via-events.max-size</name> + <value>614400</value> + </property> + + <property> + <name>tez.am.aggressive.scheduling</name> + <value>false</value> + </property> + + <property> + <name>tez.am.resource.memory.mb</name> + <value>1024</value> + </property> + + <property> + <name>tez.am.java.opts</name> + <value>-server -Xmx1024m -Djava.net.preferIPv4Stack=true -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC</value> + </property> + + <property> + <name>tez.am.grouping.split-waves</name> + <value>1.7</value> + </property> + + <property> + <name>tez.am.container.reuse.enabled</name> + <value>true</value> + </property> + + <property> + <name>tez.am.container.reuse.rack-fallback.enabled</name> + <value>true</value> + </property> + + <property> + <name>tez.am.container.reuse.non-local-fallback.enabled</name> + <value>true</value> + </property> + + <property> + <name>tez.am.container.session.delay-allocation-millis</name> + <value>300000</value> + </property> + + <property> + <name>tez.am.container.reuse.locality.delay-allocation-millis</name> + <value>250</value> + </property> + + <property> + <name>tez.runtime.intermediate-output.should-compress</name> + <value>true</value> + </property> + + <property> + <name>tez.runtime.intermediate-output.compress.codec</name> + <value>org.apache.hadoop.io.compress.DefaultCodec</value> + </property> + + <property> + <name>tez.runtime.intermdiate-input.is-compressed</name> + <value>true</value> + </property> + + <property> + <name>tez.runtime.intermediate-input.compress.codec</name> + <value>org.apache.hadoop.io.compress.DefaultCodec</value> + </property> + + <property> + <name>tez.task.get-task.sleep.interval-ms.max</name> + <value>100</value> + </property> + + <property> + <name>tez.runtime.job.counters.max</name> + <value>5000</value> + </property> + + <property> + <name>tez.runtime.job.counters.groups.max</name> + <value>5000</value> + </property> + +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/fd9078d8/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/metainfo.xml new file mode 100644 index 0000000..cbd1a35 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/metainfo.xml @@ -0,0 +1,61 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>TEZ</name> + <comment>Tez is the next generation Hadoop Query Processing framework written on top of YARN.</comment> + <version>0.4.0.2.1.1.0</version> + <components> + <component> + <name>TEZ_CLIENT</name> + <category>CLIENT</category> + <commandScript> + <script>scripts/tez_client.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + </components> + <osSpecifics> + <osSpecific> + <osType>any</osType> + <packages> + <package> + <type>rpm</type> + <name>tez</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <configuration-dependencies> + <config-type>global</config-type> + <config-type>tez-site.xml</config-type> + </configuration-dependencies> + + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/fd9078d8/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/params.py new file mode 100644 index 0000000..316c777 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/params.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +# server configurations +config = Script.get_config() + +conf_dir = "/etc/tez/conf" +pid_dir = default('/configurations/global/tez_pid_dir', '/var/run/hadoop-tez/') +log_dir = default('/configurations/global/tez_log_dir', '/var/log/hadoop-tez/') + +hadoop_home = '/usr' +java64_home = config['hostLevelParams']['java_home'] + +tez_user = default('/configurations/global/tez_user', 'tez') +user_group = config['configurations']['global']['user_group'] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/fd9078d8/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/tez.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/tez.py b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/tez.py new file mode 100644 index 0000000..42ac9f9 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/tez.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * + +def tez(): + import params + + Directory( params.conf_dir, + owner = params.tez_user, + group = params.user_group + ) + + XmlConfig( "tez-site.xml", + conf_dir = params.conf_dir, + configurations = params.config['configurations']['tez-site'], + owner = params.tez_user, + group = params.user_group, + mode = 0664 + ) + + tez_TemplateConfig( ['tez-env.sh']) + + +def tez_TemplateConfig(name): + import params + + if not isinstance(name, list): + name = [name] + + for x in name: + TemplateConfig( format("{conf_dir}/{x}"), + owner = params.tez_user + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/fd9078d8/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/tez_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/tez_client.py b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/tez_client.py new file mode 100644 index 0000000..5c8c4aa --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/tez_client.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import sys +from resource_management import * +from tez import tez + +class TezClient(Script): + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + tez() + + def status(self, env): + raise ClientComponentHasNoStatus() + +#for tests +def main(): + command_type = 'install' + command_data_file = '/root/workspace/Pig/input.json' + basedir = '/root/workspace/Pig/main' + sys.argv = ["", command_type, command_data_file, basedir] + + TezClient().execute() + +if __name__ == "__main__": + #main() + TezClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/fd9078d8/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/templates/tez-env.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/templates/tez-env.sh.j2 b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/templates/tez-env.sh.j2 new file mode 100644 index 0000000..6fbadff --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/templates/tez-env.sh.j2 @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Tez specific configuration +export TEZ_LOG_DIR={{log_dir}} +export TEZ_PID_DIR={{pid_dir}} +export TEZ_CONF_DIR={{conf_dir}} + +# Set HADOOP_HOME to point to a specific hadoop install directory +export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + +# The java implementation to use. +export JAVA_HOME={{java64_home}} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/fd9078d8/ambari-server/src/test/python/stacks/2.1.1/TEZ/test_tez_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1.1/TEZ/test_tez_client.py b/ambari-server/src/test/python/stacks/2.1.1/TEZ/test_tez_client.py new file mode 100644 index 0000000..21a3700 --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.1.1/TEZ/test_tez_client.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +from mock.mock import MagicMock, call, patch +from stacks.utils.RMFTestCase import * + +class TestTezClient(RMFTestCase): + + def test_configure_defualt(self): + self.executeScript("2.1.1/services/TEZ/package/scripts/tez_client.py", + classname = "TezClient", + command = "configure", + config_file="default.json" + ) + + self.assertResourceCalled('Directory', '/etc/tez/conf', + owner = 'tez', + group = 'hadoop' + ) + + self.assertResourceCalled('XmlConfig', 'tez-site.xml', + owner = 'tez', + group = 'hadoop', + conf_dir = '/etc/tez/conf', + configurations = self.getConfig()['configurations']['tez-site'], + mode = 0664 + ) + + self.assertResourceCalled('TemplateConfig', '/etc/tez/conf/tez-env.sh', + owner = 'tez' + ) + + self.assertNoMoreResources() + + http://git-wip-us.apache.org/repos/asf/ambari/blob/fd9078d8/ambari-server/src/test/python/stacks/2.1.1/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1.1/configs/default.json b/ambari-server/src/test/python/stacks/2.1.1/configs/default.json index b7062a1..9069853 100644 --- a/ambari-server/src/test/python/stacks/2.1.1/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.1.1/configs/default.json @@ -475,6 +475,12 @@ "yarn.application.classpath": "/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*", "yarn.resourcemanager.admin.address": "c6402.ambari.apache.org:8141", "yarn.nodemanager.aux-services.mapreduce_shuffle.class": "org.apache.hadoop.mapred.ShuffleHandler" + }, + "tez-site": { + "tez.am.log.level": "WARN", + "tez.lib.uris": "file:///usr/lib/tez/,file:///usr/lib/tez/lib/", + "tez.staging-dir": "/tmp/${user.name}/staging", + "tez.am.am-rm.heartbeat.interval-ms.max": "250" } }, "configurationTags": {
