Repository: bigtop Updated Branches: refs/heads/master bc8a256df -> 0c0e2fca4
BIGTOP-1657. bigtop-deploy puppets to support tez Signed-off-by: Olaf Flebbe <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/0c0e2fca Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/0c0e2fca Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/0c0e2fca Branch: refs/heads/master Commit: 0c0e2fca427ec30952eccf2019a7b0efb6b68b45 Parents: bc8a256 Author: Evans Ye <[email protected]> Authored: Wed May 6 03:35:40 2015 +0000 Committer: Olaf Flebbe <[email protected]> Committed: Thu May 7 21:41:00 2015 +0200 ---------------------------------------------------------------------- .../puppet/hieradata/bigtop/cluster.yaml | 4 ++ bigtop-deploy/puppet/hieradata/site.yaml | 1 + bigtop-deploy/puppet/manifests/cluster.pp | 8 ++++ .../modules/hadoop-hive/templates/hive-site.xml | 7 +++ .../puppet/modules/hadoop/manifests/init.pp | 2 +- .../modules/hadoop/templates/hadoop-env.sh | 12 +++-- .../modules/hadoop/templates/mapred-site.xml | 17 +++++++ .../puppet/modules/tez/manifests/init.pp | 22 +++++++++ bigtop-deploy/puppet/modules/tez/tests/init.pp | 17 +++++++ .../src/common/bigtop-utils/init-hcfs.groovy | 48 ++++++++++++++++++-- .../src/common/hadoop/init-hcfs.json | 5 +- 11 files changed, 134 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml index 1ea6407..44cb1b7 100644 --- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml +++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml @@ -137,3 +137,7 @@ hue::server::kerberos_realm: "%{hiera('kerberos::site::realm')}" giraph::client::zookeeper_quorum: "%{hiera('bigtop::hadoop_head_node')}" hadoop-hive::client::hbase_zookeeper_quorum: "%{hiera('hadoop-hbase::common_config::zookeeper_quorum')}" + +# tez +hadoop::common::tez_conf_dir: "/etc/tez/conf" +hadoop::common::tez_jars: "/usr/lib/tez" http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/hieradata/site.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml index 54060f8..00861c4 100644 --- a/bigtop-deploy/puppet/hieradata/site.yaml +++ b/bigtop-deploy/puppet/hieradata/site.yaml @@ -29,6 +29,7 @@ hadoop::hadoop_storage_dirs: # - spark # - sqoop # - tachyon +# - tez # - yarn # - zookeeper http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/manifests/cluster.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp index 43e090f..35ef195 100644 --- a/bigtop-deploy/puppet/manifests/cluster.pp +++ b/bigtop-deploy/puppet/manifests/cluster.pp @@ -147,6 +147,11 @@ if ($hadoop_security_authentication == "kerberos") { include hadoop-zookeeper::server } + if ($all or "tez" in $components) { + include tez::client + Class['tez::client'] -> Exec<| title == "init hdfs" |> + } + # class hadoop::rsync_hdfs isn't used anywhere #Exec<| title == "init hdfs" |> -> Class['Hadoop::Rsync_hdfs'] } @@ -203,4 +208,7 @@ class hadoop_gateway_node inherits hadoop_cluster_node { if ($all or "zookeeper" in $components) { include hadoop-zookeeper::client } + if ($all or "tez" in $components) { + include tez::client + } } http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml b/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml index 5861d97..4e33907 100644 --- a/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml +++ b/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml @@ -42,6 +42,13 @@ </property> <% end %> +<% if @all or @components.include? "tez" -%> +<property> + <name>hive.execution.engine</name> + <value>tez</value> +</property> +<% end %> + <property> <name>javax.jdo.option.ConnectionURL</name> <value>jdbc:derby:;databaseName=/var/lib/hive/metastore/metastore_db;create=true</value> http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp index 45168ba..749773d 100644 --- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp @@ -294,7 +294,7 @@ class hadoop ($hadoop_security_authentication = "simple", $mapreduce_jobhistory_host = undef, $mapreduce_jobhistory_port = "10020", $mapreduce_jobhistory_webapp_port = "19888", - $mapreduce_framework_name = "yarn", + $mapreduce_framework_name = undef, $mapred_data_dirs = suffix($hadoop::hadoop_storage_dirs, "/mapred"), $mapreduce_cluster_temp_dir = "/mapred/system", $yarn_app_mapreduce_am_staging_dir = "/user", http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh b/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh index 318589f..f959f5b 100644 --- a/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh +++ b/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh @@ -34,10 +34,6 @@ # The java implementation to use. Required. <%= shell_config("JAVA_HOME", "hadoop_java_home") %> -# tez environment, needed to enable tez -<%= shell_config("TEZ_CONF_DIR") %> -<%= shell_config("TEZ_JARS") %> - # Extra Java CLASSPATH elements. Optional. <%= shell_config("HADOOP_CLASSPATH") %> @@ -84,6 +80,14 @@ # The scheduling priority for daemon processes. See 'man nice'. <%= shell_config("HADOOP_NICENESS") %> +# tez environment, needed to enable tez +<% if (@all or @components.include? "tez") -%> +<%= shell_config("TEZ_CONF_DIR") %> +<%= shell_config("TEZ_JARS") %> +# Add tez into HADOOP_CLASSPATH +export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${TEZ_CONF_DIR}:${TEZ_JARS}/*:${TEZ_JARS}/lib/* +<% end -%> + ### WARNING: the following is NOT really optional. It is a shame that stock Hadoop ### hadoop_env.sh doesn't make it clear -- you can NOT turn com.sun.management.jmxremote off ### and have a working Hadoop cluster. http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml index 1c8f3fd..2723e82 100644 --- a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml +++ b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml @@ -86,11 +86,28 @@ </property> <% end %> +<% if defined? @mapreduce_framework_name -%> <property> <name>mapreduce.framework.name</name> <value><%= @mapreduce_framework_name %></value> </property> +<!-- If not specified, we automatically choose the best framework for users --> +<% else -%> +<% if @all or @components.include? "tez" -%> + <property> + <name>mapreduce.framework.name</name> + <value>yarn-tez</value> + </property> + +<% else -%> + <property> + <name>mapreduce.framework.name</name> + <value>yarn</value> + </property> + +<% end -%> +<% end -%> <property> <name>mapreduce.cluster.local.dir</name> <value><%= @mapred_data_dirs.join(",") %></value> http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/modules/tez/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/tez/manifests/init.pp b/bigtop-deploy/puppet/modules/tez/manifests/init.pp new file mode 100644 index 0000000..6db2220 --- /dev/null +++ b/bigtop-deploy/puppet/modules/tez/manifests/init.pp @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class tez { + class client { + package { "tez": + ensure => latest, + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/modules/tez/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/tez/tests/init.pp b/bigtop-deploy/puppet/modules/tez/tests/init.pp new file mode 100644 index 0000000..9ee0636 --- /dev/null +++ b/bigtop-deploy/puppet/modules/tez/tests/init.pp @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include tez +tez::client { "test-tez": } http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy b/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy index a224843..d07ef01 100644 --- a/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy +++ b/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy @@ -139,6 +139,9 @@ def final FileSystem fs = FileSystem.get(conf); LOG.info("PROVISIONING WITH FILE SYSTEM : " + fs.getClass()); +// Longest back off time to check whether the file system is ready for write +def final int maxBackOff = 64; + /** * Make a directory. Note when providing input to this functino that if * nulls are given, the commands will work but behaviour varies depending on @@ -150,7 +153,19 @@ LOG.info("PROVISIONING WITH FILE SYSTEM : " + fs.getClass()); * @param group can be null, */ def mkdir = { FileSystem fsys, Path dname, FsPermission mode, String user, String group -> - fsys.mkdirs(dname); + boolean success = false; + for(i = 1; i <= maxBackOff; i*=2) { + try { + success = fsys.mkdirs(dname) + break; + } catch(Exception e) { + LOG.info("Failed to create directory " + dname + "... Retry after " + i + " second(s)"); + Thread.sleep(i*1000); + } + } + if (!success) { + LOG.info("Can not create directory " + dname + " on " + fsys.getClass()); + } if (user != null) { fsys.setOwner(dname, user, group); } @@ -256,8 +271,21 @@ def copyJars = { FileSystem fsys, File input, String jarstr, Path target -> return validJar && filename.contains(jarstr) } }).each({ jar_file -> - copied++; - fsys.copyFromLocalFile(new Path(jar_file.getAbsolutePath()), target) + boolean success = false; + for(i = 1; i <= maxBackOff; i*=2) { + try { + fsys.copyFromLocalFile(new Path(jar_file.getAbsolutePath()), target) + copied++; + success = true; + break; + } catch(Exception e) { + LOG.info("Failed to upload " + jar_file.getAbsolutePath() + " to " + target + "... Retry after " + i + " second(s)"); + Thread.sleep(i*1000); + } + if (!success) { + LOG.info("Can not upload " + jar_file.getAbsolutePath() + " to " + target + " on " + fsys.getClass()); + } + } }); return copied; } @@ -298,4 +326,18 @@ total_jars += copyJars(fs, new File(PIG_HOME), "", new Path(OOZIE_SHARE, "pig")) +LOG.info("Now copying Jars into the DFS for tez "); +LOG.info("This might take a few seconds..."); + +def final TEZ_APPS = "/apps"; +def final TEZ_HOME = "/usr/lib/tez/"; + +total_jars += copyJars(fs, + new File(TEZ_HOME, "lib/"), "", + new Path(TEZ_APPS, "tez/lib")) + +total_jars += copyJars(fs, + new File(TEZ_HOME), "", + new Path(TEZ_APPS, "tez")) + LOG.info("Total jars copied into the DFS : " + total_jars); http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-packages/src/common/hadoop/init-hcfs.json ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hadoop/init-hcfs.json b/bigtop-packages/src/common/hadoop/init-hcfs.json index 88beb1a..7f9b6cc 100644 --- a/bigtop-packages/src/common/hadoop/init-hcfs.json +++ b/bigtop-packages/src/common/hadoop/init-hcfs.json @@ -88,7 +88,10 @@ ["/user/oozie/share/lib/distcp", null, null, null], ["/user/oozie/share/lib/pig", null, null, null], ["/var/log/spark/apps","1777","spark","spark"], - ["/user/hbase","755","hbase",null] + ["/user/hbase","755","hbase",null], + ["/apps",null,null,null], + ["/apps/tez",null,null,null], + ["/apps/tez/lib",null,null,null] ], "user": [ ["tom", "0755", null],
