BIGTOP-958. update deployment code for HCatalog
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/39af29da Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/39af29da Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/39af29da Branch: refs/heads/master Commit: 39af29da43fca8513a1ccca8aa01bc276c62d13a Parents: a36d79e Author: Roman Shaposhnik <[email protected]> Authored: Wed May 1 11:30:38 2013 -0700 Committer: Roman Shaposhnik <[email protected]> Committed: Wed May 1 14:52:08 2013 -0700 ---------------------------------------------------------------------- bigtop-deploy/puppet/manifests/cluster.pp | 8 +- .../puppet/modules/hcatalog/manifests/init.pp | 54 ++++ .../modules/hcatalog/templates/hcatalog-server | 8 + .../puppet/modules/hcatalog/templates/webhcat.xml | 228 +++++++++++++++ .../puppet/modules/hcatalog/tests/init.pp | 17 + 5 files changed, 314 insertions(+), 1 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/39af29da/bigtop-deploy/puppet/manifests/cluster.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp index 605c6a9..11aeb24 100644 --- a/bigtop-deploy/puppet/manifests/cluster.pp +++ b/bigtop-deploy/puppet/manifests/cluster.pp @@ -229,12 +229,18 @@ class hadoop_head_node inherits hadoop_worker_node { kerberos_realm => $kerberos_realm, } - hadoop-sqoop::server { "sqoop server": } hadoop-sqoop::client { "sqoop client": } + hcatalog::server { "hcatalog server": + kerberos_realm => $kerberos_realm, + } + hcatalog::webhcat::server { "webhcat server": + kerberos_realm => $kerberos_realm, + } + hue::server { "hue server": rm_url => $hadoop_rm_url, rm_proxy_url => $hadoop_rm_proxy_url, http://git-wip-us.apache.org/repos/asf/bigtop/blob/39af29da/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp b/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp new file mode 100644 index 0000000..f9c07aa --- /dev/null +++ b/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class hcatalog { + define server($port = "9083", $kerberos_realm = "") { + package { "hcatalog-server": + ensure => latest, + } + + file { "/etc/default/hcatalog-server": + content => template("hcatalog/hcatalog-server"), + require => Package["hcatalog-server"], + } + + service { "hcatalog-server": + ensure => running, + require => [ Package["hcatalog-server"], File["/etc/default/hcatalog-server"] ], + hasrestart => true, + hasstatus => true, + } + } + + class webhcat { + define server($port = "50111", $kerberos_realm = "") { + package { "webhcat-server": + ensure => latest, + } + + file { "/etc/webhcat/conf/webhcat.xml": + content => template("hcatalog/webhcat.xml"), + require => Package["webhcat-server"], + } + + service { "webhcat-server": + ensure => running, + require => [ Package["webhcat-server"], File["/etc/webhcat/conf/webhcat.xml"] ], + hasrestart => true, + hasstatus => true, + } + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/39af29da/bigtop-deploy/puppet/modules/hcatalog/templates/hcatalog-server ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hcatalog/templates/hcatalog-server b/bigtop-deploy/puppet/modules/hcatalog/templates/hcatalog-server new file mode 100644 index 0000000..7291c51 --- /dev/null +++ b/bigtop-deploy/puppet/modules/hcatalog/templates/hcatalog-server @@ -0,0 +1,8 @@ +export METASTORE_PORT=<%= @port %> + +export HADOOP_PREFIX=/usr +export HIVE_HOME=/usr/lib/hive +export HCAT_LOG_DIR=/var/log/hcatalog +export HCAT_PID_DIR=/var/run/hcatalog +#FIXME: workaround for HIVE-2757 (NOTE: HADOOP_HOME is obsolete for Hadoop .23) +export HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop} http://git-wip-us.apache.org/repos/asf/bigtop/blob/39af29da/bigtop-deploy/puppet/modules/hcatalog/templates/webhcat.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hcatalog/templates/webhcat.xml b/bigtop-deploy/puppet/modules/hcatalog/templates/webhcat.xml new file mode 100644 index 0000000..42c5aee --- /dev/null +++ b/bigtop-deploy/puppet/modules/hcatalog/templates/webhcat.xml @@ -0,0 +1,228 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +<!-- The default settings for Templeton. --> +<!-- Edit templeton-site.xml to change settings for your local --> +<!-- install. --> + +<configuration> + + <property> + <name>templeton.port</name> + <value><%= @port %></value> + <description>The HTTP port for the main server.</description> + </property> + + <property> + <name>templeton.hadoop.conf.dir</name> + <value>${env.HADOOP_CONF_DIR}</value> + <description>The path to the Hadoop configuration.</description> + </property> + + <property> + <name>templeton.jar</name> + <value>${env.TEMPLETON_HOME}/share/webhcat/svr/webhcat-0.5.0-SNAPSHOT.jar</value> + <description>The path to the Templeton jar file.</description> + </property> + + <property> + <name>templeton.libjars</name> + <value>${env.TEMPLETON_HOME}/share/webhcat/svr/lib/zookeeper-3.4.3.jar</value> + <description>Jars to add to the classpath.</description> + </property> + + <property> + <name>templeton.override.jars</name> + <value>hdfs:///user/templeton/ugi.jar</value> + <description> + Jars to add the the HADOOP_CLASSPATH for all Map Reduce jobs. + This is a list of jars that must exist on hdfs that are added + to the Distributed Cache. + </description> + </property> + + <property> + <name>templeton.override.enabled</name> + <value>false</value> + <description> + Enable the override path in templeton.override.jars + </description> + </property> + + <property> + <name>templeton.streaming.jar</name> + <value>hdfs:///user/templeton/hadoop-streaming.jar</value> + <description>The hdfs path to the Hadoop streaming jar file.</description> + </property> + + <property> + <name>templeton.hadoop</name> + <value>${env.HADOOP_PREFIX}/bin/hadoop</value> + <description>The path to the Hadoop executable.</description> + </property> + + <property> + <name>templeton.pig.archive</name> + <value>hdfs:///apps/templeton/pig-0.10.1.tar.gz</value> + <description>The path to the Pig archive.</description> + </property> + + <property> + <name>templeton.pig.path</name> + <value>pig-0.10.1.tar.gz/pig-0.10.1/bin/pig</value> + <description>The path to the Pig executable.</description> + </property> + + <property> + <name>templeton.hcat</name> + <value>${env.HCAT_PREFIX}/bin/hcat</value> + <description>The path to the hcatalog executable.</description> + </property> + + <property> + <name>templeton.hive.archive</name> + <value>hdfs:///apps/templeton/hive-0.10.0.tar.gz</value> + <description>The path to the Hive archive.</description> + </property> + + <property> + <name>templeton.hive.path</name> + <value>hive-0.10.0.tar.gz/hive-0.10.0/bin/hive</value> + <description>The path to the Hive executable.</description> + </property> + + <property> + <name>templeton.hive.properties</name> + <value>hive.metastore.local=false,hive.metastore.uris=thrift://localhost:9933,hive.metastore.sasl.enabled=false</value> + <description>Properties to set when running hive.</description> + </property> + + <property> + <name>templeton.exec.encoding</name> + <value>UTF-8</value> + <description>The encoding of the stdout and stderr data.</description> + </property> + + <property> + <name>templeton.exec.timeout</name> + <value>10000</value> + <description> + How long in milliseconds a program is allowed to run on the + Templeton box. + </description> + </property> + + <property> + <name>templeton.exec.max-procs</name> + <value>16</value> + <description>The maximum number of processes allowed to run at once.</description> + </property> + + <property> + <name>templeton.exec.max-output-bytes</name> + <value>1048576</value> + <description> + The maximum number of bytes from stdout or stderr stored in ram. + </description> + </property> + + <property> + <name>templeton.controller.mr.child.opts</name> + <value>-server -Xmx256m -Djava.net.preferIPv4Stack=true</value> + <description>Java options to be passed to templeton controller map task. + The default value of mapreduce child "-Xmx" (heap memory limit) + might be close to what is allowed for a map task. + Even if templeton controller map task does not need much + memory, the jvm (with -server option?) + allocates the max memory when it starts. This along with the + memory used by pig/hive client it starts can end up exceeding + the max memory configured to be allowed for a map task + Use this option to set -Xmx to lower value + </description> + </property> + + <property> + <name>templeton.exec.envs</name> + <value>HADOOP_PREFIX,HADOOP_HOME,JAVA_HOME,HIVE_HOME</value> + <description>The environment variables passed through to exec.</description> + </property> + + <property> + <name>templeton.zookeeper.hosts</name> + <value>127.0.0.1:2181</value> + <description>ZooKeeper servers, as comma separated host:port pairs</description> + </property> + + <property> + <name>templeton.zookeeper.session-timeout</name> + <value>30000</value> + <description>ZooKeeper session timeout in milliseconds</description> + </property> + + <property> + <name>templeton.callback.retry.interval</name> + <value>10000</value> + <description>How long to wait between callback retry attempts in milliseconds</description> + </property> + + <property> + <name>templeton.callback.retry.attempts</name> + <value>5</value> + <description>How many times to retry the callback</description> + </property> + + <property> + <name>templeton.storage.class</name> + <value>org.apache.hcatalog.templeton.tool.HDFSStorage</value> + <description>The class to use as storage</description> + </property> + + <property> + <name>templeton.storage.root</name> + <value>/templeton-hadoop</value> + <description>The path to the directory to use for storage</description> + </property> + + <property> + <name>templeton.hdfs.cleanup.interval</name> + <value>43200000</value> + <description>The maximum delay between a thread's cleanup checks</description> + </property> + + <property> + <name>templeton.hdfs.cleanup.maxage</name> + <value>604800000</value> + <description>The maximum age of a templeton job</description> + </property> + + <property> + <name>templeton.zookeeper.cleanup.interval</name> + <value>43200000</value> + <description>The maximum delay between a thread's cleanup checks</description> + </property> + + <property> + <name>templeton.zookeeper.cleanup.maxage</name> + <value>604800000</value> + <description>The maximum age of a templeton job</description> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/bigtop/blob/39af29da/bigtop-deploy/puppet/modules/hcatalog/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hcatalog/tests/init.pp b/bigtop-deploy/puppet/modules/hcatalog/tests/init.pp new file mode 100644 index 0000000..59920ae --- /dev/null +++ b/bigtop-deploy/puppet/modules/hcatalog/tests/init.pp @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +hcatalog::server { "test-hcatalog-server": } +hcatalog::webhcat::server { "test-hcatalog-webhcat-server": }
