Updated Branches: refs/heads/master 89f7117f2 -> c0e3d974f
BIGTOP-547: create an HDFS setup helper script Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/c0e3d974 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/c0e3d974 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/c0e3d974 Branch: refs/heads/master Commit: c0e3d974f21193cefa2b363ac632d8dd7b1bdceb Parents: 89f7117 Author: Mark Grover <[email protected]> Authored: Mon Jan 28 17:45:31 2013 -0800 Committer: Roman Shaposhnik <[email protected]> Committed: Mon Feb 18 18:37:03 2013 -0800 ---------------------------------------------------------------------- bigtop-packages/src/common/hadoop/init-hdfs.sh | 79 +++++++++++++++ .../src/common/hadoop/install_hadoop.sh | 1 + bigtop-packages/src/deb/hadoop/hadoop-hdfs.install | 1 + bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec | 1 + 4 files changed, 82 insertions(+), 0 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/c0e3d974/bigtop-packages/src/common/hadoop/init-hdfs.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hadoop/init-hdfs.sh b/bigtop-packages/src/common/hadoop/init-hdfs.sh new file mode 100755 index 0000000..57496a8 --- /dev/null +++ b/bigtop-packages/src/common/hadoop/init-hdfs.sh @@ -0,0 +1,79 @@ +#!/bin/bash -ex +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Use this script to initialize HDFS directory structure for various components to run. This script can be run from any node in the Hadoop cluster but should only be run once by one node only. If you are planning on using oozie, we recommend that you run this script from a node that has hive, pig, sqoop, etc. installed. Unless you are using psuedo distributed cluster, this node is most likely NOT your namenode +# Steps to be performed before running this script: +# 1. Stop the namenode and datanode services if running. +# 2. Format namenode (sudo -u hdfs hdfs namenode -format). +# 3. Start the namenode and datanode services on appropriate nodes. +sudo -u hdfs hadoop fs -mkdir /tmp +sudo -u hdfs hadoop fs -chmod -R 1777 /tmp +sudo -u hdfs hadoop fs -mkdir /var +sudo -u hdfs hadoop fs -mkdir /var/log +sudo -u hdfs hadoop fs -chmod -R 1775 /var/log +sudo -u hdfs hadoop fs -chown yarn:mapred /var/log +sudo -u hdfs hadoop fs -mkdir /hbase +sudo -u hdfs hadoop fs -chown hbase /hbase +sudo -u hdfs hadoop fs -mkdir /benchmarks +sudo -u hdfs hadoop fs -chmod -R 777 /benchmarks +sudo -u hdfs hadoop fs -mkdir /user +sudo -u hdfs hadoop fs -mkdir /user/history +sudo -u hdfs hadoop fs -chown mapred /user/history +sudo -u hdfs hadoop fs -mkdir /user/jenkins +sudo -u hdfs hadoop fs -chmod -R 777 /user/jenkins +sudo -u hdfs hadoop fs -chown jenkins /user/jenkins +sudo -u hdfs hadoop fs -mkdir /user/hive +sudo -u hdfs hadoop fs -chmod -R 777 /user/hive +sudo -u hdfs hadoop fs -chown hive /user/hive +sudo -u hdfs hadoop fs -mkdir /user/root +sudo -u hdfs hadoop fs -chmod -R 777 /user/root +sudo -u hdfs hadoop fs -chown root /user/root +sudo -u hdfs hadoop fs -mkdir /user/hue +sudo -u hdfs hadoop fs -chmod -R 777 /user/hue +sudo -u hdfs hadoop fs -chown hue /user/hue +sudo -u hdfs hadoop fs -mkdir /user/oozie +# Do more setup for oozie +sudo -u hdfs hadoop fs -mkdir /user/oozie/share +sudo -u hdfs hadoop fs -mkdir /user/oozie/share/lib +sudo -u hdfs hadoop fs -mkdir /user/oozie/share/lib/hive +sudo -u hdfs hadoop fs -mkdir /user/oozie/share/lib/mapreduce-streaming +sudo -u hdfs hadoop fs -mkdir /user/oozie/share/lib/distcp +sudo -u hdfs hadoop fs -mkdir /user/oozie/share/lib/pig +sudo -u hdfs hadoop fs -mkdir /user/oozie/share/lib/sqoop +# Copy over files from local filesystem to HDFS that oozie might need +if ls /usr/lib/hive/lib/*.jar &> /dev/null; then + sudo -u hdfs hadoop fs -put /usr/lib/hive/lib/*.jar /user/oozie/share/lib/hive +fi + +if ls /usr/lib/hadoop-mapreduce/hadoop-streaming*.jar &> /dev/null; then + sudo -u hdfs hadoop fs -put /usr/lib/hadoop-mapreduce/hadoop-streaming*.jar /user/oozie/share/lib/mapreduce-streaming +fi + +if ls /usr/lib/hadoop-mapreduce/hadoop-distcp*.jar &> /dev/null; then + sudo -u hdfs hadoop fs -put /usr/lib/hadoop-mapreduce/hadoop-distcp*.jar /user/oozie/share/lib/distcp +fi + +if ls /usr/lib/pig/{lib/,}*.jar &> /dev/null; then + sudo -u hdfs hadoop fs -put /usr/lib/pig/{lib/,}*.jar /user/oozie/share/lib/pig +fi + +if ls /usr/lib/sqoop/{lib/,}*.jar &> /dev/null; then + sudo -u hdfs hadoop fs -put /usr/lib/sqoop/{lib/,}*.jar /user/share/lib/sqoop +fi + +sudo -u hdfs hadoop fs -chmod -R 777 /user/oozie +sudo -u hdfs hadoop fs -chown -R oozie /user/oozie http://git-wip-us.apache.org/repos/asf/bigtop/blob/c0e3d974/bigtop-packages/src/common/hadoop/install_hadoop.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hadoop/install_hadoop.sh b/bigtop-packages/src/common/hadoop/install_hadoop.sh index 2f3714d..76cc735 100755 --- a/bigtop-packages/src/common/hadoop/install_hadoop.sh +++ b/bigtop-packages/src/common/hadoop/install_hadoop.sh @@ -189,6 +189,7 @@ done install -d -m 0755 ${SYSTEM_LIBEXEC_DIR} cp ${BUILD_DIR}/libexec/* ${SYSTEM_LIBEXEC_DIR}/ cp ${DISTRO_DIR}/hadoop-layout.sh ${SYSTEM_LIBEXEC_DIR}/ +install -m 0755 ${DISTRO_DIR}/init-hdfs.sh ${SYSTEM_LIBEXEC_DIR}/ # hadoop jar install -d -m 0755 ${HADOOP_DIR} http://git-wip-us.apache.org/repos/asf/bigtop/blob/c0e3d974/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install b/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install index 9fb7adb..9ce5994 100644 --- a/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install +++ b/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install @@ -2,6 +2,7 @@ /etc/hadoop/conf.empty/hdfs-site.xml /usr/lib/hadoop-hdfs /usr/lib/hadoop/libexec/hdfs-config.sh +/usr/lib/hadoop/libexec/init-hdfs.sh /usr/bin/hdfs /var/lib/hadoop-hdfs /var/log/hadoop-hdfs http://git-wip-us.apache.org/repos/asf/bigtop/blob/c0e3d974/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec index 21b6e68..8dff524 100644 --- a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec +++ b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec @@ -580,6 +580,7 @@ fi %attr(0775,hdfs,hadoop) %{log_hdfs} %attr(0755,hdfs,hadoop) %{state_hdfs} %attr(1777,hdfs,hadoop) %{state_hdfs}/cache +%{lib_hadoop}/libexec/init-hdfs.sh %files mapreduce %defattr(-,root,root)
