Author: tomwhite
Date: Thu Sep 18 02:44:59 2008
New Revision: 696615
URL: http://svn.apache.org/viewvc?rev=696615&view=rev
Log:
HADOOP-4117. Improve configurability of Hadoop EC2 instances.
Added:
hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-init-remote.sh
hadoop/core/trunk/src/contrib/ec2/bin/image/ec2-run-user-data
Removed:
hadoop/core/trunk/src/contrib/ec2/bin/image/hadoop-init
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/contrib/ec2/bin/create-hadoop-image
hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template
hadoop/core/trunk/src/contrib/ec2/bin/image/create-hadoop-image-remote
hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master
hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=696615&r1=696614&r2=696615&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Sep 18 02:44:59 2008
@@ -341,6 +341,9 @@
HADOOP-4075. test-patch.sh now spits out ant commands that it runs.
(Ramya R via nigel)
+ HADOOP-4117. Improve configurability of Hadoop EC2 instances.
+ (tomwhite)
+
OPTIMIZATIONS
HADOOP-3556. Removed lock contention in MD5Hash by changing the
Modified: hadoop/core/trunk/src/contrib/ec2/bin/create-hadoop-image
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/create-hadoop-image?rev=696615&r1=696614&r2=696615&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/create-hadoop-image (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/create-hadoop-image Thu Sep 18
02:44:59 2008
@@ -63,7 +63,7 @@
# Copy setup scripts
scp $SSH_OPTS "$bin"/hadoop-ec2-env.sh "[EMAIL PROTECTED]:/mnt"
scp $SSH_OPTS "$bin"/image/create-hadoop-image-remote "[EMAIL PROTECTED]:/mnt"
-scp $SSH_OPTS "$bin"/image/hadoop-init "[EMAIL PROTECTED]:"
+scp $SSH_OPTS "$bin"/image/ec2-run-user-data "[EMAIL PROTECTED]:/etc/init.d"
# Copy private key and certificate (for bundling image)
scp $SSH_OPTS $EC2_KEYDIR/pk*.pem "[EMAIL PROTECTED]:/mnt"
Modified: hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template?rev=696615&r1=696614&r2=696615&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template Thu Sep 18
02:44:59 2008
@@ -51,10 +51,8 @@
# Enable public access to JobTracker and TaskTracker web interfaces
ENABLE_WEB_PORTS=true
-# Boot parameters
-MAX_MAP_TASKS=2
-MAX_REDUCE_TASKS=2
-COMPRESS=true
+# The script to run on instance boot.
+USER_DATA_FILE=hadoop-ec2-init-remote.sh
# The EC2 instance type: m1.small, m1.large, m1.xlarge
INSTANCE_TYPE="m1.small"
Added: hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-init-remote.sh
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-init-remote.sh?rev=696615&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-init-remote.sh (added)
+++ hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-init-remote.sh Thu Sep 18
02:44:59 2008
@@ -0,0 +1,150 @@
+#!/usr/bin/env bash
+
+################################################################################
+# Script that is run on each EC2 instance on boot. It is passed in the EC2 user
+# data, so should not exceed 16K in size.
+################################################################################
+
+################################################################################
+# Initialize variables
+################################################################################
+
+# Slaves are started after the master, and are told its address by sending a
+# modified copy of this file which sets the MASTER_HOST variable.
+# A node knows if it is the master or not by inspecting the security group
+# name. If it is the master then it retrieves its address using instance data.
+MASTER_HOST=%MASTER_HOST% # Interpolated before being sent to EC2 node
+SECURITY_GROUPS=`wget -q -O -
http://169.254.169.254/latest/meta-data/security-groups`
+IS_MASTER=`echo $SECURITY_GROUPS | awk '{ a = match ($0, "-master$"); if (a)
print "true"; else print "false"; }'`
+if [ "$IS_MASTER" == "true" ]; then
+ MASTER_HOST=`wget -q -O -
http://169.254.169.254/latest/meta-data/local-hostname`
+fi
+
+HADOOP_HOME=`ls -d /usr/local/hadoop-*`
+
+################################################################################
+# Hadoop configuration
+# Modify this section to customize your Hadoop cluster.
+################################################################################
+
+cat > $HADOOP_HOME/conf/hadoop-site.xml <<EOF
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+<property>
+ <name>hadoop.tmp.dir</name>
+ <value>/mnt/hadoop</value>
+</property>
+
+<property>
+ <name>fs.default.name</name>
+ <value>hdfs://$MASTER_HOST:50001</value>
+</property>
+
+<property>
+ <name>mapred.job.tracker</name>
+ <value>hdfs://$MASTER_HOST:50002</value>
+</property>
+
+<property>
+ <name>tasktracker.http.threads</name>
+ <value>80</value>
+</property>
+
+<property>
+ <name>mapred.tasktracker.map.tasks.maximum</name>
+ <value>3</value>
+</property>
+
+<property>
+ <name>mapred.tasktracker.reduce.tasks.maximum</name>
+ <value>3</value>
+</property>
+
+<property>
+ <name>mapred.output.compress</name>
+ <value>true</value>
+</property>
+
+<property>
+ <name>mapred.output.compression.type</name>
+ <value>BLOCK</value>
+</property>
+
+<property>
+ <name>dfs.client.block.write.retries</name>
+ <value>3</value>
+</property>
+
+</configuration>
+EOF
+
+# Configure Hadoop for Ganglia
+# overwrite hadoop-metrics.properties
+cat > $HADOOP_HOME/conf/hadoop-metrics.properties <<EOF
+
+# Ganglia
+# we push to the master gmond so hostnames show up properly
+dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+dfs.period=10
+dfs.servers=$MASTER_HOST:8649
+
+mapred.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+mapred.period=10
+mapred.servers=$MASTER_HOST:8649
+
+jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+jvm.period=10
+jvm.servers=$MASTER_HOST:8649
+EOF
+
+################################################################################
+# Start services
+################################################################################
+
+[ ! -f /etc/hosts ] && echo "127.0.0.1 localhost" > /etc/hosts
+
+mkdir -p /mnt/hadoop/logs
+
+# not set on boot
+export USER="root"
+
+if [ "$IS_MASTER" == "true" ]; then
+ # MASTER
+ # Prep Ganglia
+ sed -i -e "s|\( *mcast_join *=.*\)|#\1|" \
+ -e "s|\( *bind *=.*\)|#\1|" \
+ -e "s|\( *mute *=.*\)| mute = yes|" \
+ -e "s|\( *location *=.*\)| location = \"master-node\"|" \
+ /etc/gmond.conf
+ mkdir -p /mnt/ganglia/rrds
+ chown -R ganglia:ganglia /mnt/ganglia/rrds
+ rm -rf /var/lib/ganglia; cd /var/lib; ln -s /mnt/ganglia ganglia; cd
+ service gmond start
+ service gmetad start
+ apachectl start
+
+ # Hadoop
+ # only format on first boot
+ [ ! -e /mnt/hadoop/dfs ] && "$HADOOP_HOME"/bin/hadoop namenode -format
+
+ "$HADOOP_HOME"/bin/hadoop-daemon.sh start namenode
+ "$HADOOP_HOME"/bin/hadoop-daemon.sh start jobtracker
+else
+ # SLAVE
+ # Prep Ganglia
+ sed -i -e "s|\( *mcast_join *=.*\)|#\1|" \
+ -e "s|\( *bind *=.*\)|#\1|" \
+ -e "s|\(udp_send_channel {\)|\1\n host=$MASTER_HOST|" \
+ /etc/gmond.conf
+ service gmond start
+
+ # Hadoop
+ "$HADOOP_HOME"/bin/hadoop-daemon.sh start datanode
+ "$HADOOP_HOME"/bin/hadoop-daemon.sh start tasktracker
+fi
+
+# Run this script on next boot
+rm -f /var/ec2/ec2-run-user-data.*
Modified: hadoop/core/trunk/src/contrib/ec2/bin/image/create-hadoop-image-remote
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/image/create-hadoop-image-remote?rev=696615&r1=696614&r2=696615&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/image/create-hadoop-image-remote
(original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/image/create-hadoop-image-remote Thu
Sep 18 02:44:59 2008
@@ -53,8 +53,9 @@
-e 's|# export HADOOP_OPTS=.*|export HADOOP_OPTS=-server|' \
/usr/local/hadoop-$HADOOP_VERSION/conf/hadoop-env.sh
-# Do configuration on instance startup
-echo "/root/hadoop-init" >> /etc/rc.d/rc.local
+# Run user data as script on instance startup
+chmod +x /etc/init.d/ec2-run-user-data
+echo "/etc/init.d/ec2-run-user-data" >> /etc/rc.d/rc.local
# Setup root user bash environment
echo "export JAVA_HOME=/usr/local/jdk${JAVA_VERSION}" >> /root/.bash_profile
Added: hadoop/core/trunk/src/contrib/ec2/bin/image/ec2-run-user-data
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/image/ec2-run-user-data?rev=696615&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/image/ec2-run-user-data (added)
+++ hadoop/core/trunk/src/contrib/ec2/bin/image/ec2-run-user-data Thu Sep 18
02:44:59 2008
@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# ec2-run-user-data - Run instance user-data if it looks like a script.
+#
+# Only retrieves and runs the user-data script once per instance. If
+# you want the user-data script to run again (e.g., on the next boot)
+# then add this command in the user-data script:
+# rm -f /var/ec2/ec2-run-user-data.*
+#
+# History:
+# 2008-05-16 Eric Hammond <[EMAIL PROTECTED]>
+# - Initial version including code from Kim Scheibel, Jorge Oliveira
+# 2008-08-06 Tom White
+# - Updated to use mktemp on fedora
+#
+
+prog=$(basename $0)
+logger="logger -t $prog"
+curl="curl --retry 3 --silent --show-error --fail"
+instance_data_url=http://169.254.169.254/2008-02-01
+
+# Wait until networking is up on the EC2 instance.
+perl -MIO::Socket::INET -e '
+ until(new IO::Socket::INET("169.254.169.254:80")){print"Waiting for
network...\n";sleep 1}
+' | $logger
+
+# Exit if we have already run on this instance (e.g., previous boot).
+ami_id=$($curl $instance_data_url/meta-data/ami-id)
+been_run_file=/var/ec2/$prog.$ami_id
+mkdir -p $(dirname $been_run_file)
+if [ -f $been_run_file ]; then
+ $logger < $been_run_file
+ exit
+fi
+
+# Retrieve the instance user-data and run it if it looks like a script
+user_data_file=`mktemp -t ec2-user-data.XXXXXXXXXX`
+chmod 700 $user_data_file
+$logger "Retrieving user-data"
+$curl -o $user_data_file $instance_data_url/user-data 2>&1 | $logger
+if [ ! -s $user_data_file ]; then
+ $logger "No user-data available"
+elif head -1 $user_data_file | egrep -v '^#!'; then
+ $logger "Skipping user-data as it does not begin with #!"
+else
+ $logger "Running user-data"
+ echo "user-data has already been run on this instance" > $been_run_file
+ $user_data_file 2>&1 | logger -t "user-data"
+ $logger "user-data exit code: $?"
+fi
+rm -f $user_data_file
Modified: hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master?rev=696615&r1=696614&r2=696615&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master Thu Sep 18
02:44:59 2008
@@ -81,7 +81,7 @@
# Start a master
echo "Starting master with AMI $AMI_IMAGE"
USER_DATA="MASTER_HOST=master,MAX_MAP_TASKS=$MAX_MAP_TASKS,MAX_REDUCE_TASKS=$MAX_REDUCE_TASKS,COMPRESS=$COMPRESS"
-INSTANCE=`ec2-run-instances $AMI_IMAGE -n 1 -g $CLUSTER_MASTER -k $KEY_NAME -d
"$USER_DATA" -t $INSTANCE_TYPE | grep INSTANCE | awk '{print $2}'`
+INSTANCE=`ec2-run-instances $AMI_IMAGE -n 1 -g $CLUSTER_MASTER -k $KEY_NAME -f
"$bin"/$USER_DATA_FILE -t $INSTANCE_TYPE | grep INSTANCE | awk '{print $2}'`
echo "Waiting for instance $INSTANCE to start"
while true; do
printf "."
Modified: hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves?rev=696615&r1=696614&r2=696615&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves Thu Sep 18
02:44:59 2008
@@ -44,9 +44,11 @@
MASTER_HOST=`cat $MASTER_PRIVATE_IP_PATH`
MASTER_ZONE=`cat $MASTER_ZONE_PATH`
+# Substituting master hostname
+sed -e "s|%MASTER_HOST%|$MASTER_HOST|" "$bin"/$USER_DATA_FILE >
"$bin"/$USER_DATA_FILE.slave
+
# Start slaves
echo "Adding $1 node(s) to cluster group $CLUSTER with AMI $AMI_IMAGE"
-USER_DATA="MASTER_HOST=$MASTER_HOST,MAX_MAP_TASKS=$MAX_MAP_TASKS,MAX_REDUCE_TASKS=$MAX_REDUCE_TASKS,COMPRESS=$COMPRESS"
-ec2-run-instances $AMI_IMAGE -n "$NO_INSTANCES" -g "$CLUSTER" -k "$KEY_NAME"
-d "$USER_DATA" -t "$INSTANCE_TYPE" -z "$MASTER_ZONE" | grep INSTANCE | awk
'{print $2}'
-
+ec2-run-instances $AMI_IMAGE -n "$NO_INSTANCES" -g "$CLUSTER" -k "$KEY_NAME"
-f "$bin"/$USER_DATA_FILE.slave -t "$INSTANCE_TYPE" -z "$MASTER_ZONE" | grep
INSTANCE | awk '{print $2}'
+rm "$bin"/$USER_DATA_FILE.slave
