Author: apurtell
Date: Thu Nov 26 08:00:54 2009
New Revision: 884450
URL: http://svn.apache.org/viewvc?rev=884450&view=rev
Log:
HBASE-2012 [EC2] LZO support
Modified:
hadoop/hbase/branches/0.20/CHANGES.txt
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/create-hbase-image
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-zookeeper
Modified: hadoop/hbase/branches/0.20/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/CHANGES.txt?rev=884450&r1=884449&r2=884450&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.20/CHANGES.txt Thu Nov 26 08:00:54 2009
@@ -20,6 +20,7 @@
HBASE-1987 The Put object has no simple read methods for checking what
has already been added (Ryan Smith via Stack)
HBASE-2009 [EC2] Support mapreduce
+ HBASE-2012 [EC2] LZO support
Release 0.20.2 - November 18th, 2009
INCOMPATIBLE CHANGES
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/create-hbase-image
URL:
http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/create-hbase-image?rev=884450&r1=884449&r2=884450&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/create-hbase-image (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/create-hbase-image Thu Nov
26 08:00:54 2009
@@ -18,23 +18,26 @@
# Create a HBase AMI.
# Inspired by Jonathan Siegel's EC2 script
(http://blogsiegel.blogspot.com/2006/08/sandboxing-amazon-ec2.html)
-# allow override of INSTANCE_TYPE from the command line
-[ ! -z $1 ] && INSTANCE_TYPE=$1
+# allow override of SLAVE_INSTANCE_TYPE from the command line
+[ ! -z $1 ] && SLAVE_INSTANCE_TYPE=$1
# Import variables
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
. "$bin"/hbase-ec2-env.sh
-echo "INSTANCE_TYPE is $INSTANCE_TYPE."
-echo "ARCH is $ARCH."
+type=$SLAVE_INSTANCE_TYPE
+arch=$SLAVE_ARCH
-AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase |
grep $HBASE_VERSION | grep $ARCH | grep available | awk '{print $2}'`
+echo "INSTANCE_TYPE is $type"
+echo "ARCH is $arch"
+
+AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase |
grep $HBASE_VERSION | grep $arch | grep available | awk '{print $2}'`
[ ! -z $AMI_IMAGE ] && echo "AMI already registered, use: ec2-deregister
$AMI_IMAGE" && exit 1
echo "Starting a AMI with ID $BASE_AMI_IMAGE."
-OUTPUT=`ec2-run-instances $BASE_AMI_IMAGE $TOOL_OPTS -k root -t $INSTANCE_TYPE`
+OUTPUT=`ec2-run-instances $BASE_AMI_IMAGE $TOOL_OPTS -k root -t $type`
BOOTING_INSTANCE=`echo $OUTPUT | awk '{print $6}'`
echo "Instance is $BOOTING_INSTANCE."
@@ -69,9 +72,9 @@
scp $SSH_OPTS $EC2_CERT "r...@$hostname:/mnt"
# Connect to it
-ssh $SSH_OPTS "r...@$hostname" "sh -c \"INSTANCE_TYPE=$INSTANCE_TYPE
/mnt/create-hbase-image-remote\""
+ssh $SSH_OPTS "r...@$hostname" "sh -c \"INSTANCE_TYPE=$type ARCH=$arch
/mnt/create-hbase-image-remote\""
# Register image
-ec2-register $TOOL_OPTS $S3_BUCKET/hbase-$HBASE_VERSION-$ARCH.manifest.xml
+ec2-register $TOOL_OPTS $S3_BUCKET/hbase-$HBASE_VERSION-$arch.manifest.xml
echo "Terminate with: ec2-terminate-instances $BOOTING_INSTANCE"
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh
URL:
http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh?rev=884450&r1=884449&r2=884450&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh Thu Nov 26
08:00:54 2009
@@ -45,10 +45,10 @@
TOOL_OPTS=`echo -K "$EC2_PRIVATE_KEY" -C "$EC2_CERT"`
# The version of HBase to use.
-HBASE_VERSION=0.20.1
+HBASE_VERSION=0.20.2
# The version of Hadoop to use.
-HADOOP_VERSION=$HBASE_VERSION
+HADOOP_VERSION=0.20.1
# The Amazon S3 bucket where the HBase AMI is stored.
# The default value is for public images, so can be left if you are using
running a public image.
@@ -64,7 +64,10 @@
USER_DATA_FILE=hbase-ec2-init-remote.sh
# Use only c1.xlarge unless you know what you are doing
-INSTANCE_TYPE=${INSTANCE_TYPE:-c1.xlarge}
+MASTER_INSTANCE_TYPE=${MASTER_INSTANCE_TYPE:-c1.xlarge}
+
+# Use only c1.xlarge unless you know what you are doing
+SLAVE_INSTANCE_TYPE=${SLAVE_INSTANCE_TYPE:-c1.xlarge}
# Use only c1.medium unless you know what you are doing
ZOO_INSTANCE_TYPE=${ZOO_INSTANCE_TYPE:-c1.medium}
@@ -89,17 +92,20 @@
JAVA_VERSION=1.6.0_16
# SUPPORTED_ARCHITECTURES = ['i386', 'x86_64']
-# The download URL for the Sun JDK. Visit
http://java.sun.com/javase/downloads/index.jsp and get the URL for the "Linux
self-extracting file".
-if [ "$INSTANCE_TYPE" = "m1.small" -o "$INSTANCE_TYPE" = "c1.medium" ]; then
- ARCH='i386'
+if [ "$SLAVE_INSTANCE_TYPE" = "m1.small" -o "$SLAVE_INSTANCE_TYPE" =
"c1.medium" ]; then
+ SLAVE_ARCH='i386'
BASE_AMI_IMAGE="ami-48aa4921" #
ec2-public-images/fedora-8-i386-base-v1.10.manifest.xml
- AMI_IMAGE="ami-c644a7af"
-
JAVA_BINARY_URL='http://iridiant.s3.amazonaws.com/jdk/jdk-6u16-linux-i586.bin'
else
- ARCH='x86_64'
+ SLAVE_ARCH='x86_64'
BASE_AMI_IMAGE="ami-f61dfd9f" #
ec2-public-images/fedora-8-x86_64-base-v1.10.manifest.xml
- AMI_IMAGE="ami-f244a79b"
- JAVA_BINARY_URL='http://iridiant.s3.amazonaws.com/jdk/jdk-6u16-linux-x64.bin'
+fi
+
+if [ "$MASTER_INSTANCE_TYPE" = "m1.small" -o "$MASTER_INSTANCE_TYPE" =
"c1.medium" ]; then
+ MASTER_ARCH='i386'
+ MASTER_AMI_IMAGE="ami-c644a7af"
+else
+ MASTER_ARCH='x86_64'
+ MASTER_AMI_IMAGE="ami-f244a79b"
fi
if [ "$ZOO_INSTANCE_TYPE" = "m1.small" -o "$ZOO_INSTANCE_TYPE" = "c1.medium"
]; then
Modified:
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote
URL:
http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote?rev=884450&r1=884449&r2=884450&view=diff
==============================================================================
---
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote
(original)
+++
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote
Thu Nov 26 08:00:54 2009
@@ -15,7 +15,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
# Create a Hbase AMI. Runs on the EC2 instance.
# Import variables
@@ -23,8 +22,13 @@
bin=`cd "$bin"; pwd`
. "$bin"/hbase-ec2-env.sh
-echo "Remote: INSTANCE_TYPE is $INSTANCE_TYPE."
-echo "Remote: ARCH is $ARCH."
+type=$INSTANCE_TYPE
+[ -z "$type" ] && type=$SLAVE_INSTANCE_TYPE
+arch=$ARCH
+[ -z "$arch" ] && arch=$SLAVE_ARCH
+
+echo "Remote: INSTANCE_TYPE is $type"
+echo "Remote: ARCH is $arch"
# Remove environment script since it contains sensitive information
rm -f "$bin"/hbase-ec2-env.sh
@@ -32,14 +36,14 @@
# Install Java
echo "Downloading and installing java binary."
cd /usr/local
-wget -nv -O java.bin $JAVA_BINARY_URL
+wget -nv -O java.bin
http://iridiant.s3.amazonaws.com/jdk/jdk-${JAVA_VERSION}-linux-${arch}.bin
sh java.bin
rm -f java.bin
# Install tools
echo "Installing rpms."
yum -y update
-yum -y install rsync lynx screen ganglia-gmetad ganglia-gmond ganglia-web
httpd php
+yum -y install rsync lynx screen ganglia-gmetad ganglia-gmond ganglia-web
httpd php lzo-devel
yum -y clean all
# Install Hadoop
@@ -89,13 +93,20 @@
# Ensure logging in to new hosts is seamless.
echo ' StrictHostKeyChecking no' >> /etc/ssh/ssh_config
+# Install LZO for HBase
+echo "Installing LZO codec support for HBase"
+cd /usr/local/hbase-${HBASE_VERSION}
+wget -nv
http://iridiant.s3.amazonaws.com/hbase/lzo-linux-${HADOOP_VERSION}.tar.gz
+tar xzf /var/tmp/lzo-linux-${HADOOP_VERSION}.tar.gz
+rm /var/tmp/lzo-linux-${HADOOP_VERSION}.tar.gz
+
# Bundle and upload image
cd ~root
# Don't need to delete .bash_history since it isn't written until exit.
df -h
-ec2-bundle-vol -d /mnt -k /mnt/pk*.pem -c /mnt/cert*.pem -u $AWS_ACCOUNT_ID -s
3072 -p hbase-$HBASE_VERSION-$ARCH -r $ARCH
+ec2-bundle-vol -d /mnt -k /mnt/pk*.pem -c /mnt/cert*.pem -u $AWS_ACCOUNT_ID -s
3072 -p hbase-$HBASE_VERSION-$arch -r $arch
-ec2-upload-bundle -b $S3_BUCKET -m
/mnt/hbase-$HBASE_VERSION-$ARCH.manifest.xml -a $AWS_ACCESS_KEY_ID -s
$AWS_SECRET_ACCESS_KEY
+ec2-upload-bundle -b $S3_BUCKET -m
/mnt/hbase-$HBASE_VERSION-$arch.manifest.xml -a $AWS_ACCESS_KEY_ID -s
$AWS_SECRET_ACCESS_KEY
# End
echo Done
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master
URL:
http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master?rev=884450&r1=884449&r2=884450&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master
(original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master Thu Nov
26 08:00:54 2009
@@ -29,6 +29,11 @@
bin=`cd "$bin"; pwd`
. "$bin"/hbase-ec2-env.sh
+type=$MASTER_INSTANCE_TYPE
+[ -z "$type" ] && type=$SLAVE_INSTANCE_TYPE
+arch=$MASTER_ARCH
+[ -z "$arch" ] && arch=$SLAVE_ARCH
+
if [ -z $AWS_ACCOUNT_ID ]; then
echo "Please set AWS_ACCOUNT_ID in $bin/hbase-ec2-env.sh."
exit 1
@@ -47,15 +52,15 @@
fi
# Finding HBase image
-[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep
$S3_BUCKET | grep $HBASE_VERSION | grep $ARCH | grep available | awk '{print
$2}'`
+[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep
$S3_BUCKET | grep $HBASE_VERSION | grep $arch | grep available | awk '{print
$2}'`
# Start a master
-echo "Starting master with AMI $AMI_IMAGE (arch $ARCH)"
+echo "Starting master with AMI $AMI_IMAGE (arch $arch)"
# Substituting zookeeper quorum
ZOOKEEPER_QUORUM=`cat $ZOOKEEPER_QUORUM_PATH`
sed -e "s|%ZOOKEEPER_QUORUM%|$ZOOKEEPER_QUORUM|" \
"$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.master
-INSTANCE=`ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_MASTER -k
root -f "$bin"/$USER_DATA_FILE.master -t $INSTANCE_TYPE | grep INSTANCE | awk
'{print $2}'`
+INSTANCE=`ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_MASTER -k
root -f "$bin"/$USER_DATA_FILE.master -t $type | grep INSTANCE | awk '{print
$2}'`
echo -n "Waiting for instance $INSTANCE to start"
while true; do
printf "."
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves
URL:
http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves?rev=884450&r1=884449&r2=884450&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves
(original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves Thu Nov
26 08:00:54 2009
@@ -41,7 +41,7 @@
exit 1
fi
-[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep
$S3_BUCKET | grep $HBASE_VERSION | grep $ARCH |grep available | awk '{print
$2}'`
+[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep
$S3_BUCKET | grep $HBASE_VERSION | grep $SLAVE_ARCH |grep available | awk
'{print $2}'`
MASTER_HOST=`cat $MASTER_PRIVATE_IP_PATH`
MASTER_ZONE=`cat $MASTER_ZONE_PATH`
@@ -53,7 +53,7 @@
"$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.slave
# Start slaves
-echo "Starting $NO_INSTANCES AMI(s) with ID $AMI_IMAGE (arch $ARCH) in group
$CLUSTER in zone $MASTER_ZONE"
-ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n "$NO_INSTANCES" -g "$CLUSTER" -k
root -f "$bin"/$USER_DATA_FILE.slave -t "$INSTANCE_TYPE" -z "$MASTER_ZONE" |
grep INSTANCE | awk '{print $2}'
+echo "Starting $NO_INSTANCES AMI(s) with ID $AMI_IMAGE (arch $SLAVE_ARCH) in
group $CLUSTER in zone $MASTER_ZONE"
+ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n "$NO_INSTANCES" -g "$CLUSTER" -k
root -f "$bin"/$USER_DATA_FILE.slave -t "$SLAVE_INSTANCE_TYPE" -z
"$MASTER_ZONE" | grep INSTANCE | awk '{print $2}'
rm "$bin"/$USER_DATA_FILE.slave
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-zookeeper
URL:
http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-zookeeper?rev=884450&r1=884449&r2=884450&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-zookeeper
(original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-zookeeper Thu
Nov 26 08:00:54 2009
@@ -36,9 +36,9 @@
. "$bin"/hbase-ec2-env.sh
type=$ZOO_INSTANCE_TYPE
-[ -z "$type" ] && type=$INSTANCE_TYPE
+[ -z "$type" ] && type=$SLAVE_INSTANCE_TYPE
arch=$ZOO_ARCH
-[ -z "$arch" ] && arch=$ARCH
+[ -z "$arch" ] && arch=$SLAVE_ARCH
# Finding HBase image
[ -z "$ZOO_AMI_IMAGE" ] && ZOO_AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a |
grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION | grep $arch |grep available
| awk '{print $2}'`
