Author: apurtell
Date: Wed Nov 25 02:26:11 2009
New Revision: 883962
URL: http://svn.apache.org/viewvc?rev=883962&view=rev
Log:
HBASE-2009 [EC2] Support mapreduce
Modified:
hadoop/hbase/branches/0.20/CHANGES.txt
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-remote.sh
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-zookeeper-remote.sh
Modified: hadoop/hbase/branches/0.20/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/CHANGES.txt?rev=883962&r1=883961&r2=883962&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.20/CHANGES.txt Wed Nov 25 02:26:11 2009
@@ -19,6 +19,7 @@
(Jeremiah Jacquet via Stack)
HBASE-1987 The Put object has no simple read methods for checking what
has already been added (Ryan Smith via Stack)
+ HBASE-2009 [EC2] Support mapreduce
Release 0.20.2 - November 18th, 2009
INCOMPATIBLE CHANGES
Modified:
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-remote.sh
URL:
http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-remote.sh?rev=883962&r1=883961&r2=883962&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-remote.sh
(original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-remote.sh Wed
Nov 25 02:26:11 2009
@@ -19,7 +19,7 @@
# Hadoop configuration
###############################################################################
-cat > $HADOOP_HOME/conf/hadoop-site.xml <<EOF
+cat > $HADOOP_HOME/conf/core-site.xml <<EOF
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
@@ -29,11 +29,38 @@
</property>
<property>
<name>fs.default.name</name>
- <value>hdfs://$MASTER_HOST:50001</value>
+ <value>hdfs://$MASTER_HOST:8020</value>
</property>
</configuration>
EOF
+cat > $HADOOP_HOME/conf/hdfs-site.xml <<EOF
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+<property>
+ <name>fs.default.name</name>
+ <value>hdfs://$MASTER_HOST:8020</value>
+</property>
+</configuration>
+EOF
+
+cat > $HADOOP_HOME/conf/mapred-site.xml <<EOF
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+<property>
+ <name>mapred.job.tracker</name>
+ <value>$MASTER_HOST:8021</value>
+</property>
+</configuration>
+EOF
+
+# Update classpath to include HBase jars and config
+cat >> $HADOOP_HOME/conf/hadoop-env.sh <<EOF
+HADOOP_CLASSPATH="$HBASE_HOME/hbase-${HBASE_VERSION}.jar:$HBASE_HOME/lib/AgileJSON-2009-03-30.jar:$HBASE_HOME/lib/json.jar:$HBASE_HOME/lib/zookeeper-3.2.1.jar:$HBASE_HOME/conf"
+EOF
+
# Configure Hadoop for Ganglia
# overwrite hadoop-metrics.properties
cat > $HADOOP_HOME/conf/hadoop-metrics.properties <<EOF
@@ -57,20 +84,8 @@
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
- <name>fs.default.name</name>
- <value>hdfs://$MASTER_HOST:50001</value>
-</property>
-<property>
- <name>dfs.replication</name>
- <value>3</value>
-</property>
-<property>
- <name>dfs.client.block.write.retries</name>
- <value>100</value>
-</property>
-<property>
<name>hbase.rootdir</name>
- <value>hdfs://$MASTER_HOST:50001/hbase</value>
+ <value>hdfs://$MASTER_HOST:8020/hbase</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
@@ -81,10 +96,6 @@
<value>$ZOOKEEPER_QUORUM</value>
</property>
<property>
- <name>zookeeper.session.timeout</name>
- <value>60000</value>
-</property>
-<property>
<name>hbase.regionserver.handler.count</name>
<value>100</value>
</property>
@@ -96,9 +107,31 @@
<name>hbase.hstore.blockingStoreFiles</name>
<value>15</value>
</property>
+<property>
+ <name>dfs.replication</name>
+ <value>3</value>
+</property>
+<property>
+ <name>dfs.client.block.write.retries</name>
+ <value>100</value>
+</property>
+<property>
+ <name>zookeeper.session.timeout</name>
+ <value>60000</value>
+</property>
+<property>
+ <name>hbase.tmp.dir</name>
+ <value>/mnt/hbase</value>
+</property>
</configuration>
EOF
+# Override JVM options
+cat >> $HBASE_HOME/conf/hbase-env.sh <<EOF
+export HBASE_MASTER_OPTS="-XX:+UseConcMarkSweepGC -XX:+DoEscapeAnalysis
-XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps
-Xloggc:/mnt/hbase/logs/hbase-master-gc.log"
+export HBASE_REGIONSERVER_OPTS="-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=88 -XX:+DoEscapeAnalysis -XX:+AggressiveOpts
-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps
-Xloggc:/mnt/hbase/logs/hbase-regionserver-gc.log"
+EOF
+
# Configure HBase for Ganglia
# overwrite hadoop-metrics.properties
cat > $HBASE_HOME/conf/hadoop-metrics.properties <<EOF
@@ -123,7 +156,7 @@
# up epoll limits
# ok if this fails, only valid for kernels 2.6.27+
-sysctl -w fs.epoll.max_user_instances=32768
+sysctl -w fs.epoll.max_user_instances=32768 > /dev/null 2>&1
mkdir -p /mnt/hadoop/logs
mkdir -p /mnt/hbase/logs
@@ -155,7 +188,9 @@
"$HADOOP_HOME"/bin/hadoop-daemon.sh start datanode
- sleep 10
+ "$HADOOP_HOME"/bin/hadoop-daemon.sh start jobtracker
+
+ sleep 10
"$HBASE_HOME"/bin/hbase-daemon.sh start master
@@ -174,6 +209,8 @@
"$HBASE_HOME"/bin/hbase-daemon.sh start regionserver
+ "$HADOOP_HOME"/bin/hadoop-daemon.sh start tasktracker
+
fi
# Run this script on next boot
Modified:
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-zookeeper-remote.sh
URL:
http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-zookeeper-remote.sh?rev=883962&r1=883961&r2=883962&view=diff
==============================================================================
---
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-zookeeper-remote.sh
(original)
+++
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-zookeeper-remote.sh
Wed Nov 25 02:26:11 2009
@@ -40,7 +40,7 @@
# up epoll limits
# ok if this fails, only valid for kernels 2.6.27+
-sysctl -w fs.epoll.max_user_instance=32768
+sysctl -w fs.epoll.max_user_instance=32768 > /dev/null 2>&1
mkdir -p /mnt/hbase/logs
mkdir -p /mnt/hbase/zk
