Author: cdouglas
Date: Wed Jul 16 11:46:09 2008
New Revision: 677375
URL: http://svn.apache.org/viewvc?rev=677375&view=rev
Log:
HADOOP-3755. Update gridmix to work with HOD 0.4 Contributed by Runping Qi.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/benchmarks/gridmix/generateData.sh
hadoop/core/trunk/src/benchmarks/gridmix/gridmix-env
hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/maxentHod
hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/monsterQueriesHod
hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/textSortHod
hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/webdataScanHod
hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/webdataSortHod
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=677375&r1=677374&r2=677375&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Jul 16 11:46:09 2008
@@ -120,6 +120,8 @@
HADOOP-3721. Refactor CompositeRecordReader and related mapred.join classes
to make them clearer. (cdouglas)
+ HADOOP-3755. Update gridmix to work with HOD 0.4 (Runping Qi via cdouglas)
+
Release 0.18.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/core/trunk/src/benchmarks/gridmix/generateData.sh
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/benchmarks/gridmix/generateData.sh?rev=677375&r1=677374&r2=677375&view=diff
==============================================================================
--- hadoop/core/trunk/src/benchmarks/gridmix/generateData.sh (original)
+++ hadoop/core/trunk/src/benchmarks/gridmix/generateData.sh Wed Jul 16
11:46:09 2008
@@ -13,6 +13,7 @@
if [ -z ${NUM_MAPS} ] ; then
NUM_MAPS=100
fi
+
INDIRECT_DATA_FILES=200
# If the env var USE_REAL_DATASET is set, then use the params to generate the
bigger (real) dataset.
@@ -37,7 +38,7 @@
-D mapred.output.compress=true \
-D mapred.map.output.compression.type=BLOCK \
-outFormat org.apache.hadoop.mapred.SequenceFileOutputFormat \
- ${VARCOMPSEQ}
+ ${VARCOMPSEQ} &
${HADOOP_HOME}/bin/hadoop jar \
${EXAMPLE_JAR} randomtextwriter \
@@ -50,7 +51,7 @@
-D mapred.output.compress=true \
-D mapred.map.output.compression.type=BLOCK \
-outFormat org.apache.hadoop.mapred.SequenceFileOutputFormat \
- ${FIXCOMPSEQ}
+ ${FIXCOMPSEQ} &
${HADOOP_HOME}/bin/hadoop jar \
${EXAMPLE_JAR} randomtextwriter \
@@ -62,7 +63,7 @@
-D test.randomtextwrite.max_words_value=200 \
-D mapred.output.compress=false \
-outFormat org.apache.hadoop.mapred.TextOutputFormat \
- ${VARINFLTEXT}
+ ${VARINFLTEXT} &
${HADOOP_HOME}/bin/hadoop jar \
${EXAMPLE_JAR} randomtextwriter \
@@ -75,4 +76,4 @@
-D mapred.output.compress=true \
-D mapred.map.output.compression.type=BLOCK \
-outFormat org.apache.hadoop.mapred.TextOutputFormat \
- ${FIXCOMPTEXT}
+ ${FIXCOMPTEXT} &
Modified: hadoop/core/trunk/src/benchmarks/gridmix/gridmix-env
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/benchmarks/gridmix/gridmix-env?rev=677375&r1=677374&r2=677375&view=diff
==============================================================================
--- hadoop/core/trunk/src/benchmarks/gridmix/gridmix-env (original)
+++ hadoop/core/trunk/src/benchmarks/gridmix/gridmix-env Wed Jul 16 11:46:09
2008
@@ -15,17 +15,17 @@
# Hadoop example jar
# set var only if it has not already been set externally
if [ -z "${EXAMPLE_JAR}" ] ; then
- export EXAMPLE_JAR="${HADOOP_HOME}/hadoop-*-examples.jar"
+ export EXAMPLE_JAR="${HADOOP_HOME}/hadoop-*examples.jar"
fi
# Hadoop test jar
# set var only if it has not already been set externally
if [ -z "${APP_JAR}" ] ; then
- export APP_JAR="${HADOOP_HOME}/hadoop-*-test.jar"
+ export APP_JAR="${HADOOP_HOME}/hadoop-*test.jar"
fi
# Hadoop streaming jar
# set var only if it has not already been set externally
if [ -z "${STREAM_JAR}" ] ; then
- export STREAM_JAR="${HADOOP_HOME}/contrib/hadoop-*-streaming.jar"
+ export STREAM_JAR="${HADOOP_HOME}/contrib/streaming/hadoop-*streaming.jar"
fi
# Location on default filesystem for writing gridmix data (usually HDFS)
# Default: /gridmix/data
@@ -64,9 +64,13 @@
export INTERVAL_BETWEEN_SUBMITION=20
## Hod
-#export
HOD_OPTIONS="--ringmaster.hadoop-tar-ball=/path/to/hadoop-0.15.0-dev.tar.gz"
-#export HOD_CONFIG=
-#export ALL_HOD_OPTIONS="-c ${HOD_CONFIG} ${HOD_OPTIONS}"
-#export SMALL_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -m 5"
-#export MEDIUM_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -m 50"
-#export LARGE_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -m 100"
+#export HOD_OPTIONS=""
+
+export CLUSTER_DIR_BASE=$GRID_MIX_HOME/CLUSTER_DIR_BASE
+export HOD_CONFIG=
+export ALL_HOD_OPTIONS="-c ${HOD_CONFIG} ${HOD_OPTIONS}
+export SMALL_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -n 5"
+export MEDIUM_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -n 50"
+export LARGE_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -n 100"
+export USE_REAL_DATASET=false
+
Modified: hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/maxentHod
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/maxentHod?rev=677375&r1=677374&r2=677375&view=diff
==============================================================================
--- hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/maxentHod
(original)
+++ hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/maxentHod Wed
Jul 16 11:46:09 2008
@@ -3,9 +3,13 @@
GRID_DIR=`dirname "$0"`
GRID_DIR=`cd "$GRID_DIR"; pwd`
source $GRID_DIR/../gridmix-env
+
for ((i=0; i < $NUM_OF_LARGE_JOBS_FOR_ENTROPY_CLASS; i++))
do
echo $i
- hod $LARGE_JOB_HOD_OPTIONS --hod.script=$GRID_MIX_HOME/maxent/maxent.large
2>&1 > maxent.large.$i.out &
+ CLUSTER_DIR=$CLUSTER_DIR_BASE/maxent.large.$i
+ mkdir $CLUSTER_DIR
+ #hod $LARGE_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/maxent/maxent.large 2>&1 > maxent.large.$i.out &
+ hod script -d $CLUSTER_DIR $LARGE_JOB_HOD_OPTIONS -s
$GRID_MIX_HOME/maxent/maxent.large 2>&1 > maxent.large.$i.out &
$GRID_MIX_HOME/submissionScripts/sleep_if_too_busy
done
Modified:
hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/monsterQueriesHod
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/monsterQueriesHod?rev=677375&r1=677374&r2=677375&view=diff
==============================================================================
---
hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/monsterQueriesHod
(original)
+++
hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/monsterQueriesHod
Wed Jul 16 11:46:09 2008
@@ -7,20 +7,27 @@
for ((i=0; i < $NUM_OF_SMALL_JOBS_PER_CLASS; i++))
do
echo $i
- hod $SMALL_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/monsterQuery/monster_query.small 2>&1 >
monster_query.small.$i.out &
+ CLUSTER_DIR=$CLUSTER_DIR_BASE/monster_query.small.$i
+ mkdir $CLUSTER_DIR
+
+ hod script -d $CLUSTER_DIR $SMALL_JOB_HOD_OPTIONS -s
$GRID_MIX_HOME/monsterQuery/monster_query.small 2>&1 >
monster_query.small.$i.out &
$GRID_MIX_HOME/submissionScripts/sleep_if_too_busy
done
for ((i=0; i < $NUM_OF_MEDIUM_JOBS_PER_CLASS; i++))
do
echo $i
- hod $MEDIUM_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/monsterQuery/monster_query.medium 2>&1 >
monster_query.medium.$i.out &
+ CLUSTER_DIR=$CLUSTER_DIR_BASE/monster_query.medium.$i
+ mkdir $CLUSTER_DIR
+ hod script -d $CLUSTER_DIR $MEDIUM_JOB_HOD_OPTIONS -s
$GRID_MIX_HOME/monsterQuery/monster_query.medium 2>&1 >
monster_query.medium.$i.out &
$GRID_MIX_HOME/submissionScripts/sleep_if_too_busy
done
for ((i=0; i < $NUM_OF_LARGE_JOBS_PER_CLASS; i++))
do
echo $i
- hod $LARGE_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/monsterQuery/monster_query.large 2>&1 >
monster_query.large.$i.out &
+ CLUSTER_DIR=$CLUSTER_DIR_BASE/monster_query.large.$i
+ mkdir $CLUSTER_DIR
+ hod script -d $CLUSTER_DIR $LARGE_JOB_HOD_OPTIONS -s
$GRID_MIX_HOME/monsterQuery/monster_query.large 2>&1 >
monster_query.large.$i.out &
$GRID_MIX_HOME/submissionScripts/sleep_if_too_busy
done
Modified: hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/textSortHod
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/textSortHod?rev=677375&r1=677374&r2=677375&view=diff
==============================================================================
--- hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/textSortHod
(original)
+++ hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/textSortHod Wed
Jul 16 11:46:09 2008
@@ -7,33 +7,58 @@
for ((i=0; i < $NUM_OF_SMALL_JOBS_PER_CLASS; i++))
do
echo $i
- hod $SMALL_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/pipesort/text-sort.small 2>&1 >
pipesort.small.$i.out &
+ CLUSTER_DIR=$CLUSTER_DIR_BASE/pipesort.small.$i
+ mkdir $CLUSTER_DIR
+
+ hod script -d $CLUSTER_DIR $SMALL_JOB_HOD_OPTIONS -s
$GRID_MIX_HOME/pipesort/text-sort.small 2>&1 > pipesort.small.$i.out &
$GRID_MIX_HOME/submissionScripts/sleep_if_too_busy
- hod $SMALL_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/streamsort/text-sort.small 2>&1 >
streamsort.small.$i.out &
+
+ CLUSTER_DIR=$CLUSTER_DIR_BASE/streamsort.small.$i
+ mkdir $CLUSTER_DIR
+ hod script -d $CLUSTER_DIR $SMALL_JOB_HOD_OPTIONS -s
$GRID_MIX_HOME/streamsort/text-sort.small 2>&1 > streamsort.small.$i.out &
$GRID_MIX_HOME/submissionScripts/sleep_if_too_busy
- hod $SMALL_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/javasort/text-sort.small 2>&1 >
javasort.small.$i.out &
+
+ CLUSTER_DIR=$CLUSTER_DIR_BASE/javasort.small.$i
+ mkdir $CLUSTER_DIR
+ hod script -d $CLUSTER_DIR $SMALL_JOB_HOD_OPTIONS -s
$GRID_MIX_HOME/javasort/text-sort.small 2>&1 > javasort.small.$i.out &
$GRID_MIX_HOME/submissionScripts/sleep_if_too_busy
done
for ((i=0; i < $NUM_OF_MEDIUM_JOBS_PER_CLASS; i++))
do
echo $i
- hod $MEDIUM_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/pipesort/text-sort.medium 2>&1 >
pipesort.medium.$i.out &
+ CLUSTER_DIR=$CLUSTER_DIR_BASE/pipesort.medium.$i
+ mkdir $CLUSTER_DIR
+ hod script -d $CLUSTER_DIR $MEDIUM_JOB_HOD_OPTIONS -s
$GRID_MIX_HOME/pipesort/text-sort.medium 2>&1 > pipesort.medium.$i.out &
$GRID_MIX_HOME/submissionScripts/sleep_if_too_busy
- hod $MEDIUM_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/streamsort/text-sort.medium 2>&1 >
streamsort.medium.$i.out &
+
+ CLUSTER_DIR=$CLUSTER_DIR_BASE/streamsort.medium.$i
+ mkdir $CLUSTER_DIR
+ hod script -d $CLUSTER_DIR $MEDIUM_JOB_HOD_OPTIONS -s
$GRID_MIX_HOME/streamsort/text-sort.medium 2>&1 > streamsort.medium.$i.out &
$GRID_MIX_HOME/submissionScripts/sleep_if_too_busy
- hod $MEDIUM_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/javasort/text-sort.medium 2>&1 >
javasort.medium.$i.out &
+
+ CLUSTER_DIR=$CLUSTER_DIR_BASE/javasort.medium.$i
+ mkdir $CLUSTER_DIR
+ hod script -d $CLUSTER_DIR $MEDIUM_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/javasort/text-sort.medium 2>&1 >
javasort.medium.$i.out &
$GRID_MIX_HOME/submissionScripts/sleep_if_too_busy
done
for ((i=0; i < $NUM_OF_LARGE_JOBS_PER_CLASS; i++))
do
echo $i
- hod $LARGE_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/pipesort/text-sort.large 2>&1 >
pipesort.large.$i.out &
+ CLUSTER_DIR=$CLUSTER_DIR_BASE/pipesort.large.$i
+ mkdir $CLUSTER_DIR
+ hod script -d $CLUSTER_DIR $LARGE_JOB_HOD_OPTIONS -s
$GRID_MIX_HOME/pipesort/text-sort.large 2>&1 > pipesort.large.$i.out &
$GRID_MIX_HOME/submissionScripts/sleep_if_too_busy
- hod $LARGE_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/streamsort/text-sort.large 2>&1 >
streamsort.large.$i.out &
+
+ CLUSTER_DIR=$CLUSTER_DIR_BASE/streamsort.large.$i
+ mkdir $CLUSTER_DIR
+ hod script -d $CLUSTER_DIR $LARGE_JOB_HOD_OPTIONS -s
$GRID_MIX_HOME/streamsort/text-sort.large 2>&1 > streamsort.large.$i.out &
$GRID_MIX_HOME/submissionScripts/sleep_if_too_busy
- hod $LARGE_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/javasort/text-sort.large 2>&1 >
javasort.large.$i.out &
+
+ CLUSTER_DIR=$CLUSTER_DIR_BASE/javasort.large.$i
+ mkdir $CLUSTER_DIR
+ hod script -d $CLUSTER_DIR $LARGE_JOB_HOD_OPTIONS -s
$GRID_MIX_HOME/javasort/text-sort.large 2>&1 > javasort.large.$i.out &
$GRID_MIX_HOME/submissionScripts/sleep_if_too_busy
done
Modified:
hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/webdataScanHod
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/webdataScanHod?rev=677375&r1=677374&r2=677375&view=diff
==============================================================================
--- hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/webdataScanHod
(original)
+++ hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/webdataScanHod
Wed Jul 16 11:46:09 2008
@@ -7,7 +7,9 @@
for ((i=0; i < $NUM_OF_SMALL_JOBS_PER_CLASS; i++))
do
echo $i
- hod $SMALL_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/webdatascan/webdata_scan.small 2>&1 >
webdata_scan.small.$i.out&
+ CLUSTER_DIR=$CLUSTER_DIR_BASE/webdata_scan.small.$i
+ mkdir $CLUSTER_DIR
+ hod script -d $CLUSTER_DIR $SMALL_JOB_HOD_OPTIONS -s
$GRID_MIX_HOME/webdatascan/webdata_scan.small 2>&1 > webdata_scan.small.$i.out&
$GRID_MIX_HOME/submissionScripts/sleep_if_too_busy
done
@@ -15,14 +17,18 @@
for ((i=0; i < $NUM_OF_MEDIUM_JOBS_PER_CLASS; i++))
do
echo $i
- hod $MEDIUM_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/webdatascan/webdata_scan.medium 2>&1 >
webdata_scan.medium.$i.out &
+ CLUSTER_DIR=$CLUSTER_DIR_BASE/webdata_scan.medium.$i
+ mkdir $CLUSTER_DIR
+ hod script -d $CLUSTER_DIR $MEDIUM_JOB_HOD_OPTIONS -s
$GRID_MIX_HOME/webdatascan/webdata_scan.medium 2>&1 >
webdata_scan.medium.$i.out &
$GRID_MIX_HOME/submissionScripts/sleep_if_too_busy
done
for ((i=0; i < $NUM_OF_LARGE_JOBS_PER_CLASS; i++))
do
echo $i
- hod $LARGE_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/webdatascan/webdata_scan.large 2>&1 >
webdata_scan.large.$i.out &
+ CLUSTER_DIR=$CLUSTER_DIR_BASE/webdata_scan.large.$i
+ mkdir $CLUSTER_DIR
+ hod script -d $CLUSTER_DIR $LARGE_JOB_HOD_OPTIONS -s
$GRID_MIX_HOME/webdatascan/webdata_scan.large 2>&1 > webdata_scan.large.$i.out
&
$GRID_MIX_HOME/submissionScripts/sleep_if_too_busy
done
Modified:
hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/webdataSortHod
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/webdataSortHod?rev=677375&r1=677374&r2=677375&view=diff
==============================================================================
--- hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/webdataSortHod
(original)
+++ hadoop/core/trunk/src/benchmarks/gridmix/submissionScripts/webdataSortHod
Wed Jul 16 11:46:09 2008
@@ -4,11 +4,14 @@
GRID_DIR=`cd "$GRID_DIR"; pwd`
source $GRID_DIR/../gridmix-env
-
for ((i=0; i < $NUM_OF_LARGE_JOBS_PER_CLASS; i++))
do
echo $i
- hod $LARGE_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/webdatasort/webdata_sort.large 2>&1 >
webdata_sort.large.$i.out &
+ CLUSTER_DIR=$CLUSTER_DIR_BASE/webdata_sort.large.$i
+ mkdir $CLUSTER_DIR
+ #hod $LARGE_JOB_HOD_OPTIONS
--hod.script=$GRID_MIX_HOME/webdatasort/webdata_sort.large 2>&1 >
webdata_sort.large.$i.out &
+ echo "hod script -d $CLUSTER_DIR $LARGE_JOB_HOD_OPTIONS -s
$GRID_MIX_HOME/webdatasort/webdata_sort.large "
+ hod script -d $CLUSTER_DIR $LARGE_JOB_HOD_OPTIONS -s
$GRID_MIX_HOME/webdatasort/webdata_sort.large 2>&1 > webdata_sort.large.$i.out
&
$GRID_MIX_HOME/submissionScripts/sleep_if_too_busy
done
