Updated Branches: refs/heads/master 71a77cdfe -> 0868e7110
MRQL-9: Make BSP mode compatible with Hama 0.7.0 Project: http://git-wip-us.apache.org/repos/asf/incubator-mrql/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-mrql/commit/0868e711 Tree: http://git-wip-us.apache.org/repos/asf/incubator-mrql/tree/0868e711 Diff: http://git-wip-us.apache.org/repos/asf/incubator-mrql/diff/0868e711 Branch: refs/heads/master Commit: 0868e71100c9cc47942acc7f205a5e947311d23f Parents: 71a77cd Author: fegaras <[email protected]> Authored: Fri May 3 17:42:38 2013 -0500 Committer: fegaras <[email protected]> Committed: Fri May 3 17:42:38 2013 -0500 ---------------------------------------------------------------------- Makefile | 6 +++--- build.xml | 14 +++++++++----- conf/mrql-env.sh | 8 +++++--- src/BSP/BSPPlan.java | 14 +++++--------- src/Config.java | 1 + 5 files changed, 23 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/0868e711/Makefile ---------------------------------------------------------------------- diff --git a/Makefile b/Makefile index c9be632..1009d22 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ #-------------------------------------------------------------------------------- # # Makefile for MRQL -# Requires: jflex, cup (they can be install as Linux packages) +# Requires: jflex (it can be installed as a Linux package) # #-------------------------------------------------------------------------------- @@ -35,7 +35,7 @@ JAVAC = ${JAVA_HOME}/bin/javac -g:none -d classes JAVA = ${JAVA_HOME}/bin/java JAR = ${JAVA_HOME}/bin/jar JFLEX = jflex --quiet --nobak -CUP = cup -nosummary +CUP = ${JAVA} -jar ${CUP_JAR} -nosummary GEN = ${JAVA} Gen.Main sources := src/*.java @@ -59,7 +59,7 @@ common: clean_build mrql_parser json_parser clean_build: @rm -rf classes tmp - @mkdir -p classes tmp + @mkdir -p classes tmp tests/results tests/results/memory tests/results/hadoop tests/results/bsp mrql_parser: @${JFLEX} src/mrql.lex -d tmp http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/0868e711/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index b7045d1..680d7d2 100644 --- a/build.xml +++ b/build.xml @@ -19,7 +19,7 @@ <!-- Ant build file for MRQL - Requires: jflex, cup (they can be install as Linux packages) + Requires: jflex (it can be installed as a Linux package) --> @@ -104,6 +104,10 @@ <delete dir="tmp"/> <mkdir dir="classes"/> <mkdir dir="tmp"/> + <mkdir dir="tests/results"/> + <mkdir dir="tests/results/memory"/> + <mkdir dir="tests/results/hadoop"/> + <mkdir dir="tests/results/bsp"/> </target> <target name="mrql_parser" @@ -120,11 +124,11 @@ <arg line="-o"/> <arg file="tmp/mrql.cup"/> </java> - <exec executable="cup"> + <java jar="${CUP_JAR}" fork='true'> <arg line="-nosummary"/> <arg line="-parser MRQLParser"/> <arg file="tmp/mrql.cup"/> - </exec> + </java> <move todir="tmp"> <filelist dir="."> <file name="sym.java"/> @@ -141,12 +145,12 @@ <arg line="-d"/> <arg file="tmp"/> </exec> - <exec executable="cup"> + <java jar="${CUP_JAR}" fork='true'> <arg line="-nosummary"/> <arg line="-parser JSONParser"/> <arg line="-symbols jsym"/> <arg file="src/JSON.cup"/> - </exec> + </java> <move todir="tmp"> <filelist dir="."> <file name="jsym.java"/> http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/0868e711/conf/mrql-env.sh ---------------------------------------------------------------------- diff --git a/conf/mrql-env.sh b/conf/mrql-env.sh index be8b3e1..53bb0da 100644 --- a/conf/mrql-env.sh +++ b/conf/mrql-env.sh @@ -47,10 +47,12 @@ HAMA_CONF=${MRQL_HOME}/conf/conf-hama CUP_JAR=/usr/share/java/cup.jar # Hadoop libraries are from a Hadoop tarball -MRQL_CLASSPATH=classes:${HAMA_JAR}:${HADOOP_HOME}/hadoop-core-${HADOOP_VERSION}.jar:${HADOOP_HOME}/lib/commons-cli-1.2.jar +HADOOP_JARS=${HADOOP_HOME}/hadoop-core-${HADOOP_VERSION}.jar:${HADOOP_HOME}/lib/commons-cli-1.2.jar # Hadoop libraries are from a YARN tarball -# MRQL_CLASSPATH=classes:$HAMA_JAR:$HADOOP_HOME/share/hadoop/common/hadoop-common-$HADOOP_VERSION.jar:$HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-client-core-$HADOOP_VERSION.jar:$HADOOP_HOME/share/hadoop/hdfs/hadoop-hdfs-$HADOOP_VERSION.jar:$HADOOP_HOME/share/hadoop/common/lib/hadoop-annotations-$HADOOP_VERSION.jar:$HADOOP_HOME/share/hadoop/common/lib/commons-cli-1.2.jar +#HADOOP_JARS=${HADOOP_HOME}/share/hadoop/common/hadoop-common-$HADOOP_VERSION.jar:${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-mapreduce-client-core-$HADOOP_VERSION.jar:${HADOOP_HOME}/share/hadoop/hdfs/hadoop-hdfs-$HADOOP_VERSION.jar:${HADOOP_HOME}/share/hadoop/common/lib/hadoop-annotations-$HADOOP_VERSION.jar:${HADOOP_HOME}/share/hadoop/common/lib/commons-cli-1.2.jar # Hadoop libraries are from a Cloudera package -#MRQL_CLASSPATH=classes:$HAMA_JAR:/usr/lib/hadoop-0.20/hadoop-core.jar:/usr/lib/hadoop-0.20/lib/commons-cli-1.2.jar +#HADOOP_JARS=/usr/lib/hadoop-0.20/hadoop-core.jar:/usr/lib/hadoop-0.20/lib/commons-cli-1.2.jar + +MRQL_CLASSPATH=classes:${HAMA_JAR}:${HADOOP_JARS} http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/0868e711/src/BSP/BSPPlan.java ---------------------------------------------------------------------- diff --git a/src/BSP/BSPPlan.java b/src/BSP/BSPPlan.java index 4c22c41..8168789 100644 --- a/src/BSP/BSPPlan.java +++ b/src/BSP/BSPPlan.java @@ -57,8 +57,6 @@ final public class BSPPlan extends Plan { Bag msg_cache; // the cache that holds all local data in memory Tuple local_cache; - // syncronization point for local_cache - final static Integer cache_sync = new Integer(0); /** shuffle values to BSP peers based on uniform hashing on key */ private static String shuffle ( MRData key ) { @@ -76,7 +74,7 @@ final public class BSPPlan extends Plan { try { // this case is only used for checking the exit condition of repeat/closure boolean exit = mr_exit.get(); - if (!exit) + if (!exit && all_peers.length > 1) // this peer is not ready to exit, so no peer should exit for ( String p: this_peer.getAllPeerNames() ) this_peer.send(p,new MRContainer(more_supersteps)); @@ -227,12 +225,10 @@ final public class BSPPlan extends Plan { }; pair.set(0,msg_cache); pair.set(1,state); - synchronized (cache_sync) { - this_peer = peer; - cache = local_cache; - // evaluate one superstep - result = (Tuple)superstep_fnc.eval(pair); - }; + this_peer = peer; + cache = local_cache; + // evaluate one superstep + result = (Tuple)superstep_fnc.eval(pair); Bag msgs = (Bag)result.get(0); exit = ((MR_bool)result.get(2)).get(); state = result.get(1); http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/0868e711/src/Config.java ---------------------------------------------------------------------- diff --git a/src/Config.java b/src/Config.java index be06206..00c1993 100644 --- a/src/Config.java +++ b/src/Config.java @@ -225,6 +225,7 @@ final public class Config { } else if (args[i].equals("-trace_execution")) { trace_execution = true; trace_exp_execution = true; + compile_functional_arguments = false; i++; } else if (args[i].equals("-methods")) { System.out.print("\nImported methods: ");
