Updated Branches: refs/heads/master 8d528af82 -> e5316d068
BroadcastTest2 --> BroadcastTest Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/e96bd006 Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/e96bd006 Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/e96bd006 Branch: refs/heads/master Commit: e96bd0068f13907a45507030e9ca0b178c193823 Parents: a8d0981 Author: Mosharaf Chowdhury <mosha...@cs.berkeley.edu> Authored: Wed Oct 16 21:28:03 2013 -0700 Committer: Mosharaf Chowdhury <mosha...@cs.berkeley.edu> Committed: Wed Oct 16 21:33:33 2013 -0700 ---------------------------------------------------------------------- .../apache/spark/examples/BroadcastTest.scala | 15 ++++- .../apache/spark/examples/BroadcastTest2.scala | 59 -------------------- 2 files changed, 12 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/e96bd006/examples/src/main/scala/org/apache/spark/examples/BroadcastTest.scala ---------------------------------------------------------------------- diff --git a/examples/src/main/scala/org/apache/spark/examples/BroadcastTest.scala b/examples/src/main/scala/org/apache/spark/examples/BroadcastTest.scala index 868ff81..529709c 100644 --- a/examples/src/main/scala/org/apache/spark/examples/BroadcastTest.scala +++ b/examples/src/main/scala/org/apache/spark/examples/BroadcastTest.scala @@ -22,12 +22,19 @@ import org.apache.spark.SparkContext object BroadcastTest { def main(args: Array[String]) { if (args.length == 0) { - System.err.println("Usage: BroadcastTest <master> [<slices>] [numElem]") + System.err.println("Usage: BroadcastTest <master> [slices] [numElem] [broadcastAlgo] [blockSize]") System.exit(1) } - val sc = new SparkContext(args(0), "Broadcast Test", + val bcName = if (args.length > 3) args(3) else "Http" + val blockSize = if (args.length > 4) args(4) else "4096" + + System.setProperty("spark.broadcast.factory", "org.apache.spark.broadcast." + bcName + "BroadcastFactory") + System.setProperty("spark.broadcast.blockSize", blockSize) + + val sc = new SparkContext(args(0), "Broadcast Test 2", System.getenv("SPARK_HOME"), Seq(System.getenv("SPARK_EXAMPLES_JAR"))) + val slices = if (args.length > 1) args(1).toInt else 2 val num = if (args.length > 2) args(2).toInt else 1000000 @@ -36,13 +43,15 @@ object BroadcastTest { arr1(i) = i } - for (i <- 0 until 2) { + for (i <- 0 until 3) { println("Iteration " + i) println("===========") + val startTime = System.nanoTime val barr1 = sc.broadcast(arr1) sc.parallelize(1 to 10, slices).foreach { i => println(barr1.value.size) } + println("Iteration %d took %.0f milliseconds".format(i, (System.nanoTime - startTime) / 1E6)) } System.exit(0) http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/e96bd006/examples/src/main/scala/org/apache/spark/examples/BroadcastTest2.scala ---------------------------------------------------------------------- diff --git a/examples/src/main/scala/org/apache/spark/examples/BroadcastTest2.scala b/examples/src/main/scala/org/apache/spark/examples/BroadcastTest2.scala deleted file mode 100644 index 4b96d0c..0000000 --- a/examples/src/main/scala/org/apache/spark/examples/BroadcastTest2.scala +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.examples - -import org.apache.spark.SparkContext - -object BroadcastTest2 { - def main(args: Array[String]) { - if (args.length == 0) { - System.err.println("Usage: BroadcastTest2 <master> [slices] [numElem] [broadcastAlgo] [blockSize]") - System.exit(1) - } - - val bcName = if (args.length > 3) args(3) else "Http" - val blockSize = if (args.length > 4) args(4) else "4096" - - System.setProperty("spark.broadcast.factory", "org.apache.spark.broadcast." + bcName + "BroadcastFactory") - System.setProperty("spark.broadcast.blockSize", blockSize) - - val sc = new SparkContext(args(0), "Broadcast Test 2", - System.getenv("SPARK_HOME"), Seq(System.getenv("SPARK_EXAMPLES_JAR"))) - - val slices = if (args.length > 1) args(1).toInt else 2 - val num = if (args.length > 2) args(2).toInt else 1000000 - - var arr1 = new Array[Int](num) - for (i <- 0 until arr1.length) { - arr1(i) = i - } - - for (i <- 0 until 3) { - println("Iteration " + i) - println("===========") - val startTime = System.nanoTime - val barr1 = sc.broadcast(arr1) - sc.parallelize(1 to 10, slices).foreach { - i => println(barr1.value.size) - } - println("Iteration %d took %.0f milliseconds".format(i, (System.nanoTime - startTime) / 1E6)) - } - - System.exit(0) - } -}
