Updated Branches: refs/heads/branch-0.8 1d929832f -> cee3b4365
Merge pull request #63 from pwendell/master Fixing spark streaming example and a bug in examples build. - Examples assembly included a log4j.properties which clobbered Spark's - Example had an error where some classes weren't serializable - Did some other clean-up in this example (cherry picked from commit 28e9c2abc0884d096fc3be1e2d1f9ee18ffc3261) Signed-off-by: Patrick Wendell <pwend...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/cee3b436 Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/cee3b436 Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/cee3b436 Branch: refs/heads/branch-0.8 Commit: cee3b4365b3fdef0712caa9ac5ca9360d45148e1 Parents: 1d92983 Author: Matei Zaharia <ma...@eecs.berkeley.edu> Authored: Tue Oct 15 23:59:56 2013 -0700 Committer: Patrick Wendell <pwend...@gmail.com> Committed: Wed Oct 16 10:37:55 2013 -0700 ---------------------------------------------------------------------- .../examples/clickstream/PageViewGenerator.scala | 13 +++++++++---- project/SparkBuild.scala | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/cee3b436/examples/src/main/scala/org/apache/spark/streaming/examples/clickstream/PageViewGenerator.scala ---------------------------------------------------------------------- diff --git a/examples/src/main/scala/org/apache/spark/streaming/examples/clickstream/PageViewGenerator.scala b/examples/src/main/scala/org/apache/spark/streaming/examples/clickstream/PageViewGenerator.scala index 884d6d6..de70c50 100644 --- a/examples/src/main/scala/org/apache/spark/streaming/examples/clickstream/PageViewGenerator.scala +++ b/examples/src/main/scala/org/apache/spark/streaming/examples/clickstream/PageViewGenerator.scala @@ -17,17 +17,19 @@ package org.apache.spark.streaming.examples.clickstream -import java.net.{InetAddress,ServerSocket,Socket,SocketException} -import java.io.{InputStreamReader, BufferedReader, PrintWriter} +import java.net.ServerSocket +import java.io.PrintWriter import util.Random /** Represents a page view on a website with associated dimension data.*/ -class PageView(val url : String, val status : Int, val zipCode : Int, val userID : Int) { +class PageView(val url : String, val status : Int, val zipCode : Int, val userID : Int) + extends Serializable { override def toString() : String = { "%s\t%s\t%s\t%s\n".format(url, status, zipCode, userID) } } -object PageView { + +object PageView extends Serializable { def fromString(in : String) : PageView = { val parts = in.split("\t") new PageView(parts(0), parts(1).toInt, parts(2).toInt, parts(3).toInt) @@ -39,6 +41,9 @@ object PageView { * This should be used in tandem with PageViewStream.scala. Example: * $ ./run-example spark.streaming.examples.clickstream.PageViewGenerator 44444 10 * $ ./run-example spark.streaming.examples.clickstream.PageViewStream errorRatePerZipCode localhost 44444 + * + * When running this, you may want to set the root logging level to ERROR in + * conf/log4j.properties to reduce the verbosity of the output. * */ object PageViewGenerator { val pages = Map("http://foo.com/"; -> .7, http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/cee3b436/project/SparkBuild.scala ---------------------------------------------------------------------- diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 0843170..f6cbc1d 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -311,6 +311,7 @@ object SparkBuild extends Build { mergeStrategy in assembly := { case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard case m if m.toLowerCase.matches("meta-inf.*\\.sf$") => MergeStrategy.discard + case "log4j.properties" => MergeStrategy.discard case "META-INF/services/org.apache.hadoop.fs.FileSystem" => MergeStrategy.concat case "reference.conf" => MergeStrategy.concat case _ => MergeStrategy.first
