Repository: incubator-gearpump Updated Branches: refs/heads/master a47e1a6db -> ac8ac0392
[GEARPUMP-330] Allow examples to run in `sbt run` and Intellij Author: manuzhang <[email protected]> Closes #200 from manuzhang/run_examples_in_ide. Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/ac8ac039 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/ac8ac039 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/ac8ac039 Branch: refs/heads/master Commit: ac8ac039217725f9f924ec901fdf7adfd9f51657 Parents: a47e1a6 Author: manuzhang <[email protected]> Authored: Wed Jul 26 10:28:25 2017 +0800 Committer: manuzhang <[email protected]> Committed: Wed Jul 26 10:28:41 2017 +0800 ---------------------------------------------------------------------- docs/contents/introduction/quick-start.md | 34 ++++++++++ docs/mkdocs.yml | 3 +- .../examples/wordcount/dsl/WordCount.scala | 8 ++- project/BuildExamples.scala | 69 ++++++++++++-------- project/BuildExperiments.scala | 5 +- project/plugins.sbt | 4 +- 6 files changed, 88 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ac8ac039/docs/contents/introduction/quick-start.md ---------------------------------------------------------------------- diff --git a/docs/contents/introduction/quick-start.md b/docs/contents/introduction/quick-start.md new file mode 100644 index 0000000..c38e50f --- /dev/null +++ b/docs/contents/introduction/quick-start.md @@ -0,0 +1,34 @@ +This quick start will walk you through executing your first Gearpump pipeline to run WordCount written in Stream DSL. + +### Set up development environment +1. Download and install [Java Development Kit(JDK)](http://www.oracle.com/technetwork/java/javase/downloads/index.html) 1.8. Verify that [JAVA_HOME](https://docs.oracle.com/javase/8/docs/technotes/guides/troubleshoot/envvars001.html) is set and points to your JDK installation. +2. Download and install [sbt](http://www.scala-sbt.org/download.html) by following [sbt's installation guide](http://www.scala-sbt.org/0.13/docs/Setup.html) for your specific operating system. + +### Run WordCount + +1. Run WordCount example with sbt + + + :::bash + sbt "project gearpump-examples-wordcount" run + + +2. Select the third main class in the sbt console + + :::bash + Multiple main classes detected, select one to run: + [1] org.apache.gearpump.streaming.examples.wordcount.WordCount + [2] org.apache.gearpump.streaming.examples.wordcount.dsl.WindowedWordCount + [3] org.apache.gearpump.streaming.examples.wordcount.dsl.WordCount + + Enter number: 3 + + If everything goes fine, the following output is expected + + :::bash + (is,1)(bingo!!,2)(a,1)(good,1)(This,1)(start,,1) + + + + + http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ac8ac039/docs/mkdocs.yml ---------------------------------------------------------------------- diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 7ce5287..ae5cf1b 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -12,7 +12,8 @@ docs_dir: tmp pages: - Overview: 'index.md' -- Introduction: +- Introduction: + - 'Quick Start': 'introduction/quick-start.md' - 'Submit Your 1st Application': 'introduction/submit-your-1st-application.md' - 'Client Command Line': 'introduction/commandline.md' - 'Basic Concepts': 'introduction/basic-concepts.md' http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ac8ac039/examples/streaming/wordcount/src/main/scala/org/apache/gearpump/streaming/examples/wordcount/dsl/WordCount.scala ---------------------------------------------------------------------- diff --git a/examples/streaming/wordcount/src/main/scala/org/apache/gearpump/streaming/examples/wordcount/dsl/WordCount.scala b/examples/streaming/wordcount/src/main/scala/org/apache/gearpump/streaming/examples/wordcount/dsl/WordCount.scala index 5d1e607..65f63d2 100644 --- a/examples/streaming/wordcount/src/main/scala/org/apache/gearpump/streaming/examples/wordcount/dsl/WordCount.scala +++ b/examples/streaming/wordcount/src/main/scala/org/apache/gearpump/streaming/examples/wordcount/dsl/WordCount.scala @@ -19,6 +19,7 @@ package org.apache.gearpump.streaming.examples.wordcount.dsl import org.apache.gearpump.cluster.client.ClientContext +import org.apache.gearpump.cluster.embedded.EmbeddedCluster import org.apache.gearpump.cluster.main.{ArgumentsParser, CLIOption} import org.apache.gearpump.streaming.dsl.scalaapi.StreamApp import org.apache.gearpump.streaming.dsl.scalaapi.StreamApp._ @@ -30,16 +31,19 @@ object WordCount extends AkkaApp with ArgumentsParser { override val options: Array[(String, CLIOption[Any])] = Array.empty override def main(akkaConf: Config, args: Array[String]): Unit = { - val context = ClientContext(akkaConf) + val cluster = new EmbeddedCluster(akkaConf) + cluster.start() + val context: ClientContext = cluster.newClientContext val app = StreamApp("dsl", context) val data = "This is a good start, bingo!! bingo!!" app.source(data.lines.toList, 1, "source"). // word => (word, count) flatMap(line => line.split("[\\s]+")).map((_, 1)). // (word, count1), (word, count2) => (word, count1 + count2) - groupByKey().sum.log + groupByKey().sum.map(print) context.submit(app).waitUntilFinish() context.close() + cluster.stop() } } http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ac8ac039/project/BuildExamples.scala ---------------------------------------------------------------------- diff --git a/project/BuildExamples.scala b/project/BuildExamples.scala index d7390b5..00affb0 100644 --- a/project/BuildExamples.scala +++ b/project/BuildExamples.scala @@ -42,7 +42,7 @@ object BuildExamples extends sbt.Build { lazy val example_hbase = Project( id = "gearpump-examples-hbase", base = file("examples/streaming/hbase"), - settings = commonSettings ++ noPublish ++ myAssemblySettings ++ + settings = exampleSettings("org.apache.gearpump.streaming.examples.hbase.HBaseConn") ++ Seq( libraryDependencies ++= Seq( "org.apache.hadoop" % "hadoop-common" % hadoopVersion @@ -50,38 +50,37 @@ object BuildExamples extends sbt.Build { exclude("commons-beanutils", "commons-beanutils") exclude("asm", "asm") exclude("org.ow2.asm", "asm") - ), - mainClass in(Compile, packageBin) := - Some("org.apache.gearpump.streaming.examples.hbase.HBaseConn"), - - target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" / - CrossVersion.binaryScalaVersion(scalaVersion.value) - ) - ) dependsOn(streaming % "test->test; provided", core % "provided", external_hbase) + ) + ) ++ include("examples/streaming/hbase", "external/hbase") + ) dependsOn(core, streaming % "compile; test->test", external_hbase) lazy val wordcountJava = Project( id = "gearpump-examples-wordcountjava", base = file("examples/streaming/wordcount-java"), - settings = exampleSettings("org.apache.gearpump.streaming.examples.wordcountjava.WordCount") - ).dependsOn(core % "provided", streaming % "test->test; provided") + settings = exampleSettings("org.apache.gearpump.streaming.examples.wordcountjava.WordCount") ++ + include("examples/streaming/wordcount-java") + ).dependsOn(core, streaming % "compile; test->test") lazy val wordcount = Project( id = "gearpump-examples-wordcount", base = file("examples/streaming/wordcount"), - settings = exampleSettings("org.apache.gearpump.streaming.examples.wordcount.WordCount") - ).dependsOn(core % "provided", streaming % "test->test; provided") + settings = exampleSettings("org.apache.gearpump.streaming.examples.wordcount.dsl.WordCount") ++ + include("examples/streaming/wordcount") + ).dependsOn(core, streaming % "compile; test->test") lazy val sol = Project( id = "gearpump-examples-sol", base = file("examples/streaming/sol"), - settings = exampleSettings("org.apache.gearpump.streaming.examples.sol.SOL") - ).dependsOn(core % "provided", streaming % "test->test; provided") + settings = exampleSettings("org.apache.gearpump.streaming.examples.sol.SOL") ++ + include("examples/streaming/sol") + ).dependsOn(core, streaming % "compile; test->test") lazy val complexdag = Project( id = "gearpump-examples-complexdag", base = file("examples/streaming/complexdag"), - settings = exampleSettings("org.apache.gearpump.streaming.examples.complexdag.Dag") - ).dependsOn(core % "provided", streaming % "test->test; provided") + settings = exampleSettings("org.apache.gearpump.streaming.examples.complexdag.Dag") ++ + include("examples/streaming/complexdag") + ).dependsOn(core, streaming % "compile; test->test") lazy val distributedshell = Project( id = "gearpump-examples-distributedshell", @@ -91,8 +90,8 @@ object BuildExamples extends sbt.Build { Some("org.apache.gearpump.examples.distributedshell.DistributedShell"), target in assembly := baseDirectory.value.getParentFile / "target" / CrossVersion.binaryScalaVersion(scalaVersion.value) - ) - ).dependsOn(core % "test->test; provided") + ) ++ include("examples/distributedshell") + ).dependsOn(core % "compile; test->test") lazy val distributeservice = Project( id = "gearpump-examples-distributeservice", @@ -109,8 +108,8 @@ object BuildExamples extends sbt.Build { "io.spray" %% "spray-can" % sprayVersion, "io.spray" %% "spray-routing-shapeless2" % sprayVersion ) - ) - ).dependsOn(core % "test->test; provided") + ) ++ include("examples/distributeservice") + ).dependsOn(core % "compile; test->test") lazy val fsio = Project( id = "gearpump-examples-fsio", @@ -128,15 +127,16 @@ object BuildExamples extends sbt.Build { exclude("asm", "asm") exclude("org.ow2.asm", "asm") ) - ) - ).dependsOn(core % "provided", streaming % "test->test; provided") + ) ++ include("examples/streaming/fsio") + ).dependsOn(core, streaming % "compile; test->test") lazy val examples_kafka = Project( id = "gearpump-examples-kafka", base = file("examples/streaming/kafka"), settings = - exampleSettings("org.apache.gearpump.streaming.examples.kafka.wordcount.KafkaWordCount") - ).dependsOn(core % "provided", streaming % "test->test; provided", external_kafka) + exampleSettings("org.apache.gearpump.streaming.examples.kafka.wordcount.KafkaWordCount") ++ + include("examples/streaming/kafka", "external/kafka") + ).dependsOn(core, streaming % "compile; test->test", external_kafka) lazy val examples_state = Project( id = "gearpump-examples-state", @@ -155,15 +155,18 @@ object BuildExamples extends sbt.Build { exclude("org.ow2.asm", "asm"), "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion ) - ) - ).dependsOn(core % "provided", streaming % "test->test; provided", + ) ++ include("examples/streaming/state", + "external/hadoopfs", "external/monoid", "external/serializer", "external/kafka") + ).dependsOn(core, streaming % "compile; test->test", external_hadoopfs, external_monoid, external_serializer, external_kafka) lazy val pagerank = Project( id = "gearpump-examples-pagerank", base = file("examples/pagerank"), - settings = exampleSettings("org.apache.gearpump.experiments.pagerank.example.PageRankExample") - ).dependsOn(core % "provided", streaming % "test->test; provided") + settings = + exampleSettings("org.apache.gearpump.experiments.pagerank.example.PageRankExample") ++ + include("examples/pagerank") + ).dependsOn(core, streaming % "compile; test->test") private def exampleSettings(className: String): Seq[Def.Setting[_]] = commonSettings ++ noPublish ++ myAssemblySettings ++ Seq( @@ -172,4 +175,12 @@ object BuildExamples extends sbt.Build { target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" / CrossVersion.binaryScalaVersion(scalaVersion.value) ) + + private def include(files: String*): Seq[Def.Setting[_]] = Seq( + assemblyExcludedJars in assembly := { + val cp = (fullClasspath in assembly).value + cp.filterNot(p => + files.exists(p.data.getAbsolutePath.contains)) + } + ) } http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ac8ac039/project/BuildExperiments.scala ---------------------------------------------------------------------- diff --git a/project/BuildExperiments.scala b/project/BuildExperiments.scala index 0d3adb0..4022e95 100644 --- a/project/BuildExperiments.scala +++ b/project/BuildExperiments.scala @@ -74,8 +74,9 @@ object BuildExperiments extends sbt.Build { libraryDependencies ++= Seq( "redis.clients" % "jedis" % jedisVersion ) - ) - ).dependsOn(core % "provided", streaming % "test->test; provided") + )) + .dependsOn(core % "provided", streaming % "test->test; provided") + .disablePlugins(sbtassembly.AssemblyPlugin) lazy val storm = Project( id = "gearpump-experiments-storm", http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ac8ac039/project/plugins.sbt ---------------------------------------------------------------------- diff --git a/project/plugins.sbt b/project/plugins.sbt index 33a7e50..b8bae36 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -18,11 +18,13 @@ resolvers += Resolver.bintrayIvyRepo("fvunicorn", "sbt-plugins") +resolvers += Resolver.bintrayIvyRepo("manuzhang", "sbt-plugins") + resolvers += Classpaths.sbtPluginReleases addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.8") -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.4" +addSbtPlugin("io.gearpump.sbt" % "sbt-assembly" % "0.14.5" exclude("org.apache.maven", "maven-plugin-api")) addSbtPlugin("io.gearpump.sbt" % "sbt-pack" % "0.7.7")
