[GEARPUMP-226] refactor build files Refactored build files as follows
1. break `Build.scala` into parts for different module categories, e.g. `BuildExternals`, `BuildDashboard`. 2. libraries to shade are now added as managed dependencies, shaded with their depending projects and finally assembled to a uber jar. For example, `gearpump-core-$version.jar` will now contain both the gearpump sources and shaded libraries, whose package names are both `org.apache.gearpump`. The original dependencies are excluded from `pack`. 3. package names in the codebase are changed as needed by 2 4. `gearpump-core` is now `provided` for `gearpump-streaming` and won't be transitive dependencies for projects depending on`gearpump-streaming`. `gearpump-core` needs to be explicitly included as `provided` dependency so I add it to the published pom files for users. Author: manuzhang <[email protected]> Closes #108 from manuzhang/fix_build. Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/0b56de39 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/0b56de39 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/0b56de39 Branch: refs/heads/master Commit: 0b56de39919452787b2533ac3d10ac1dd46b2f9d Parents: cbf59fb Author: manuzhang <[email protected]> Authored: Fri Nov 11 13:10:51 2016 +0800 Committer: huafengw <[email protected]> Committed: Fri Nov 11 13:10:51 2016 +0800 ---------------------------------------------------------------------- .travis.yml | 6 +- .../gearpump/transport/netty/MessageBatch.java | 2 +- .../apache/gearpump/cluster/UserConfig.scala | 2 +- .../apache/gearpump/metrics/AkkaReporter.scala | 2 +- .../org/apache/gearpump/metrics/Counter.scala | 4 +- .../org/apache/gearpump/metrics/Histogram.scala | 4 +- .../apache/gearpump/metrics/JvmMetricsSet.scala | 4 +- .../org/apache/gearpump/metrics/Meter.scala | 2 +- .../org/apache/gearpump/metrics/Metrics.scala | 2 +- .../metrics/MetricsReporterService.scala | 7 +- .../serializer/FastKryoSerializer.scala | 11 +- .../serializer/GearpumpSerialization.scala | 3 +- .../org/apache/gearpump/util/FileUtils.scala | 2 +- .../scala/org/apache/gearpump/util/Util.scala | 10 +- .../gearpump/jarstore/FileServerSpec.scala | 7 +- .../apache/gearpump/metrics/MetricsSpec.scala | 3 +- .../gearpump/serializer/SerializerSpec.scala | 21 +- .../apache/gearpump/util/FileUtilsSpec.scala | 4 +- .../gearpump/cluster/utils/SystemOperation.java | 2 +- .../cluster/worker/CGroupProcessLauncher.scala | 2 +- .../apache/gearpump/redis/RedisMessage.scala | 10 +- project/Build.scala | 559 ------------------- project/BuildDashboard.scala | 119 ++++ project/BuildExample.scala | 236 -------- project/BuildExamples.scala | 181 ++++++ project/BuildExperiments.scala | 118 ++++ project/BuildExternals.scala | 124 ++++ project/BuildGearpump.scala | 252 +++++++++ project/BuildIntegrationTest.scala | 93 --- project/BuildIntegrationTests.scala | 93 +++ project/BuildShaded.scala | 142 ----- project/Dependencies.scala | 102 ++++ project/Docs.scala | 61 ++ project/Pack.scala | 21 +- project/build.properties | 2 +- project/plugins.sbt | 2 +- .../streaming/appmaster/ClockService.scala | 2 +- .../dsl/window/impl/WindowRunner.scala | 8 +- .../streaming/metrics/ProcessorAggregator.scala | 3 +- .../streaming/task/SerializerResolver.scala | 2 +- .../gearpump/streaming/task/Subscription.scala | 2 +- .../gearpump/streaming/task/TaskActor.scala | 2 +- 42 files changed, 1135 insertions(+), 1099 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index e7e8abd..7151f88 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,10 +3,8 @@ language: - scala sudo: false before_script: -- rm -rf $HOME/.m2 -- rm -rf $HOME/.ivy2 -- mkdir -p $HOME/.sbt/launchers/0.13.11/ -- curl -L -o $HOME/.sbt/launchers/0.13.11/sbt-launch.jar http://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/0.13.11/sbt-launch.jar +- mkdir -p $HOME/.sbt/launchers/0.13.13/ +- curl -L -o $HOME/.sbt/launchers/0.13.13/sbt-launch.jar http://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/0.13.13/sbt-launch.jar script: - echo "TRAVIS_PULL_REQUEST" $TRAVIS_PULL_REQUEST - echo "TRAVIS_BRANCH" $TRAVIS_BRANCH http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/java/org/apache/gearpump/transport/netty/MessageBatch.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/gearpump/transport/netty/MessageBatch.java b/core/src/main/java/org/apache/gearpump/transport/netty/MessageBatch.java index d9de871..13ce9fa 100644 --- a/core/src/main/java/org/apache/gearpump/transport/netty/MessageBatch.java +++ b/core/src/main/java/org/apache/gearpump/transport/netty/MessageBatch.java @@ -18,7 +18,7 @@ package org.apache.gearpump.transport.netty; -import org.apache.gearpump.google.common.io.Closeables; +import com.google.common.io.Closeables; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBufferOutputStream; import org.jboss.netty.buffer.ChannelBuffers; http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/cluster/UserConfig.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/cluster/UserConfig.scala b/core/src/main/scala/org/apache/gearpump/cluster/UserConfig.scala index 3f42808..1946cfb 100644 --- a/core/src/main/scala/org/apache/gearpump/cluster/UserConfig.scala +++ b/core/src/main/scala/org/apache/gearpump/cluster/UserConfig.scala @@ -20,8 +20,8 @@ package org.apache.gearpump.cluster import akka.actor.{ActorSystem, ExtendedActorSystem} import akka.serialization.JavaSerializer +import com.google.common.io.BaseEncoding -import org.apache.gearpump.google.common.io.BaseEncoding /** * Immutable configuration http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/metrics/AkkaReporter.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/metrics/AkkaReporter.scala b/core/src/main/scala/org/apache/gearpump/metrics/AkkaReporter.scala index acd9493..9092beb 100644 --- a/core/src/main/scala/org/apache/gearpump/metrics/AkkaReporter.scala +++ b/core/src/main/scala/org/apache/gearpump/metrics/AkkaReporter.scala @@ -22,7 +22,7 @@ import scala.collection.JavaConverters._ import akka.actor.{ActorRef, ActorSystem} -import org.apache.gearpump.codahale.metrics.{Gauge => CodaGauge, MetricRegistry} +import com.codahale.metrics.{Gauge => CodaGauge, MetricRegistry} import org.apache.gearpump.metrics.Metrics.{Counter => CounterData, Gauge => GaugeData, Histogram => HistogramData, Meter => MeterData} import org.apache.gearpump.metrics.MetricsReporterService.ReportTo import org.apache.gearpump.util.LogUtil http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/metrics/Counter.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/metrics/Counter.scala b/core/src/main/scala/org/apache/gearpump/metrics/Counter.scala index 2a4a195..e778e3d 100644 --- a/core/src/main/scala/org/apache/gearpump/metrics/Counter.scala +++ b/core/src/main/scala/org/apache/gearpump/metrics/Counter.scala @@ -18,10 +18,10 @@ package org.apache.gearpump.metrics -import org.apache.gearpump.codahale.metrics.{Counter => CodaHaleCounter} +import com.codahale.metrics.{Counter => CodaHaleCounter} /** - * @see org.apache.gearpump.codahale.metrics.Counter + * @see com.codahale.metrics.Counter */ class Counter(val name: String, counter: CodaHaleCounter, sampleRate: Int = 1) { private var sampleCount = 0L http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/metrics/Histogram.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/metrics/Histogram.scala b/core/src/main/scala/org/apache/gearpump/metrics/Histogram.scala index 9d1e500..3483c1e 100644 --- a/core/src/main/scala/org/apache/gearpump/metrics/Histogram.scala +++ b/core/src/main/scala/org/apache/gearpump/metrics/Histogram.scala @@ -18,10 +18,10 @@ package org.apache.gearpump.metrics -import org.apache.gearpump.codahale.metrics.{Histogram => CodaHaleHistogram} +import com.codahale.metrics.{Histogram => CodaHaleHistogram} /** - * @see org.apache.gearpump.codahale.metrics.Histogram + * @see com.codahale.metrics.Histogram */ class Histogram(val name: String, histogram: CodaHaleHistogram, sampleRate: Int = 1) { private var sampleCount = 0L http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/metrics/JvmMetricsSet.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/metrics/JvmMetricsSet.scala b/core/src/main/scala/org/apache/gearpump/metrics/JvmMetricsSet.scala index 436d14f..56b4743 100644 --- a/core/src/main/scala/org/apache/gearpump/metrics/JvmMetricsSet.scala +++ b/core/src/main/scala/org/apache/gearpump/metrics/JvmMetricsSet.scala @@ -21,8 +21,8 @@ package org.apache.gearpump.metrics import java.util import scala.collection.JavaConverters._ -import org.apache.gearpump.codahale.metrics.jvm.{MemoryUsageGaugeSet, ThreadStatesGaugeSet} -import org.apache.gearpump.codahale.metrics.{Metric, MetricSet} +import com.codahale.metrics.jvm.{MemoryUsageGaugeSet, ThreadStatesGaugeSet} +import com.codahale.metrics.{Metric, MetricSet} class JvmMetricsSet(name: String) extends MetricSet { http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/metrics/Meter.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/metrics/Meter.scala b/core/src/main/scala/org/apache/gearpump/metrics/Meter.scala index d4b7871..55aa73f 100644 --- a/core/src/main/scala/org/apache/gearpump/metrics/Meter.scala +++ b/core/src/main/scala/org/apache/gearpump/metrics/Meter.scala @@ -18,7 +18,7 @@ package org.apache.gearpump.metrics -import org.apache.gearpump.codahale.metrics.{Meter => CodaHaleMeter} +import com.codahale.metrics.{Meter => CodaHaleMeter} /** See org.apache.gearpump.codahale.metrics.Meter */ class Meter(val name: String, meter: CodaHaleMeter, sampleRate: Int = 1) { http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/metrics/Metrics.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/metrics/Metrics.scala b/core/src/main/scala/org/apache/gearpump/metrics/Metrics.scala index 1ee3798..3737361 100644 --- a/core/src/main/scala/org/apache/gearpump/metrics/Metrics.scala +++ b/core/src/main/scala/org/apache/gearpump/metrics/Metrics.scala @@ -23,7 +23,7 @@ import scala.collection.JavaConverters._ import akka.actor._ import org.slf4j.Logger -import org.apache.gearpump.codahale.metrics._ +import com.codahale.metrics._ import org.apache.gearpump.metrics import org.apache.gearpump.util.LogUtil http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/metrics/MetricsReporterService.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/metrics/MetricsReporterService.scala b/core/src/main/scala/org/apache/gearpump/metrics/MetricsReporterService.scala index 94aa114..620dc61 100644 --- a/core/src/main/scala/org/apache/gearpump/metrics/MetricsReporterService.scala +++ b/core/src/main/scala/org/apache/gearpump/metrics/MetricsReporterService.scala @@ -20,12 +20,11 @@ package org.apache.gearpump.metrics import java.net.InetSocketAddress import java.util.concurrent.TimeUnit -import scala.concurrent.duration._ +import scala.concurrent.duration._ import akka.actor.{Actor, ActorRef} - -import org.apache.gearpump.codahale.metrics.graphite.{Graphite, GraphiteReporter} -import org.apache.gearpump.codahale.metrics.{MetricFilter, Slf4jReporter} +import com.codahale.metrics.{MetricFilter, Slf4jReporter} +import com.codahale.metrics.graphite.{Graphite, GraphiteReporter} import org.apache.gearpump.metrics.Metrics.{DemandMoreMetrics, ReportMetrics} import org.apache.gearpump.metrics.MetricsReporterService.ReportTo import org.apache.gearpump.util.Constants._ http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/serializer/FastKryoSerializer.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/serializer/FastKryoSerializer.scala b/core/src/main/scala/org/apache/gearpump/serializer/FastKryoSerializer.scala index f258c0f..cb3d2fd 100644 --- a/core/src/main/scala/org/apache/gearpump/serializer/FastKryoSerializer.scala +++ b/core/src/main/scala/org/apache/gearpump/serializer/FastKryoSerializer.scala @@ -19,19 +19,18 @@ package org.apache.gearpump.serializer import akka.actor.ExtendedActorSystem - -import org.apache.gearpump.esotericsoftware.kryo.Kryo.DefaultInstantiatorStrategy -import org.apache.gearpump.objenesis.strategy.StdInstantiatorStrategy -import org.apache.gearpump.romix.serialization.kryo.KryoSerializerWrapper +import com.esotericsoftware.kryo.Kryo.DefaultInstantiatorStrategy +import com.romix.akka.serialization.kryo.{KryoBasedSerializer, KryoSerializer} import org.apache.gearpump.serializer.FastKryoSerializer.KryoSerializationException import org.apache.gearpump.util.LogUtil +import org.objenesis.strategy.StdInstantiatorStrategy class FastKryoSerializer(system: ExtendedActorSystem) extends Serializer { private val LOG = LogUtil.getLogger(getClass) private val config = system.settings.config - private val kryoSerializer = new KryoSerializerWrapper(system) + private val kryoSerializer: KryoBasedSerializer = new KryoSerializer(system).serializer private val kryo = kryoSerializer.kryo val strategy = new DefaultInstantiatorStrategy strategy.setFallbackInstantiatorStrategy(new StdInstantiatorStrategy) @@ -40,7 +39,7 @@ class FastKryoSerializer(system: ExtendedActorSystem) extends Serializer { override def serialize(message: Any): Array[Byte] = { try { - kryoSerializer.toBinary(message) + kryoSerializer.toBinary(message.asInstanceOf[AnyRef]) } catch { case ex: java.lang.IllegalArgumentException => val clazz = message.getClass http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/serializer/GearpumpSerialization.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/serializer/GearpumpSerialization.scala b/core/src/main/scala/org/apache/gearpump/serializer/GearpumpSerialization.scala index 524089d..45a5481 100644 --- a/core/src/main/scala/org/apache/gearpump/serializer/GearpumpSerialization.scala +++ b/core/src/main/scala/org/apache/gearpump/serializer/GearpumpSerialization.scala @@ -18,10 +18,9 @@ package org.apache.gearpump.serializer +import com.esotericsoftware.kryo.{Kryo, Serializer => KryoSerializer} import com.typesafe.config.Config import org.slf4j.Logger - -import org.apache.gearpump.esotericsoftware.kryo.{Kryo, Serializer => KryoSerializer} import org.apache.gearpump.util.{Constants, LogUtil} class GearpumpSerialization(config: Config) { http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/util/FileUtils.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/util/FileUtils.scala b/core/src/main/scala/org/apache/gearpump/util/FileUtils.scala index e3df37b..283a64a 100644 --- a/core/src/main/scala/org/apache/gearpump/util/FileUtils.scala +++ b/core/src/main/scala/org/apache/gearpump/util/FileUtils.scala @@ -21,7 +21,7 @@ package org.apache.gearpump.util import java.io.{File, IOException} import java.nio.charset.Charset -import org.apache.gearpump.google.common.io.Files +import com.google.common.io.Files object FileUtils { private val UTF8 = Charset.forName("UTF-8") http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/util/Util.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/util/Util.scala b/core/src/main/scala/org/apache/gearpump/util/Util.scala index 0faa46a..8ee0e26 100644 --- a/core/src/main/scala/org/apache/gearpump/util/Util.scala +++ b/core/src/main/scala/org/apache/gearpump/util/Util.scala @@ -66,8 +66,14 @@ object Util { def startProcess(options: Array[String], classPath: Array[String], mainClass: String, arguments: Array[String]): RichProcess = { val java = System.getProperty("java.home") + "/bin/java" - val command = List(java) ++ options ++ - List("-cp", classPath.mkString(File.pathSeparator), mainClass) ++ arguments + + val command = List(java) ++ + // java.lang.VerifyError will be caused without "-noverify" + // TODO: investigate the cause and remove this + Array("-noverify") ++ + options ++ + List("-cp", classPath.mkString(File.pathSeparator), mainClass) ++ + arguments LOG.info(s"Starting executor process java $mainClass ${arguments.mkString(" ")} " + s"\n ${options.mkString(" ")}") val logger = new ProcessLogRedirector() http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/test/scala/org/apache/gearpump/jarstore/FileServerSpec.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/gearpump/jarstore/FileServerSpec.scala b/core/src/test/scala/org/apache/gearpump/jarstore/FileServerSpec.scala index c99a031..39b6261 100644 --- a/core/src/test/scala/org/apache/gearpump/jarstore/FileServerSpec.scala +++ b/core/src/test/scala/org/apache/gearpump/jarstore/FileServerSpec.scala @@ -22,9 +22,9 @@ import java.io.File import java.util.concurrent.TimeUnit import akka.actor.ActorSystem -import com.typesafe.config.{ConfigValueFactory, ConfigValue} +import com.google.common.io.Files +import com.typesafe.config.ConfigValueFactory import org.apache.gearpump.cluster.TestUtil -import org.apache.gearpump.google.common.io.Files import org.apache.gearpump.jarstore.local.LocalJarStore import org.apache.gearpump.util.{FileUtils, LogUtil} import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} @@ -39,7 +39,7 @@ class FileServerSpec extends WordSpecLike with Matchers with BeforeAndAfterAll { val host = "localhost" private val LOG = LogUtil.getLogger(getClass) - var system: ActorSystem = null + var system: ActorSystem = _ override def afterAll { if (null != system) { @@ -75,7 +75,6 @@ class FileServerSpec extends WordSpecLike with Matchers with BeforeAndAfterAll { "The file server" should { "serve the data previously stored" in { - val rootDir = Files.createTempDir() val localJarStore: JarStore = new LocalJarStore val conf = TestUtil.DEFAULT_CONFIG.withValue("gearpump.jarstore.rootpath", http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/test/scala/org/apache/gearpump/metrics/MetricsSpec.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/gearpump/metrics/MetricsSpec.scala b/core/src/test/scala/org/apache/gearpump/metrics/MetricsSpec.scala index 5881640..0855553 100644 --- a/core/src/test/scala/org/apache/gearpump/metrics/MetricsSpec.scala +++ b/core/src/test/scala/org/apache/gearpump/metrics/MetricsSpec.scala @@ -18,12 +18,13 @@ package org.apache.gearpump.metrics +import com.codahale.metrics.{Counter => CodaHaleCounter, Histogram => CodaHaleHistogram, Meter => CodaHaleMeter} + import org.mockito.Matchers._ import org.mockito.Mockito._ import org.scalatest.mock.MockitoSugar import org.scalatest.{FlatSpec, Matchers} -import org.apache.gearpump.codahale.metrics.{Counter => CodaHaleCounter, Histogram => CodaHaleHistogram, Meter => CodaHaleMeter} class MetricsSpec extends FlatSpec with Matchers with MockitoSugar { http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/test/scala/org/apache/gearpump/serializer/SerializerSpec.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/gearpump/serializer/SerializerSpec.scala b/core/src/test/scala/org/apache/gearpump/serializer/SerializerSpec.scala index f4bd114..0772a5e 100644 --- a/core/src/test/scala/org/apache/gearpump/serializer/SerializerSpec.scala +++ b/core/src/test/scala/org/apache/gearpump/serializer/SerializerSpec.scala @@ -18,20 +18,23 @@ package org.apache.gearpump.serializer -import scala.collection.JavaConverters._ -import scala.concurrent.Await -import scala.concurrent.duration.Duration - import akka.actor.{ActorSystem, ExtendedActorSystem} + +import com.esotericsoftware.kryo.io.{Input, Output} +import com.esotericsoftware.kryo.{Kryo, Serializer => KryoSerializer} import com.typesafe.config.{ConfigFactory, ConfigValueFactory} -import org.scalatest.mock.MockitoSugar -import org.scalatest.{FlatSpec, Matchers} import org.apache.gearpump.cluster.TestUtil -import org.apache.gearpump.esotericsoftware.kryo.io.{Input, Output} -import org.apache.gearpump.esotericsoftware.kryo.{Kryo, Serializer => KryoSerializer} import org.apache.gearpump.serializer.SerializerSpec._ +import org.scalatest.mock.MockitoSugar +import org.scalatest.{FlatSpec, Matchers} + +import scala.collection.JavaConverters._ +import scala.concurrent.Await +import scala.concurrent.duration.Duration + + class SerializerSpec extends FlatSpec with Matchers with MockitoSugar { val config = ConfigFactory.empty.withValue("gearpump.serializers", ConfigValueFactory.fromAnyRef(Map(classOf[ClassA].getName -> classOf[ClassASerializer].getName, @@ -70,7 +73,7 @@ object SerializerSpec { class ClassASerializer extends KryoSerializer[ClassA] { override def write(kryo: Kryo, output: Output, `object`: ClassA): Unit = { - output.writeString(classOf[ClassA].getName.toString) + output.writeString(classOf[ClassA].getName) } override def read(kryo: Kryo, input: Input, `type`: Class[ClassA]): ClassA = { http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/test/scala/org/apache/gearpump/util/FileUtilsSpec.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/gearpump/util/FileUtilsSpec.scala b/core/src/test/scala/org/apache/gearpump/util/FileUtilsSpec.scala index 66abc36..97b35ad 100644 --- a/core/src/test/scala/org/apache/gearpump/util/FileUtilsSpec.scala +++ b/core/src/test/scala/org/apache/gearpump/util/FileUtilsSpec.scala @@ -18,13 +18,13 @@ package org.apache.gearpump.util +import com.google.common.io.Files + import java.io.File import java.util import org.scalatest.FlatSpec -import org.apache.gearpump.google.common.io.Files - class FileUtilsSpec extends FlatSpec { val TXT = """ http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java ---------------------------------------------------------------------- diff --git a/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java b/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java index 4983dde..f52afc7 100644 --- a/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java +++ b/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java @@ -17,7 +17,7 @@ */ package org.apache.gearpump.cluster.utils; -import org.apache.gearpump.google.common.io.CharStreams; +import com.google.common.io.CharStreams; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/experiments/cgroup/src/main/scala/org/apache/gearpump/cluster/worker/CGroupProcessLauncher.scala ---------------------------------------------------------------------- diff --git a/experiments/cgroup/src/main/scala/org/apache/gearpump/cluster/worker/CGroupProcessLauncher.scala b/experiments/cgroup/src/main/scala/org/apache/gearpump/cluster/worker/CGroupProcessLauncher.scala index 74a4047..bf291cf 100644 --- a/experiments/cgroup/src/main/scala/org/apache/gearpump/cluster/worker/CGroupProcessLauncher.scala +++ b/experiments/cgroup/src/main/scala/org/apache/gearpump/cluster/worker/CGroupProcessLauncher.scala @@ -43,7 +43,7 @@ class CGroupProcessLauncher(val config: Config) extends ExecutorProcessLauncher } override def createProcess( - appId: Int, executorId: Int, resource: Resource, appConfig: Config, options: Array[String], + appId: Int, executorId: Int, resource: Resource, appConfig: Config, options: Array[String], classPath: Array[String], mainClass: String, arguments: Array[String]): RichProcess = { val cgroupCommand = if (executorId != APP_MASTER) { cgroupManager.map(_.startNewExecutor(appConfig, resource.slots, appId, http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/experiments/redis/src/main/scala/org/apache/gearpump/redis/RedisMessage.scala ---------------------------------------------------------------------- diff --git a/experiments/redis/src/main/scala/org/apache/gearpump/redis/RedisMessage.scala b/experiments/redis/src/main/scala/org/apache/gearpump/redis/RedisMessage.scala index 2988f5b..ea738d6 100644 --- a/experiments/redis/src/main/scala/org/apache/gearpump/redis/RedisMessage.scala +++ b/experiments/redis/src/main/scala/org/apache/gearpump/redis/RedisMessage.scala @@ -147,7 +147,7 @@ object RedisMessage { */ case class LPUSH(key: Array[Byte], value: Array[Byte]) { def this(key: String, value: String) = { - this(key, toBytes(value)) + this(toBytes(key), toBytes(value)) } } @@ -184,7 +184,7 @@ object RedisMessage { */ case class RPUSH(key: Array[Byte], value: Array[Byte]) { def this(key: String, value: String) = { - this(key, toBytes(value)) + this(toBytes(key), toBytes(value)) } } @@ -338,7 +338,7 @@ object RedisMessage { */ case class SADD(key: Array[Byte], members: Array[Byte]) { def this(key: String, members: String) = { - this(key, toBytes(members)) + this(toBytes(key), toBytes(members)) } } @@ -365,7 +365,7 @@ object RedisMessage { */ case class SREM(key: Array[Byte], member: Array[Byte]) { def this(key: String, member: String) = { - this(key, toBytes(member)) + this(toBytes(key), toBytes(member)) } } @@ -438,7 +438,7 @@ object RedisMessage { * @param increment */ case class INCRBYFLOAT(key: Array[Byte], increment: Double) { - def this(key: String, increment: Number) = { + def this(key: String, increment: Double) = { this(toBytes(key), increment) } } http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/Build.scala ---------------------------------------------------------------------- diff --git a/project/Build.scala b/project/Build.scala deleted file mode 100644 index 40b5743..0000000 --- a/project/Build.scala +++ /dev/null @@ -1,559 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import com.typesafe.sbt.SbtPgp.autoImport._ -import BuildExample.examples -import BuildIntegrationTest.integration_test -import BuildShaded._ -import de.johoop.jacoco4sbt.JacocoPlugin.jacoco -import org.scalajs.sbtplugin.ScalaJSPlugin.autoImport._ -import sbt.Keys._ -import sbt._ -import Pack.packProject -import org.scalajs.sbtplugin.cross.CrossProject -import sbtassembly.AssemblyPlugin.autoImport._ -import sbtunidoc.Plugin.UnidocKeys._ -import sbtunidoc.Plugin._ -import xerial.sbt.Sonatype._ - -object Build extends sbt.Build { - - val copySharedSourceFiles = TaskKey[Unit]("copied shared services source code") - - val akkaVersion = "2.4.3" - val apacheRepo = "https://repository.apache.org/"; - val hadoopVersion = "2.6.0" - val hbaseVersion = "1.0.0" - val commonsHttpVersion = "3.1" - val commonsLoggingVersion = "1.1.3" - val commonsLangVersion = "2.6" - val commonsIOVersion = "2.4" - val dataReplicationVersion = "0.7" - val upickleVersion = "0.3.4" - val junitVersion = "4.12" - val kafkaVersion = "0.8.2.1" - val stormVersion = "0.10.0" - val slf4jVersion = "1.7.7" - - val crossScalaVersionNumbers = Seq("2.11.8") - val scalaVersionNumber = crossScalaVersionNumbers.last - val sprayVersion = "1.3.2" - val sprayJsonVersion = "1.3.1" - val scalaTestVersion = "2.2.0" - val scalaCheckVersion = "1.11.3" - val mockitoVersion = "1.10.17" - val bijectionVersion = "0.8.0" - val scalazVersion = "7.1.1" - val algebirdVersion = "0.9.0" - val chillVersion = "0.6.0" - val distDirectory = "output" - val projectName = "gearpump" - - override def projects: Seq[Project] = (super.projects.toList ++ BuildExample.projects.toList - ++ Pack.projects.toList).toSeq - - val commonSettings = Seq(jacoco.settings: _*) ++ sonatypeSettings ++ - Seq( - resolvers ++= Seq( - "patriknw at bintray" at "http://dl.bintray.com/patriknw/maven";, - "apache-repo" at "https://repository.apache.org/content/repositories";, - "maven1-repo" at "http://repo1.maven.org/maven2";, - "maven2-repo" at "http://mvnrepository.com/artifact";, - "sonatype" at "https://oss.sonatype.org/content/repositories/releases";, - "bintray/non" at "http://dl.bintray.com/non/maven";, - "clockfly" at "http://dl.bintray.com/clockfly/maven";, - "clojars" at "http://clojars.org/repo"; - ) - // ,addCompilerPlugin("org.scalamacros" % "paradise" % "2.1.0-M5" cross CrossVersion.full) - ) ++ - Seq( - scalaVersion := scalaVersionNumber, - crossScalaVersions := crossScalaVersionNumbers, - organization := "org.apache.gearpump", - useGpg := false, - pgpSecretRing := file("./secring.asc"), - pgpPublicRing := file("./pubring.asc"), - scalacOptions ++= Seq("-Yclosure-elim", "-Yinline"), - publishMavenStyle := true, - - pgpPassphrase := Option(System.getenv().get("PASSPHRASE")).map(_.toArray), - credentials += Credentials( - "Sonatype Nexus Repository Manager", - "repository.apache.org", - System.getenv().get("SONATYPE_USERNAME"), - System.getenv().get("SONATYPE_PASSWORD")), - - pomIncludeRepository := { _ => false }, - - publishTo := { - if (isSnapshot.value) { - Some("snapshots" at apacheRepo + "content/repositories/snapshots") - } else { - Some("releases" at apacheRepo + "content/repositories/releases") - } - }, - - publishArtifact in Test := true, - - pomExtra := { - <url>https://github.com/apache/incubator-gearpump</url> - <licenses> - <license> - <name>Apache 2</name> - <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> - </license> - </licenses> - <scm> - <connection>scm:git://git.apache.org/incubator-gearpump.git</connection> - <developerConnection>scm:git:[email protected]:apache/incubator-gearpump</developerConnection> - <url>github.com/apache/incubator-gearpump</url> - </scm> - <developers> - <developer> - <id>gearpump</id> - <name>Gearpump Team</name> - <url>http://gearpump.incubator.apache.org/community.html#who-we-are</url> - </developer> - </developers> - } - ) - - val noPublish = Seq( - publish := {}, - publishLocal := {}, - publishArtifact := false, - publishArtifact in Test := false - ) - - val coreDependencies = Seq( - libraryDependencies ++= Seq( - "org.slf4j" % "slf4j-api" % slf4jVersion, - "org.slf4j" % "slf4j-log4j12" % slf4jVersion, - "commons-lang" % "commons-lang" % commonsLangVersion, - "com.google.code.findbugs" % "jsr305" % "1.3.9" % "compile", - - /** - * Overrides Netty version 3.10.3.Final used by Akka 2.4.2 to work-around netty hang issue - * (https://github.com/gearpump/gearpump/issues/2020) - * - * Akka 2.4.2 by default use Netty 3.10.3.Final, which has a serious issue which can hang - * the network. The same issue also happens in version range (3.10.0.Final, 3.10.5.Final) - * Netty 3.10.6.Final have this issue fixed, however, we find there is a 20% performance - * drop. So we decided to downgrade netty to 3.8.0.Final (Same version used in akka 2.3.12). - * - * @see https://github.com/gearpump/gearpump/pull/2017 for more discussions. - */ - "io.netty" % "netty" % "3.8.0.Final", - "com.typesafe.akka" %% "akka-remote" % akkaVersion - exclude("io.netty", "netty"), - - "com.typesafe.akka" %% "akka-cluster" % akkaVersion, - "com.typesafe.akka" %% "akka-cluster-tools" % akkaVersion, - "commons-logging" % "commons-logging" % commonsLoggingVersion, - "com.typesafe.akka" %% "akka-distributed-data-experimental" % akkaVersion, - "com.typesafe.akka" %% "akka-actor" % akkaVersion, - "com.typesafe.akka" %% "akka-agent" % akkaVersion, - "com.typesafe.akka" %% "akka-slf4j" % akkaVersion, - "com.typesafe.akka" %% "akka-kernel" % akkaVersion, - "com.typesafe.akka" %% "akka-http-experimental" % akkaVersion, - "com.typesafe.akka" %% "akka-http-spray-json-experimental" % akkaVersion, - "org.scala-lang" % "scala-reflect" % scalaVersionNumber, - "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.4", - "com.typesafe.akka" %% "akka-testkit" % akkaVersion % "test", - "org.scalatest" %% "scalatest" % scalaTestVersion % "test", - "org.scalacheck" %% "scalacheck" % scalaCheckVersion % "test", - "org.mockito" % "mockito-core" % mockitoVersion % "test", - "junit" % "junit" % junitVersion % "test" - ), - - unmanagedJars in Compile ++= Seq( - getShadedJarFile(shaded_metrics_graphite.id, version.value), - getShadedJarFile(shaded_guava.id, version.value), - getShadedJarFile(shaded_akka_kryo.id, version.value) - ) - ) - - lazy val javadocSettings = Seq( - addCompilerPlugin("com.typesafe.genjavadoc" %% "genjavadoc-plugin" % - "0.9" cross CrossVersion.full), - scalacOptions += s"-P:genjavadoc:out=${target.value}/java" - ) - - val myAssemblySettings = Seq( - test in assembly := {}, - assemblyOption in assembly ~= { - _.copy(includeScala = false) - }, - assemblyJarName in assembly := { - s"${name.value.split("-").last}-${scalaBinaryVersion.value}-${version.value}-assembly.jar" - } - ) - - val projectsWithDoc = inProjects( - core, - streaming, - external_kafka, - external_monoid, - external_serializer, - external_hbase, - external_hadoopfs) - - lazy val gearpumpUnidocSetting = scalaJavaUnidocSettings ++ Seq( - unidocProjectFilter in(ScalaUnidoc, unidoc) := projectsWithDoc, - unidocProjectFilter in(JavaUnidoc, unidoc) := projectsWithDoc, - - unidocAllSources in(ScalaUnidoc, unidoc) := { - ignoreUndocumentedPackages((unidocAllSources in(ScalaUnidoc, unidoc)).value) - }, - - // Skip class names containing $ and some internal packages in Javadocs - unidocAllSources in(JavaUnidoc, unidoc) := { - ignoreUndocumentedPackages((unidocAllSources in(JavaUnidoc, unidoc)).value) - } - ) - - private def ignoreUndocumentedPackages(packages: Seq[Seq[File]]): Seq[Seq[File]] = { - packages - .map(_.filterNot(_.getName.contains("$"))) - .map(_.filterNot(_.getCanonicalPath.contains("akka"))) - } - - private def addShadedDeps(deps: Seq[xml.Node], node: xml.Node): xml.Node = { - node match { - case elem: xml.Elem => - val child = if (elem.label == "dependencies") { - elem.child ++ deps - } else { - elem.child.map(addShadedDeps(deps, _)) - } - xml.Elem(elem.prefix, elem.label, elem.attributes, elem.scope, false, child: _*) - case _ => - node - } - } - - lazy val root = Project( - id = "gearpump", - base = file("."), - settings = commonSettings ++ noPublish ++ gearpumpUnidocSetting) - .aggregate(shaded, core, streaming, services, external_kafka, external_monoid, - external_serializer, examples, storm, yarn, external_hbase, gearpumpHadoop, packProject, - external_hadoopfs, integration_test).settings(Defaults.itSettings: _*) - .disablePlugins(sbtassembly.AssemblyPlugin) - - lazy val core = Project( - id = "gearpump-core", - base = file("core"), - settings = commonSettings ++ javadocSettings ++ coreDependencies ++ Seq( - pomPostProcess := { - (node: xml.Node) => addShadedDeps(List( - getShadedDepXML(organization.value, shaded_akka_kryo.id, version.value), - getShadedDepXML(organization.value, shaded_guava.id, version.value), - getShadedDepXML(organization.value, shaded_metrics_graphite.id, version.value)), node) - } - )).disablePlugins(sbtassembly.AssemblyPlugin) - - - lazy val cgroup = Project( - id = "gearpump-experimental-cgroup", - base = file("experiments/cgroup"), - settings = commonSettings ++ noPublish) - .dependsOn (core % "test->test; compile->compile") - .disablePlugins(sbtassembly.AssemblyPlugin) - - lazy val streaming = Project( - id = "gearpump-streaming", - base = file("streaming"), - settings = commonSettings ++ javadocSettings ++ Seq( - unmanagedJars in Compile ++= Seq( - getShadedJarFile(shaded_gs_collections.id, version.value) - ), - - pomPostProcess := { - (node: xml.Node) => addShadedDeps(List( - getShadedDepXML(organization.value, shaded_gs_collections.id, version.value)), node) - } - )) - .dependsOn(core % "test->test; compile->compile", shaded_gs_collections) - .disablePlugins(sbtassembly.AssemblyPlugin) - - lazy val external_kafka = Project( - id = "gearpump-external-kafka", - base = file("external/kafka"), - settings = commonSettings ++ javadocSettings ++ - Seq( - libraryDependencies ++= Seq( - "org.apache.kafka" %% "kafka" % kafkaVersion, - "com.twitter" %% "bijection-core" % bijectionVersion, - ("org.apache.kafka" %% "kafka" % kafkaVersion classifier ("test")) % "test" - ) - )) - .dependsOn (streaming % "test->test; provided") - .disablePlugins(sbtassembly.AssemblyPlugin) - - lazy val services_full = CrossProject("gearpump-services", file("services"), CrossType.Full). - settings( - publish := {}, - publishLocal := {} - ).disablePlugins(sbtassembly.AssemblyPlugin) - - val distDashboardDirectory = s"${distDirectory}/target/pack/dashboard/views/scalajs" - - // ScalaJs project need to be build seperately. - // sbt "project gearpump-servicesJS" compile - lazy val serviceJS: Project = services_full.js.settings(serviceJSSettings: _*) - - lazy val services: Project = services_full.jvm. - settings(serviceJvmSettings: _*) - .settings(compile in Compile <<= (compile in Compile)) - .dependsOn(streaming % "test->test;compile->compile") - - lazy val serviceJvmSettings = commonSettings ++ noPublish ++ Seq( - libraryDependencies ++= Seq( - "com.typesafe.akka" %% "akka-http-testkit" % akkaVersion % "test", - "org.scalatest" %% "scalatest" % scalaTestVersion % "test", - "com.lihaoyi" %% "upickle" % upickleVersion, - "com.softwaremill.akka-http-session" %% "core" % "0.2.5", - "com.typesafe.akka" %% "akka-http-spray-json-experimental" % akkaVersion, - "com.github.scribejava" % "scribejava-apis" % "2.4.0", - "com.ning" % "async-http-client" % "1.9.33", - "org.webjars" % "angularjs" % "1.4.9", - - // angular 1.5 breaks ui-select, but we need ng-touch 1.5 - "org.webjars.npm" % "angular-touch" % "1.5.0", - "org.webjars" % "angular-ui-router" % "0.2.15", - "org.webjars" % "bootstrap" % "3.3.6", - "org.webjars" % "d3js" % "3.5.6", - "org.webjars" % "momentjs" % "2.10.6", - "org.webjars" % "lodash" % "3.10.1", - "org.webjars" % "font-awesome" % "4.5.0", - "org.webjars" % "jquery" % "2.2.0", - "org.webjars" % "jquery-cookie" % "1.4.1", - "org.webjars.bower" % "angular-loading-bar" % "0.8.0", - "org.webjars.bower" % "angular-smart-table" % "2.1.6", - "org.webjars.bower" % "angular-motion" % "0.4.3", - "org.webjars.bower" % "bootstrap-additions" % "0.3.1", - "org.webjars.bower" % "angular-strap" % "2.3.5", - "org.webjars.npm" % "ui-select" % "0.14.2", - "org.webjars.bower" % "ng-file-upload" % "5.0.9", - "org.webjars.bower" % "vis" % "4.7.0", - "org.webjars.bower" % "clipboard.js" % "0.1.1", - "org.webjars.npm" % "dashing-deps" % "0.1.2", - "org.webjars.npm" % "dashing" % "0.4.8" - ).map(_.exclude("org.scalamacros", "quasiquotes_2.10")) - .map(_.exclude("org.scalamacros", "quasiquotes_2.10.3"))) - - lazy val serviceJSSettings = Seq( - scalaVersion := scalaVersionNumber, - crossScalaVersions := crossScalaVersionNumbers, - checksums := Seq(""), - requiresDOM := true, - libraryDependencies ++= Seq( - "com.lihaoyi" %%% "upickle" % upickleVersion, - "com.lihaoyi" %%% "utest" % "0.3.1" - ), - scalaJSStage in Global := FastOptStage, - testFrameworks += new TestFramework("utest.runner.Framework"), - requiresDOM := true, - persistLauncher in Compile := false, - persistLauncher in Test := false, - skip in packageJSDependencies := false, - scoverage.ScoverageSbtPlugin.ScoverageKeys.coverageExcludedPackages := - ".*gearpump\\.dashboard.*", - - copySharedSourceFiles := { - // scalastyle:off println - println(s"Copy shared source code to project services...") - // scalastyle:on println - }, - - artifactPath in fastOptJS in Compile := - new java.io.File(distDashboardDirectory, moduleName.value + "-fastopt.js"), - - fastOptJS in Compile <<= (fastOptJS in Compile).dependsOn(copySharedSourceFiles), - - relativeSourceMaps := true, - jsEnv in Test := new PhantomJS2Env(scalaJSPhantomJSClassLoader.value)) - - lazy val akkastream = Project( - id = "gearpump-experiments-akkastream", - base = file("experiments/akkastream"), - settings = commonSettings ++ noPublish ++ myAssemblySettings ++ - Seq( - libraryDependencies ++= Seq( - "org.json4s" %% "json4s-jackson" % "3.2.11" - ), - mainClass in(Compile, packageBin) := Some("akka.stream.gearpump.example.Test") - )) - .dependsOn(streaming % "test->test; provided") - - lazy val redis = Project( - id = "gearpump-experiments-redis", - base = file("experiments/redis"), - settings = commonSettings ++ noPublish ++ - Seq( - libraryDependencies ++= Seq( - "redis.clients" % "jedis" % "2.9.0" - ) - ) - ).dependsOn(streaming % "test->test; provided") - - lazy val storm = Project( - id = "gearpump-experiments-storm", - base = file("experiments/storm"), - settings = commonSettings ++ noPublish ++ - Seq( - libraryDependencies ++= Seq( - "commons-io" % "commons-io" % commonsIOVersion, - "org.apache.storm" % "storm-core" % stormVersion - exclude("clj-stacktrace", "clj-stacktrace") - exclude("ch.qos.logback", "logback-classic") - exclude("ch.qos.logback", "logback-core") - exclude("clj-time", "clj-time") - exclude("clout", "clout") - exclude("compojure", "compojure") - exclude("hiccup", "hiccup") - exclude("jline", "jline") - exclude("joda-time", "joda-time") - exclude("org.clojure", "core.incubator") - exclude("org.clojure", "math.numeric-tower") - exclude("org.clojure", "tools.logging") - exclude("org.clojure", "tools.cli") - exclude("org.clojure", "tools.macro") - exclude("org.mortbay.jetty", "jetty-util") - exclude("org.mortbay.jetty", "jetty") - exclude("org.ow2.asm", "asm") - exclude("org.slf4j", "log4j-over-slf4j") - exclude("org.apache.logging.log4j", "log4j-slf4j-impl") - exclude("ring", "ring-core") - exclude("ring", "ring-devel") - exclude("ring", "ring-jetty-adapter") - exclude("ring", "ring-servlet") - ) - )) - .dependsOn (streaming % "test->test; compile->compile") - .disablePlugins(sbtassembly.AssemblyPlugin) - - lazy val gearpumpHadoop = Project( - id = "gearpump-hadoop", - base = file("gearpump-hadoop"), - settings = commonSettings ++ noPublish ++ - Seq( - libraryDependencies ++= Seq( - "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion, - "org.apache.hadoop" % "hadoop-common" % hadoopVersion - ) - ) - ).dependsOn(core % "compile->compile").disablePlugins(sbtassembly.AssemblyPlugin) - - lazy val yarn = Project( - id = "gearpump-experiments-yarn", - base = file("experiments/yarn"), - settings = commonSettings ++ noPublish ++ - Seq( - libraryDependencies ++= Seq( - "org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion, - "org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion, - "org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion, - "commons-httpclient" % "commons-httpclient" % commonsHttpVersion, - "org.apache.hadoop" % "hadoop-mapreduce-client-core" % hadoopVersion, - "org.apache.hadoop" % "hadoop-yarn-server-resourcemanager" % hadoopVersion % "provided", - "org.apache.hadoop" % "hadoop-yarn-server-nodemanager" % hadoopVersion % "provided" - ) - )) - .dependsOn(services % "test->test;compile->compile", - core % "provided", gearpumpHadoop).disablePlugins(sbtassembly.AssemblyPlugin) - - lazy val external_hbase = Project( - id = "gearpump-external-hbase", - base = file("external/hbase"), - settings = commonSettings ++ javadocSettings ++ - Seq( - libraryDependencies ++= Seq( - "org.apache.hadoop" % "hadoop-common" % hadoopVersion % "provided", - "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion % "provided", - "org.apache.hadoop" % "hadoop-mapreduce-client-core" % hadoopVersion % "provided", - "org.codehaus.jackson" % "jackson-core-asl" % "1.9.13" % "provided", - "org.codehaus.jackson" % "jackson-mapper-asl" % "1.9.13" % "provided", - "org.apache.hbase" % "hbase-client" % hbaseVersion - exclude("com.github.stephenc.findbugs", "findbugs-annotations") - exclude("com.google.guava", "guava") - exclude("commons-codec", "commons-codec") - exclude("commons-io", "commons-io") - exclude("commons-lang", "commons-lang") - exclude("commons-logging", "commons-logging") - exclude("io.netty", "netty") - exclude("junit", "junit") - exclude("log4j", "log4j") - exclude("org.apache.zookeeper", "zookeeper") - exclude("org.codehaus.jackson", "jackson-mapper-asl"), - "org.apache.hbase" % "hbase-client" % hbaseVersion, - "org.apache.hbase" % "hbase-common" % hbaseVersion - exclude("com.github.stephenc.findbugs", "findbugs-annotations") - exclude("com.google.guava", "guava") - exclude("commons-codec", "commons-codec") - exclude("commons-collections", "commons-collections") - exclude("commons-io", "commons-io") - exclude("commons-lang", "commons-lang") - exclude("commons-logging", "commons-logging") - exclude("junit", "junit") - exclude("log4j", "log4j") - ) - )) - .dependsOn (streaming % "test->test; provided") - .disablePlugins(sbtassembly.AssemblyPlugin) - - lazy val external_monoid = Project( - id = "gearpump-external-monoid", - base = file("external/monoid"), - settings = commonSettings ++ javadocSettings ++ - Seq( - libraryDependencies ++= Seq( - "com.twitter" %% "algebird-core" % algebirdVersion - ) - )) - .dependsOn (streaming % "provided") - .disablePlugins(sbtassembly.AssemblyPlugin) - - lazy val external_serializer = Project( - id = "gearpump-external-serializer", - base = file("external/serializer"), - settings = commonSettings ++ javadocSettings ++ - Seq( - libraryDependencies ++= Seq( - "com.twitter" %% "chill-bijection" % chillVersion - exclude("com.esotericsoftware.kryo", "kyro") - exclude("com.esotericsoftware.minlog", "minlog") - ) - )) - .dependsOn (streaming % "provided") - .disablePlugins(sbtassembly.AssemblyPlugin) - - lazy val external_hadoopfs = Project( - id = "gearpump-external-hadoopfs", - base = file("external/hadoopfs"), - settings = commonSettings ++ javadocSettings ++ - Seq( - libraryDependencies ++= Seq( - "org.apache.hadoop" % "hadoop-common" % hadoopVersion % "provided", - "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion % "provided" - ) - )) - .dependsOn (streaming % "test->test; provided") - .disablePlugins(sbtassembly.AssemblyPlugin) -} http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildDashboard.scala ---------------------------------------------------------------------- diff --git a/project/BuildDashboard.scala b/project/BuildDashboard.scala new file mode 100644 index 0000000..c14b9d6 --- /dev/null +++ b/project/BuildDashboard.scala @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import BuildGearpump._ +import Dependencies._ +import org.scalajs.sbtplugin.cross.{CrossProject, CrossType} +import org.scalajs.sbtplugin.ScalaJSPlugin.autoImport._ +import sbt._ +import sbt.Keys._ + +object BuildDashboard extends sbt.Build { + + lazy val services: Project = services_full.jvm + .settings(serviceJvmSettings: _*) + .dependsOn(core % "provided", streaming % "test->test; provided") + + // ScalaJs project need to be build separately. + // sbt "project gearpump-servicesJS" compile + private lazy val serviceJS: Project = services_full.js.settings(serviceJSSettings: _*) + + private lazy val services_full = CrossProject("gearpump-services", file("services"), + CrossType.Full) + .settings( + publish := {}, + publishLocal := {} + ).disablePlugins(sbtassembly.AssemblyPlugin) + + private val copySharedSourceFiles = TaskKey[Unit]("copied shared services source code") + + private val distDashboardDirectory = s"${distDirectory}/target/pack/dashboard/views/scalajs" + + private lazy val serviceJvmSettings = commonSettings ++ noPublish ++ Seq( + libraryDependencies ++= Seq( + "com.typesafe.akka" %% "akka-http-testkit" % akkaVersion % "test", + "org.scalatest" %% "scalatest" % scalaTestVersion % "test", + "com.lihaoyi" %% "upickle" % upickleVersion, + "com.softwaremill.akka-http-session" %% "core" % "0.2.5", + "com.typesafe.akka" %% "akka-http-spray-json-experimental" % akkaVersion, + "com.github.scribejava" % "scribejava-apis" % "2.4.0", + "com.ning" % "async-http-client" % "1.9.33", + "org.webjars" % "angularjs" % "1.4.9", + + // angular 1.5 breaks ui-select, but we need ng-touch 1.5 + "org.webjars.npm" % "angular-touch" % "1.5.0", + "org.webjars" % "angular-ui-router" % "0.2.15", + "org.webjars" % "bootstrap" % "3.3.6", + "org.webjars" % "d3js" % "3.5.6", + "org.webjars" % "momentjs" % "2.10.6", + "org.webjars" % "lodash" % "3.10.1", + "org.webjars" % "font-awesome" % "4.5.0", + "org.webjars" % "jquery" % "2.2.0", + "org.webjars" % "jquery-cookie" % "1.4.1", + "org.webjars.bower" % "angular-loading-bar" % "0.8.0" + exclude("org.webjars.bower", "angular"), + "org.webjars.bower" % "angular-smart-table" % "2.1.6" + exclude("org.webjars.bower", "angular"), + "org.webjars.bower" % "angular-motion" % "0.4.3", + "org.webjars.bower" % "bootstrap-additions" % "0.3.1", + "org.webjars.bower" % "angular-strap" % "2.3.5" + exclude("org.webjars.bower", "angular"), + "org.webjars.npm" % "ui-select" % "0.14.2", + "org.webjars.bower" % "ng-file-upload" % "5.0.9", + "org.webjars.bower" % "vis" % "4.7.0", + "org.webjars.bower" % "clipboard.js" % "0.1.1", + "org.webjars.npm" % "dashing-deps" % "0.1.2", + "org.webjars.npm" % "dashing" % "0.4.8" + ).map(_.exclude("org.scalamacros", "quasiquotes_2.10")) + .map(_.exclude("org.scalamacros", "quasiquotes_2.10.3"))) + + private lazy val serviceJSSettings = Seq( + scalaVersion := scalaVersionNumber, + crossScalaVersions := crossScalaVersionNumbers, + checksums := Seq(""), + requiresDOM := true, + libraryDependencies ++= Seq( + "com.lihaoyi" %%% "upickle" % upickleVersion, + "com.lihaoyi" %%% "utest" % "0.3.1" + ), + scalaJSStage in Global := FastOptStage, + testFrameworks += new TestFramework("utest.runner.Framework"), + requiresDOM := true, + persistLauncher in Compile := false, + persistLauncher in Test := false, + skip in packageJSDependencies := false, + scoverage.ScoverageSbtPlugin.ScoverageKeys.coverageExcludedPackages := + ".*gearpump\\.dashboard.*", + + copySharedSourceFiles := { + // scalastyle:off println + println(s"Copy shared source code to project services...") + // scalastyle:on println + }, + + artifactPath in fastOptJS in Compile := + new java.io.File(distDashboardDirectory, moduleName.value + "-fastopt.js"), + + fastOptJS in Compile := { + (fastOptJS in Compile).dependsOn(copySharedSourceFiles).value + }, + + relativeSourceMaps := true, + jsEnv in Test := new PhantomJS2Env(scalaJSPhantomJSClassLoader.value) + ) +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildExample.scala ---------------------------------------------------------------------- diff --git a/project/BuildExample.scala b/project/BuildExample.scala deleted file mode 100644 index fadc1ec..0000000 --- a/project/BuildExample.scala +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import sbt.Keys._ -import sbt._ -import Build._ -import sbtassembly.AssemblyPlugin.autoImport._ - -object BuildExample extends sbt.Build { - - lazy val examples = Project( - id = "gearpump-examples", - base = file("examples"), - settings = commonSettings ++ noPublish - ).aggregate(wordcount, wordcountJava, complexdag, sol, fsio, examples_kafka, - distributedshell, stockcrawler, transport, examples_state, pagerank, distributeservice). - disablePlugins(sbtassembly.AssemblyPlugin) - - lazy val wordcountJava = Project( - id = "gearpump-examples-wordcountjava", - base = file("examples/streaming/wordcount-java"), - settings = commonSettings ++ noPublish ++ myAssemblySettings ++ - Seq( - mainClass in(Compile, packageBin) := - Some("org.apache.gearpump.streaming.examples.wordcountjava.WordCount"), - - target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" / - CrossVersion.binaryScalaVersion(scalaVersion.value) - ) - ) dependsOn(streaming % "test->test; provided") - - lazy val wordcount = Project( - id = "gearpump-examples-wordcount", - base = file("examples/streaming/wordcount"), - settings = commonSettings ++ noPublish ++ myAssemblySettings ++ - Seq( - mainClass in(Compile, packageBin) := - Some("org.apache.gearpump.streaming.examples.wordcount.WordCount"), - - target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" / - CrossVersion.binaryScalaVersion(scalaVersion.value) - ) - ) dependsOn(streaming % "test->test; provided") - - lazy val sol = Project( - id = "gearpump-examples-sol", - base = file("examples/streaming/sol"), - settings = commonSettings ++ noPublish ++ myAssemblySettings ++ - Seq( - mainClass in(Compile, packageBin) := Some("org.apache.gearpump.streaming.examples.sol.SOL"), - - target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" / - CrossVersion.binaryScalaVersion(scalaVersion.value) - - ) - ) dependsOn (streaming % "test->test; provided") - - lazy val complexdag = Project( - id = "gearpump-examples-complexdag", - base = file("examples/streaming/complexdag"), - settings = commonSettings ++ noPublish ++ myAssemblySettings ++ - Seq( - mainClass in(Compile, packageBin) := - Some("org.apache.gearpump.streaming.examples.complexdag.Dag"), - - target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" / - CrossVersion.binaryScalaVersion(scalaVersion.value) - ) - ) dependsOn (streaming % "test->test; provided") - - lazy val transport = Project( - id = "gearpump-examples-transport", - base = file("examples/streaming/transport"), - settings = commonSettings ++ noPublish ++ myAssemblySettings ++ - Seq( - libraryDependencies ++= Seq( - "io.spray" %% "spray-can" % sprayVersion, - "io.spray" %% "spray-routing-shapeless2" % sprayVersion, - "io.spray" %% "spray-json" % sprayJsonVersion, - "com.lihaoyi" %% "upickle" % upickleVersion - ), - mainClass in(Compile, packageBin) := - Some("org.apache.gearpump.streaming.examples.transport.Transport"), - - target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" / - CrossVersion.binaryScalaVersion(scalaVersion.value) - ) - ) dependsOn (streaming % "test->test; provided") - - lazy val distributedshell = Project( - id = "gearpump-examples-distributedshell", - base = file("examples/distributedshell"), - settings = commonSettings ++ noPublish ++ myAssemblySettings ++ - Seq( - mainClass in(Compile, packageBin) := - Some("org.apache.gearpump.examples.distributedshell.DistributedShell"), - - target in assembly := baseDirectory.value.getParentFile / "target" / - CrossVersion.binaryScalaVersion(scalaVersion.value) - ) - ) dependsOn (core % "test->test; provided") - - lazy val distributeservice = Project( - id = "gearpump-examples-distributeservice", - base = file("examples/distributeservice"), - settings = commonSettings ++ noPublish ++ myAssemblySettings ++ - Seq( - libraryDependencies ++= Seq( - "commons-httpclient" % "commons-httpclient" % commonsHttpVersion, - "commons-lang" % "commons-lang" % commonsLangVersion, - "commons-io" % "commons-io" % commonsIOVersion, - "io.spray" %% "spray-can" % sprayVersion, - "io.spray" %% "spray-routing-shapeless2" % sprayVersion - ), - mainClass in(Compile, packageBin) := - Some("org.apache.gearpump.experiments.distributeservice.DistributeService"), - - target in assembly := baseDirectory.value.getParentFile / "target" / - CrossVersion.binaryScalaVersion(scalaVersion.value) - ) - ) dependsOn (core % "test->test; provided") - - lazy val fsio = Project( - id = "gearpump-examples-fsio", - base = file("examples/streaming/fsio"), - settings = commonSettings ++ noPublish ++ myAssemblySettings ++ - Seq( - libraryDependencies ++= Seq( - "org.apache.hadoop" % "hadoop-common" % hadoopVersion - exclude("org.mortbay.jetty", "jetty-util") - exclude("org.mortbay.jetty", "jetty") - exclude("org.fusesource.leveldbjni", "leveldbjni-all") - exclude("tomcat", "jasper-runtime") - exclude("commons-beanutils", "commons-beanutils-core") - exclude("commons-beanutils", "commons-beanutils") - exclude("asm", "asm") - exclude("org.ow2.asm", "asm") - ), - mainClass in(Compile, packageBin) := - Some("org.apache.gearpump.streaming.examples.fsio.SequenceFileIO"), - - target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" / - CrossVersion.binaryScalaVersion(scalaVersion.value) - ) - ) dependsOn (streaming % "test->test; provided") - - lazy val examples_kafka = Project( - id = "gearpump-examples-kafka", - base = file("examples/streaming/kafka"), - settings = commonSettings ++ noPublish ++ myAssemblySettings ++ - Seq( - mainClass in(Compile, packageBin) := - Some("org.apache.gearpump.streaming.examples.kafka.wordcount.KafkaWordCount"), - - target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" / - CrossVersion.binaryScalaVersion(scalaVersion.value) - ) - ) dependsOn(streaming % "test->test; provided", external_kafka) - - lazy val stockcrawler = Project( - id = "gearpump-examples-stockcrawler", - base = file("examples/streaming/stockcrawler"), - settings = commonSettings ++ noPublish ++ myAssemblySettings ++ - Seq( - libraryDependencies ++= Seq( - "io.spray" %% "spray-can" % sprayVersion, - "io.spray" %% "spray-routing-shapeless2" % sprayVersion, - "com.lihaoyi" %% "upickle" % upickleVersion, - "commons-httpclient" % "commons-httpclient" % commonsHttpVersion, - "net.sourceforge.htmlcleaner" % "htmlcleaner" % "2.2", - "joda-time" % "joda-time" % "2.7", - "org.joda" % "joda-convert" % "1.2", - "io.spray" %% "spray-json" % sprayJsonVersion - ), - mainClass in(Compile, packageBin) := - Some("org.apache.gearpump.streaming.examples.stock.main.Stock"), - - target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" / - CrossVersion.binaryScalaVersion(scalaVersion.value) - ) - ) dependsOn(streaming % "test->test; provided", external_kafka % "test->test") - - lazy val examples_state = Project( - id = "gearpump-examples-state", - base = file("examples/streaming/state"), - settings = commonSettings ++ noPublish ++ myAssemblySettings ++ - Seq( - libraryDependencies ++= Seq( - "org.apache.hadoop" % "hadoop-common" % hadoopVersion - exclude("org.mortbay.jetty", "jetty-util") - exclude("org.mortbay.jetty", "jetty") - exclude("org.fusesource.leveldbjni", "leveldbjni-all") - exclude("tomcat", "jasper-runtime") - exclude("commons-beanutils", "commons-beanutils-core") - exclude("commons-beanutils", "commons-beanutils") - exclude("asm", "asm") - exclude("org.ow2.asm", "asm"), - "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion - ), - mainClass in(Compile, packageBin) := - Some("org.apache.gearpump.streaming.examples.state.MessageCountApp"), - - target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" / - CrossVersion.binaryScalaVersion(scalaVersion.value) - ) - ).dependsOn(streaming % "test->test; provided", - external_hadoopfs, external_monoid, external_serializer, external_kafka) - - lazy val pagerank = Project( - id = "gearpump-examples-pagerank", - base = file("examples/pagerank"), - settings = commonSettings ++ noPublish ++ myAssemblySettings ++ - Seq( - mainClass in(Compile, packageBin) := - Some("org.apache.gearpump.experiments.pagerank.example.PageRankExample"), - - target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" / - CrossVersion.binaryScalaVersion(scalaVersion.value) - ) - ) dependsOn (streaming % "test->test; provided") -} http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildExamples.scala ---------------------------------------------------------------------- diff --git a/project/BuildExamples.scala b/project/BuildExamples.scala new file mode 100644 index 0000000..bac84a8 --- /dev/null +++ b/project/BuildExamples.scala @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import sbt.Keys._ +import sbt._ +import BuildGearpump._ +import BuildExternals._ +import Dependencies._ +import sbtassembly.AssemblyPlugin.autoImport._ + +object BuildExamples extends sbt.Build { + + lazy val examples: Seq[ProjectReference] = Seq( + complexdag, + distributedshell, + distributeservice, + examples_kafka, + examples_state, + fsio, + pagerank, + sol, + stockcrawler, + transport, + wordcount, + wordcountJava + ) + + lazy val wordcountJava = Project( + id = "gearpump-examples-wordcountjava", + base = file("examples/streaming/wordcount-java"), + settings = exampleSettings("org.apache.gearpump.streaming.examples.wordcountjava.WordCount") + ).dependsOn(core % "provided", streaming % "test->test; provided") + + lazy val wordcount = Project( + id = "gearpump-examples-wordcount", + base = file("examples/streaming/wordcount"), + settings = exampleSettings("org.apache.gearpump.streaming.examples.wordcount.WordCount") + ).dependsOn(core % "provided", streaming % "test->test; provided") + + lazy val sol = Project( + id = "gearpump-examples-sol", + base = file("examples/streaming/sol"), + settings = exampleSettings("org.apache.gearpump.streaming.examples.sol.SOL") + ).dependsOn(core % "provided", streaming % "test->test; provided") + + lazy val complexdag = Project( + id = "gearpump-examples-complexdag", + base = file("examples/streaming/complexdag"), + settings = exampleSettings("org.apache.gearpump.streaming.examples.complexdag.Dag") + ).dependsOn(core % "provided", streaming % "test->test; provided") + + lazy val transport = Project( + id = "gearpump-examples-transport", + base = file("examples/streaming/transport"), + settings = exampleSettings("org.apache.gearpump.streaming.examples.transport.Transport") ++ + Seq( + libraryDependencies ++= Seq( + "io.spray" %% "spray-can" % sprayVersion, + "io.spray" %% "spray-routing-shapeless2" % sprayVersion, + "io.spray" %% "spray-json" % sprayJsonVersion, + "com.lihaoyi" %% "upickle" % upickleVersion + ) + ) + ).dependsOn(core % "provided", streaming % "test->test; provided") + + lazy val distributedshell = Project( + id = "gearpump-examples-distributedshell", + base = file("examples/distributedshell"), + settings = exampleSettings("org.apache.gearpump.examples.distributedshell.DistributedShell") + ).dependsOn(core % "test->test; provided") + + lazy val distributeservice = Project( + id = "gearpump-examples-distributeservice", + base = file("examples/distributeservice"), + settings = + exampleSettings("org.apache.gearpump.experiments.distributeservice.DistributeService") ++ + Seq( + libraryDependencies ++= Seq( + "commons-httpclient" % "commons-httpclient" % commonsHttpVersion, + "commons-lang" % "commons-lang" % commonsLangVersion, + "commons-io" % "commons-io" % commonsIOVersion, + "io.spray" %% "spray-can" % sprayVersion, + "io.spray" %% "spray-routing-shapeless2" % sprayVersion + ) + ) + ).dependsOn(core % "test->test; provided") + + lazy val fsio = Project( + id = "gearpump-examples-fsio", + base = file("examples/streaming/fsio"), + settings = exampleSettings("org.apache.gearpump.streaming.examples.fsio.SequenceFileIO") ++ + Seq( + libraryDependencies ++= Seq( + "org.apache.hadoop" % "hadoop-common" % hadoopVersion + exclude("org.mortbay.jetty", "jetty-util") + exclude("org.mortbay.jetty", "jetty") + exclude("org.fusesource.leveldbjni", "leveldbjni-all") + exclude("tomcat", "jasper-runtime") + exclude("commons-beanutils", "commons-beanutils-core") + exclude("commons-beanutils", "commons-beanutils") + exclude("asm", "asm") + exclude("org.ow2.asm", "asm") + ) + ) + ).dependsOn(core % "provided", streaming % "test->test; provided") + + lazy val examples_kafka = Project( + id = "gearpump-examples-kafka", + base = file("examples/streaming/kafka"), + settings = + exampleSettings("org.apache.gearpump.streaming.examples.kafka.wordcount.KafkaWordCount") + ).dependsOn(core % "provided", streaming % "test->test; provided", external_kafka) + + lazy val stockcrawler = Project( + id = "gearpump-examples-stockcrawler", + base = file("examples/streaming/stockcrawler"), + settings = exampleSettings("org.apache.gearpump.streaming.examples.stock.main.Stock") ++ + Seq( + libraryDependencies ++= Seq( + "io.spray" %% "spray-can" % sprayVersion, + "io.spray" %% "spray-routing-shapeless2" % sprayVersion, + "com.lihaoyi" %% "upickle" % upickleVersion, + "commons-httpclient" % "commons-httpclient" % commonsHttpVersion, + "net.sourceforge.htmlcleaner" % "htmlcleaner" % "2.2", + "joda-time" % "joda-time" % "2.7", + "org.joda" % "joda-convert" % "1.2", + "io.spray" %% "spray-json" % sprayJsonVersion + ) + ) + ).dependsOn(core % "provided", streaming % "test->test; provided", external_kafka % "test->test") + + lazy val examples_state = Project( + id = "gearpump-examples-state", + base = file("examples/streaming/state"), + settings = exampleSettings("org.apache.gearpump.streaming.examples.state.MessageCountApp") ++ + Seq( + libraryDependencies ++= Seq( + "org.apache.hadoop" % "hadoop-common" % hadoopVersion + exclude("org.mortbay.jetty", "jetty-util") + exclude("org.mortbay.jetty", "jetty") + exclude("org.fusesource.leveldbjni", "leveldbjni-all") + exclude("tomcat", "jasper-runtime") + exclude("commons-beanutils", "commons-beanutils-core") + exclude("commons-beanutils", "commons-beanutils") + exclude("asm", "asm") + exclude("org.ow2.asm", "asm"), + "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion + ) + ) + ).dependsOn(core % "provided", streaming % "test->test; provided", + external_hadoopfs, external_monoid, external_serializer, external_kafka) + + lazy val pagerank = Project( + id = "gearpump-examples-pagerank", + base = file("examples/pagerank"), + settings = exampleSettings("org.apache.gearpump.experiments.pagerank.example.PageRankExample") + ).dependsOn(core % "provided", streaming % "test->test; provided") + + private def exampleSettings(className: String): Seq[Def.Setting[_]] = + commonSettings ++ noPublish ++ myAssemblySettings ++ Seq( + mainClass in(Compile, packageBin) := + Some(className), + target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" / + CrossVersion.binaryScalaVersion(scalaVersion.value) + ) +} http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildExperiments.scala ---------------------------------------------------------------------- diff --git a/project/BuildExperiments.scala b/project/BuildExperiments.scala new file mode 100644 index 0000000..92a26c6 --- /dev/null +++ b/project/BuildExperiments.scala @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import BuildGearpump._ +import BuildDashboard.services +import Dependencies._ +import sbt._ +import sbt.Keys._ + +object BuildExperiments extends sbt.Build { + + lazy val experiments: Seq[ProjectReference] = Seq( + // akkastream, + cgroup, + redis, + storm, + yarn + ) + + lazy val yarn = Project( + id = "gearpump-experiments-yarn", + base = file("experiments/yarn"), + settings = commonSettings ++ noPublish ++ + Seq( + libraryDependencies ++= Seq( + "org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion, + "org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion, + "org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion, + "commons-httpclient" % "commons-httpclient" % commonsHttpVersion, + "org.apache.hadoop" % "hadoop-mapreduce-client-core" % hadoopVersion, + "org.apache.hadoop" % "hadoop-yarn-server-resourcemanager" % hadoopVersion % "provided", + "org.apache.hadoop" % "hadoop-yarn-server-nodemanager" % hadoopVersion % "provided" + ) + )) + .dependsOn(services % "test->test;compile->compile", + core % "provided", gearpumpHadoop).disablePlugins(sbtassembly.AssemblyPlugin) + + lazy val akkastream = Project( + id = "gearpump-experiments-akkastream", + base = file("experiments/akkastream"), + settings = commonSettings ++ noPublish ++ myAssemblySettings ++ + Seq( + libraryDependencies ++= Seq( + "org.json4s" %% "json4s-jackson" % "3.2.11" + ), + mainClass in(Compile, packageBin) := Some("akka.stream.gearpump.example.Test") + )) + .dependsOn(core % "provided", streaming % "test->test; provided") + + lazy val redis = Project( + id = "gearpump-experiments-redis", + base = file("experiments/redis"), + settings = commonSettings ++ noPublish ++ + Seq( + libraryDependencies ++= Seq( + "redis.clients" % "jedis" % jedisVersion + ) + ) + ).dependsOn(core % "provided", streaming % "test->test; provided") + + lazy val storm = Project( + id = "gearpump-experiments-storm", + base = file("experiments/storm"), + settings = commonSettings ++ noPublish ++ + Seq( + libraryDependencies ++= Seq( + "commons-io" % "commons-io" % commonsIOVersion, + "org.apache.storm" % "storm-core" % stormVersion + exclude("clj-stacktrace", "clj-stacktrace") + exclude("ch.qos.logback", "logback-classic") + exclude("ch.qos.logback", "logback-core") + exclude("clj-time", "clj-time") + exclude("clout", "clout") + exclude("compojure", "compojure") + exclude("hiccup", "hiccup") + exclude("jline", "jline") + exclude("joda-time", "joda-time") + exclude("org.clojure", "core.incubator") + exclude("org.clojure", "math.numeric-tower") + exclude("org.clojure", "tools.logging") + exclude("org.clojure", "tools.cli") + exclude("org.clojure", "tools.macro") + exclude("org.mortbay.jetty", "jetty-util") + exclude("org.mortbay.jetty", "jetty") + exclude("org.ow2.asm", "asm") + exclude("org.slf4j", "log4j-over-slf4j") + exclude("org.apache.logging.log4j", "log4j-slf4j-impl") + exclude("ring", "ring-core") + exclude("ring", "ring-devel") + exclude("ring", "ring-jetty-adapter") + exclude("ring", "ring-servlet") + ) + )) + .dependsOn (core % "provided", streaming % "test->test; provided") + .disablePlugins(sbtassembly.AssemblyPlugin) + + lazy val cgroup = Project( + id = "gearpump-experimental-cgroup", + base = file("experiments/cgroup"), + settings = commonSettings ++ noPublish) + .dependsOn (core % "test->test; compile->compile") + .disablePlugins(sbtassembly.AssemblyPlugin) +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildExternals.scala ---------------------------------------------------------------------- diff --git a/project/BuildExternals.scala b/project/BuildExternals.scala new file mode 100644 index 0000000..48715cb --- /dev/null +++ b/project/BuildExternals.scala @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import BuildGearpump._ +import Dependencies._ +import Docs._ +import sbt._ +import sbt.Keys._ + +object BuildExternals extends sbt.Build { + + lazy val externals: Seq[ProjectReference] = Seq( + external_hbase, + external_kafka, + external_monoid, + external_hadoopfs + ) + + lazy val external_kafka = Project( + id = "gearpump-external-kafka", + base = file("external/kafka"), + settings = commonSettings ++ javadocSettings ++ + Seq( + libraryDependencies ++= Seq( + "org.apache.kafka" %% "kafka" % kafkaVersion, + "com.twitter" %% "bijection-core" % bijectionVersion, + ("org.apache.kafka" %% "kafka" % kafkaVersion classifier ("test")) % "test" + ) + )) + .dependsOn(core % "provided", streaming % "test->test; provided") + .disablePlugins(sbtassembly.AssemblyPlugin) + + lazy val external_hbase = Project( + id = "gearpump-external-hbase", + base = file("external/hbase"), + settings = commonSettings ++ javadocSettings ++ + Seq( + libraryDependencies ++= Seq( + "org.apache.hadoop" % "hadoop-common" % hadoopVersion % "provided", + "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion % "provided", + "org.apache.hadoop" % "hadoop-mapreduce-client-core" % hadoopVersion % "provided", + "org.codehaus.jackson" % "jackson-core-asl" % "1.9.13" % "provided", + "org.codehaus.jackson" % "jackson-mapper-asl" % "1.9.13" % "provided", + "org.apache.hbase" % "hbase-client" % hbaseVersion + exclude("com.github.stephenc.findbugs", "findbugs-annotations") + exclude("com.google.guava", "guava") + exclude("commons-codec", "commons-codec") + exclude("commons-io", "commons-io") + exclude("commons-lang", "commons-lang") + exclude("commons-logging", "commons-logging") + exclude("io.netty", "netty") + exclude("junit", "junit") + exclude("log4j", "log4j") + exclude("org.apache.zookeeper", "zookeeper") + exclude("org.codehaus.jackson", "jackson-mapper-asl"), + "org.apache.hbase" % "hbase-common" % hbaseVersion + exclude("com.github.stephenc.findbugs", "findbugs-annotations") + exclude("com.google.guava", "guava") + exclude("commons-codec", "commons-codec") + exclude("commons-collections", "commons-collections") + exclude("commons-io", "commons-io") + exclude("commons-lang", "commons-lang") + exclude("commons-logging", "commons-logging") + exclude("junit", "junit") + exclude("log4j", "log4j") + ) + )) + .dependsOn (core % "provided", streaming % "test->test; provided") + .disablePlugins(sbtassembly.AssemblyPlugin) + + lazy val external_monoid = Project( + id = "gearpump-external-monoid", + base = file("external/monoid"), + settings = commonSettings ++ javadocSettings ++ + Seq( + libraryDependencies ++= Seq( + "com.twitter" %% "algebird-core" % algebirdVersion + ) + )) + .dependsOn (core % "provided", streaming % "provided") + .disablePlugins(sbtassembly.AssemblyPlugin) + + lazy val external_serializer = Project( + id = "gearpump-external-serializer", + base = file("external/serializer"), + settings = commonSettings ++ javadocSettings ++ + Seq( + libraryDependencies ++= Seq( + "com.twitter" %% "chill-bijection" % chillVersion + exclude("com.esotericsoftware.kryo", "kyro") + exclude("com.esotericsoftware.minlog", "minlog") + ) + )) + .dependsOn (core % "provided", streaming % "provided") + .disablePlugins(sbtassembly.AssemblyPlugin) + + lazy val external_hadoopfs = Project( + id = "gearpump-external-hadoopfs", + base = file("external/hadoopfs"), + settings = commonSettings ++ javadocSettings ++ + Seq( + libraryDependencies ++= Seq( + "org.apache.hadoop" % "hadoop-common" % hadoopVersion % "provided", + "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion % "provided" + ) + )) + .dependsOn(core % "provided", streaming % "test->test; provided") + .disablePlugins(sbtassembly.AssemblyPlugin) +} \ No newline at end of file
