Repository: incubator-gearpump Updated Branches: refs/heads/master cbf59fb77 -> 0b56de399
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildGearpump.scala ---------------------------------------------------------------------- diff --git a/project/BuildGearpump.scala b/project/BuildGearpump.scala new file mode 100644 index 0000000..1a52c40 --- /dev/null +++ b/project/BuildGearpump.scala @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.typesafe.sbt.SbtPgp.autoImport._ +import BuildExamples.examples +import BuildExperiments.experiments +import BuildExternals.externals +import BuildIntegrationTests.integrationTests +import BuildDashboard.services +import Dependencies._ +import Docs._ +import de.johoop.jacoco4sbt.JacocoPlugin.jacoco +import sbt.Keys._ +import sbt._ +import Pack.packProject +import sbtassembly.AssemblyPlugin.autoImport._ + +import xerial.sbt.Sonatype._ + +object BuildGearpump extends sbt.Build { + + val apacheRepo = "https://repository.apache.org/"; + val distDirectory = "output" + val projectName = "gearpump" + + val commonSettings = Seq(jacoco.settings: _*) ++ sonatypeSettings ++ + Seq( + resolvers ++= Seq( + "apache-repo" at "https://repository.apache.org/content/repositories";, + "maven1-repo" at "http://repo1.maven.org/maven2";, + "maven2-repo" at "http://mvnrepository.com/artifact";, + "sonatype" at "https://oss.sonatype.org/content/repositories/releases";, + "clojars" at "http://clojars.org/repo"; + ) + ) ++ + Seq( + scalaVersion := scalaVersionNumber, + crossScalaVersions := crossScalaVersionNumbers, + organization := "org.apache.gearpump", + useGpg := false, + pgpSecretRing := file("./secring.asc"), + pgpPublicRing := file("./pubring.asc"), + scalacOptions ++= Seq("-Yclosure-elim", "-Yinline"), + publishMavenStyle := true, + + pgpPassphrase := Option(System.getenv().get("PASSPHRASE")).map(_.toArray), + credentials += Credentials( + "Sonatype Nexus Repository Manager", + "repository.apache.org", + System.getenv().get("SONATYPE_USERNAME"), + System.getenv().get("SONATYPE_PASSWORD")), + + pomIncludeRepository := { _ => false }, + + publishTo := { + if (isSnapshot.value) { + Some("snapshots" at apacheRepo + "content/repositories/snapshots") + } else { + Some("releases" at apacheRepo + "content/repositories/releases") + } + }, + + publishArtifact in Test := true, + + pomExtra := { + <url>https://github.com/apache/incubator-gearpump</url> + <licenses> + <license> + <name>Apache 2</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + </license> + </licenses> + <scm> + <connection>scm:git://git.apache.org/incubator-gearpump.git</connection> + <developerConnection>scm:git:[email protected]:apache/incubator-gearpump</developerConnection> + <url>github.com/apache/incubator-gearpump</url> + </scm> + <developers> + <developer> + <id>gearpump</id> + <name>Gearpump Team</name> + <url>http://gearpump.incubator.apache.org/community.html#who-we-are</url> + </developer> + </developers> + }, + + pomPostProcess := { + (node: xml.Node) => changeShadedDeps( + Set( + "org.scoverage", + "org.scala-lang" + ), + List.empty[xml.Node], + node) + } + ) + + val noPublish = Seq( + publish := {}, + publishLocal := {}, + publishArtifact := false, + publishArtifact in Test := false + ) + + lazy val myAssemblySettings = Seq( + test in assembly := {}, + assemblyOption in assembly ~= { + _.copy(includeScala = false) + }, + assemblyJarName in assembly := { + s"${name.value}_${scalaBinaryVersion.value}-${version.value}.jar" + }, + assemblyShadeRules in assembly := Seq( + ShadeRule.rename("com.romix.**" -> "org.apache.gearpump.@0").inAll, + ShadeRule.rename("com.esotericsoftware.**" -> + "org.apache.gearpump.@0").inAll, + ShadeRule.rename("org.objenesis.**" -> "org.apache.gearpump.@0").inAll, + ShadeRule.rename("com.google.common.**" -> "org.apache.gearpump.@0").inAll, + ShadeRule.rename("com.google.thirdparty.**" -> "org.apache.gearpump.@0").inAll, + ShadeRule.rename("com.codahale.metrics.**" -> + "org.apache.gearpump.@0").inAll, + ShadeRule.rename("com.gs.collections.**" -> + "org.apache.gearpump.gs.collections.@0").inAll + ), + target in assembly := baseDirectory.value / "target" / scalaBinaryVersion.value + ) + + lazy val aggregated: Seq[ProjectReference] = Seq[ProjectReference]( + core, + streaming, + services, + gearpumpHadoop, + packProject + ) ++ examples ++ experiments ++ externals ++ integrationTests + + lazy val root = Project( + id = "gearpump", + base = file("."), + settings = commonSettings ++ noPublish ++ gearpumpUnidocSetting, + aggregate = aggregated) + .settings(Defaults.itSettings: _*) + .disablePlugins(sbtassembly.AssemblyPlugin) + + lazy val core = Project( + id = "gearpump-core", + base = file("core"), + settings = commonSettings ++ myAssemblySettings ++ javadocSettings ++ coreDependencies ++ + addArtifact(Artifact("gearpump-core"), sbtassembly.AssemblyKeys.assembly) ++ Seq( + + assemblyOption in assembly ~= { + _.copy(includeScala = true) + }, + + pomPostProcess := { + (node: xml.Node) => changeShadedDeps( + Set( + "com.github.romix.akka", + "com.google.guava", + "com.codahale.metrics", + "org.scoverage" + ), List.empty[xml.Node], node) + } + )) + + lazy val streaming = Project( + id = "gearpump-streaming", + base = file("streaming"), + settings = commonSettings ++ myAssemblySettings ++ javadocSettings ++ + addArtifact(Artifact("gearpump-streaming"), sbtassembly.AssemblyKeys.assembly) ++ + Seq( + assemblyMergeStrategy in assembly := { + case "geardefault.conf" => + MergeStrategy.last + case x => + val oldStrategy = (assemblyMergeStrategy in assembly).value + oldStrategy(x) + }, + + libraryDependencies ++= Seq( + "com.goldmansachs" % "gs-collections" % gsCollectionsVersion + ), + + pomPostProcess := { + (node: xml.Node) => changeShadedDeps( + Set( + "com.goldmansachs", + "org.scala-lang", + "org.scoverage" + ), + List( + getShadedDepXML(organization.value, s"${core.id}_${scalaBinaryVersion.value}", + version.value, "provided")), + node) + } + ) + ).dependsOn(core % "test->test;provided") + + lazy val gearpumpHadoop = Project( + id = "gearpump-hadoop", + base = file("gearpump-hadoop"), + settings = commonSettings ++ noPublish ++ + Seq( + libraryDependencies ++= Seq( + "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion, + "org.apache.hadoop" % "hadoop-common" % hadoopVersion + ) + ) + ).dependsOn(core % "compile->compile").disablePlugins(sbtassembly.AssemblyPlugin) + + private def changeShadedDeps(toExclude: Set[String], toInclude: List[xml.Node], + node: xml.Node): xml.Node = { + node match { + case elem: xml.Elem => + val child = + if (elem.label == "dependencies") { + elem.child.filterNot { dep => + dep.child.find(_.label == "groupId").exists(gid => toExclude.contains(gid.text)) + } ++ toInclude + } else { + elem.child.map(changeShadedDeps(toExclude, toInclude, _)) + } + xml.Elem(elem.prefix, elem.label, elem.attributes, elem.scope, false, child: _*) + case _ => + node + } + } + + private def getShadedDepXML(groupId: String, artifactId: String, + version: String, scope: String): scala.xml.Node = { + <dependency> + <groupId>{groupId}</groupId> + <artifactId>{artifactId}</artifactId> + <version>{version}</version> + <scope>{scope}</scope> + </dependency> + } +} http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildIntegrationTest.scala ---------------------------------------------------------------------- diff --git a/project/BuildIntegrationTest.scala b/project/BuildIntegrationTest.scala deleted file mode 100644 index 6eed7a2..0000000 --- a/project/BuildIntegrationTest.scala +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import sbt.Keys._ -import sbt._ -import Build._ -import sbtassembly.AssemblyPlugin.autoImport._ - -object BuildIntegrationTest extends sbt.Build { - - val jsonSimpleVersion = "1.1" - val storm09Version = "0.9.6" - - lazy val integration_test = Project( - id = "gearpump-integrationtest", - base = file("integrationtest"), - settings = commonSettings ++ noPublish - ).aggregate(it_core, it_storm09, it_storm010). - disablePlugins(sbtassembly.AssemblyPlugin) - - val itTestFilter: String => Boolean = { name => name endsWith "Suite" } - lazy val it_core = Project( - id = "gearpump-integrationtest-core", - base = file("integrationtest/core"), - settings = commonSettings ++ noPublish ++ Seq( - testOptions in IntegrationTest += Tests.Filter(itTestFilter), - libraryDependencies ++= Seq( - "com.lihaoyi" %% "upickle" % upickleVersion, - "org.scalatest" %% "scalatest" % scalaTestVersion % "it", - "org.pegdown" % "pegdown" % "1.4.2" % "it", - "org.parboiled" % "parboiled-core" % "1.1.7" % "it", - "org.parboiled" % "parboiled-java" % "1.1.7" % "it", - "org.mortbay.jetty" % "jetty-util" % "6.1.26" % "it", - "org.ow2.asm" % "asm-all" % "5.0.3" % "it" - ) - ) - ).configs(IntegrationTest).settings(Defaults.itSettings: _*) - .dependsOn( - streaming % "test->test; provided", - services % "test->test; provided", - external_kafka, - storm, - external_serializer - ).disablePlugins(sbtassembly.AssemblyPlugin) - - // Integration test for Storm 0.9.x - lazy val it_storm09 = Project( - id = "gearpump-integrationtest-storm09", - base = file("integrationtest/storm09"), - settings = commonSettings ++ noPublish ++ myAssemblySettings ++ - Seq( - libraryDependencies ++= Seq( - "org.apache.storm" % "storm-kafka" % storm09Version, - "org.apache.storm" % "storm-starter" % storm09Version, - "com.googlecode.json-simple" % "json-simple" % jsonSimpleVersion, - "org.apache.kafka" %% "kafka" % kafkaVersion - ), - target in assembly := baseDirectory.value.getParentFile / "target" / - CrossVersion.binaryScalaVersion(scalaVersion.value) - ) - ) dependsOn (storm % "provided") - - // Integration test for Storm 0.10.x - lazy val it_storm010 = Project( - id = "gearpump-integrationtest-storm010", - base = file("integrationtest/storm010"), - settings = commonSettings ++ noPublish ++ myAssemblySettings ++ - Seq( - libraryDependencies ++= Seq( - "org.apache.storm" % "storm-kafka" % stormVersion, - "org.apache.storm" % "storm-starter" % stormVersion, - "org.apache.kafka" %% "kafka" % kafkaVersion - ), - target in assembly := baseDirectory.value.getParentFile / "target" / - CrossVersion.binaryScalaVersion(scalaVersion.value) - ) - ) dependsOn (storm % "provided") -} http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildIntegrationTests.scala ---------------------------------------------------------------------- diff --git a/project/BuildIntegrationTests.scala b/project/BuildIntegrationTests.scala new file mode 100644 index 0000000..161d906 --- /dev/null +++ b/project/BuildIntegrationTests.scala @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import sbt.Keys._ +import sbt._ +import BuildGearpump._ +import BuildExperiments.storm +import BuildDashboard.services +import BuildExternals.{external_kafka, external_serializer} +import Dependencies._ +import sbtassembly.AssemblyPlugin.autoImport._ + +object BuildIntegrationTests extends sbt.Build { + + lazy val integrationTests: Seq[ProjectReference] = Seq( + it_core, + it_storm09, + it_storm010 + ) + + lazy val it_core = Project( + id = "gearpump-integrationtest-core", + base = file("integrationtest/core"), + settings = commonSettings ++ noPublish ++ Seq( + testOptions in IntegrationTest += Tests.Filter(_.endsWith("Suite")), + libraryDependencies ++= Seq( + "com.lihaoyi" %% "upickle" % upickleVersion, + "org.scalatest" %% "scalatest" % scalaTestVersion % "it", + "org.pegdown" % "pegdown" % "1.4.2" % "it", + "org.parboiled" % "parboiled-core" % "1.1.7" % "it", + "org.parboiled" % "parboiled-java" % "1.1.7" % "it", + "org.mortbay.jetty" % "jetty-util" % "6.1.26" % "it", + "org.ow2.asm" % "asm-all" % "5.0.3" % "it" + ) + ) + ).configs(IntegrationTest).settings(Defaults.itSettings: _*) + .dependsOn( + core % "provided", + streaming % "test->test; provided", + services % "test->test; provided", + external_kafka, + storm, + external_serializer + ).disablePlugins(sbtassembly.AssemblyPlugin) + + // Integration test for Storm 0.9.x + lazy val it_storm09 = Project( + id = "gearpump-integrationtest-storm09", + base = file("integrationtest/storm09"), + settings = commonSettings ++ noPublish ++ myAssemblySettings ++ + Seq( + libraryDependencies ++= Seq( + "org.apache.storm" % "storm-kafka" % storm09Version, + "org.apache.storm" % "storm-starter" % storm09Version, + "com.googlecode.json-simple" % "json-simple" % jsonSimpleVersion, + "org.apache.kafka" %% "kafka" % kafkaVersion + ), + target in assembly := baseDirectory.value.getParentFile / "target" / + CrossVersion.binaryScalaVersion(scalaVersion.value) + ) + ).dependsOn(core % "provided", storm % "provided") + + // Integration test for Storm 0.10.x + lazy val it_storm010 = Project( + id = "gearpump-integrationtest-storm010", + base = file("integrationtest/storm010"), + settings = commonSettings ++ noPublish ++ myAssemblySettings ++ + Seq( + libraryDependencies ++= Seq( + "org.apache.storm" % "storm-kafka" % stormVersion, + "org.apache.storm" % "storm-starter" % stormVersion, + "org.apache.kafka" %% "kafka" % kafkaVersion + ), + target in assembly := baseDirectory.value.getParentFile / "target" / + CrossVersion.binaryScalaVersion(scalaVersion.value) + ) + ).dependsOn(core % "provided", storm % "provided") +} http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildShaded.scala ---------------------------------------------------------------------- diff --git a/project/BuildShaded.scala b/project/BuildShaded.scala deleted file mode 100644 index a43587c..0000000 --- a/project/BuildShaded.scala +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import sbt.Keys._ -import sbt._ -import sbtassembly.AssemblyPlugin.autoImport._ - -object BuildShaded extends sbt.Build { - - val guavaVersion = "16.0.1" - val codahaleVersion = "3.0.2" - val kryoVersion = "0.4.1" - val gsCollectionsVersion = "6.2.0" - private val scalaVersionMajor = "2.11" - - val shadeAssemblySettings = Build.commonSettings ++ Seq( - scalaVersion := Build.scalaVersionNumber, - test in assembly := {}, - assemblyOption in assembly ~= { - _.copy(includeScala = false) - }, - assemblyJarName in assembly := { - s"${name.value}_$scalaVersionMajor-${version.value}.jar" - }, - target in assembly := baseDirectory.value.getParentFile / "target" / scalaVersionMajor - ) - - val shaded = Project( - id = "gearpump-shaded", - base = file("shaded") - ).aggregate(shaded_akka_kryo, shaded_gs_collections, shaded_guava, shaded_metrics_graphite) - .disablePlugins(sbtassembly.AssemblyPlugin) - - lazy val shaded_akka_kryo = Project( - id = "gearpump-shaded-akka-kryo", - base = file("shaded/akka-kryo"), - settings = shadeAssemblySettings ++ addArtifact(Artifact("gearpump-shaded-akka-kryo"), - sbtassembly.AssemblyKeys.assembly) ++ - Seq( - assemblyShadeRules in assembly := Seq( - ShadeRule.zap("com.google.protobuf.**").inAll, - ShadeRule.zap("com.typesafe.config.**").inAll, - ShadeRule.zap("akka.**").inAll, - ShadeRule.zap("org.jboss.netty.**").inAll, - ShadeRule.zap("net.jpountz.lz4.**").inAll, - ShadeRule.zap("org.uncommons.maths.**").inAll, - ShadeRule.rename("com.romix.**" -> "org.apache.gearpump.romix.@1").inAll, - ShadeRule.rename("com.esotericsoftware.**" -> - "org.apache.gearpump.esotericsoftware.@1").inAll, - ShadeRule.rename("org.objenesis.**" -> "org.apache.gearpump.objenesis.@1").inAll - ) - ) ++ - Seq( - libraryDependencies ++= Seq( - "com.github.romix.akka" %% "akka-kryo-serialization" % kryoVersion - ) - ) - ) - - lazy val shaded_gs_collections = Project( - id = "gearpump-shaded-gs-collections", - base = file("shaded/gs-collections"), - settings = shadeAssemblySettings ++ addArtifact(Artifact("gearpump-shaded-gs-collections"), - sbtassembly.AssemblyKeys.assembly) ++ - Seq( - assemblyShadeRules in assembly := Seq( - ShadeRule.rename("com.gs.collections.**" -> - "org.apache.gearpump.gs.collections.@1").inAll - ) - ) ++ - Seq( - libraryDependencies ++= Seq( - "com.goldmansachs" % "gs-collections" % gsCollectionsVersion - ) - ) - ) - - lazy val shaded_guava = Project( - id = "gearpump-shaded-guava", - base = file("shaded/guava"), - settings = shadeAssemblySettings ++ addArtifact(Artifact("gearpump-shaded-guava"), - sbtassembly.AssemblyKeys.assembly) ++ - Seq( - assemblyShadeRules in assembly := Seq( - ShadeRule.rename("com.google.**" -> "org.apache.gearpump.google.@1").inAll - ) - ) ++ - Seq( - libraryDependencies ++= Seq( - "com.google.guava" % "guava" % guavaVersion - ) - ) - ) - - lazy val shaded_metrics_graphite = Project( - id = "gearpump-shaded-metrics-graphite", - base = file("shaded/metrics-graphite"), - settings = shadeAssemblySettings ++ addArtifact(Artifact("gearpump-shaded-metrics-graphite"), - sbtassembly.AssemblyKeys.assembly) ++ - Seq( - assemblyShadeRules in assembly := Seq( - ShadeRule.rename("com.codahale.metrics.**" -> - "org.apache.gearpump.codahale.metrics.@1").inAll - ) - ) ++ - Seq( - libraryDependencies ++= Seq( - "com.codahale.metrics" % "metrics-graphite" % codahaleVersion, - "com.codahale.metrics" % "metrics-jvm" % codahaleVersion - ) - ) - ) - - def getShadedJarFile(name: String, gearpumpVersion: String): File = { - shaded.base / "target" / scalaVersionMajor / - s"${name}_$scalaVersionMajor-$gearpumpVersion.jar" - } - - def getShadedDepXML(groupId: String, artifactId: String, version: String): scala.xml.Node = { - <dependency> - <groupId>{groupId}</groupId> - <artifactId>{artifactId}</artifactId> - <version>{version}</version> - </dependency> - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/Dependencies.scala ---------------------------------------------------------------------- diff --git a/project/Dependencies.scala b/project/Dependencies.scala new file mode 100644 index 0000000..6949497 --- /dev/null +++ b/project/Dependencies.scala @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import sbt._ +import Keys._ + +object Dependencies { + + val crossScalaVersionNumbers = Seq("2.11.8") + val scalaVersionNumber = crossScalaVersionNumbers.last + val akkaVersion = "2.4.3" + val hadoopVersion = "2.6.0" + val hbaseVersion = "1.0.0" + val commonsHttpVersion = "3.1" + val commonsLoggingVersion = "1.1.3" + val commonsLangVersion = "2.6" + val commonsIOVersion = "2.4" + val dataReplicationVersion = "0.7" + val upickleVersion = "0.3.4" + val junitVersion = "4.12" + val kafkaVersion = "0.8.2.1" + val jsonSimpleVersion = "1.1" + val storm09Version = "0.9.6" + val stormVersion = "0.10.0" + val slf4jVersion = "1.7.16" + val guavaVersion = "16.0.1" + val codahaleVersion = "3.0.2" + val kryoVersion = "0.4.1" + val gsCollectionsVersion = "6.2.0" + val sprayVersion = "1.3.2" + val sprayJsonVersion = "1.3.1" + val scalaTestVersion = "2.2.0" + val scalaCheckVersion = "1.11.3" + val mockitoVersion = "1.10.17" + val bijectionVersion = "0.8.0" + val scalazVersion = "7.1.1" + val algebirdVersion = "0.9.0" + val chillVersion = "0.6.0" + val jedisVersion = "2.9.0" + + val coreDependencies = Seq( + libraryDependencies ++= Seq( + "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.slf4j" % "slf4j-log4j12" % slf4jVersion, + "commons-lang" % "commons-lang" % commonsLangVersion, + + /** + * Overrides Netty version 3.10.3.Final used by Akka 2.4.2 to work-around netty hang issue + * (https://github.com/gearpump/gearpump/issues/2020) + * + * Akka 2.4.2 by default use Netty 3.10.3.Final, which has a serious issue which can hang + * the network. The same issue also happens in version range (3.10.0.Final, 3.10.5.Final) + * Netty 3.10.6.Final have this issue fixed, however, we find there is a 20% performance + * drop. So we decided to downgrade netty to 3.8.0.Final (Same version used in akka 2.3.12). + * + * @see https://github.com/gearpump/gearpump/pull/2017 for more discussions. + */ + "io.netty" % "netty" % "3.8.0.Final", + "com.typesafe.akka" %% "akka-remote" % akkaVersion + exclude("io.netty", "netty"), + + "com.typesafe.akka" %% "akka-cluster" % akkaVersion, + "com.typesafe.akka" %% "akka-cluster-tools" % akkaVersion, + "commons-logging" % "commons-logging" % commonsLoggingVersion, + "com.typesafe.akka" %% "akka-distributed-data-experimental" % akkaVersion, + "com.typesafe.akka" %% "akka-actor" % akkaVersion, + "com.typesafe.akka" %% "akka-agent" % akkaVersion, + "com.typesafe.akka" %% "akka-slf4j" % akkaVersion, + "com.typesafe.akka" %% "akka-kernel" % akkaVersion, + "com.typesafe.akka" %% "akka-http-experimental" % akkaVersion, + "com.typesafe.akka" %% "akka-http-spray-json-experimental" % akkaVersion, + "org.scala-lang" % "scala-reflect" % scalaVersionNumber, + "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.4", + "com.github.romix.akka" %% "akka-kryo-serialization" % kryoVersion, + "com.google.guava" % "guava" % guavaVersion, + "com.codahale.metrics" % "metrics-graphite" % codahaleVersion + exclude("org.slf4j", "slf4j-api"), + "com.codahale.metrics" % "metrics-jvm" % codahaleVersion + exclude("org.slf4j", "slf4j-api"), + "com.typesafe.akka" %% "akka-testkit" % akkaVersion % "test", + "org.scalatest" %% "scalatest" % scalaTestVersion % "test", + "org.scalacheck" %% "scalacheck" % scalaCheckVersion % "test", + "org.mockito" % "mockito-core" % mockitoVersion % "test", + "junit" % "junit" % junitVersion % "test" + ) + ) +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/Docs.scala ---------------------------------------------------------------------- diff --git a/project/Docs.scala b/project/Docs.scala new file mode 100644 index 0000000..f8d433e --- /dev/null +++ b/project/Docs.scala @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import BuildGearpump.{core, streaming} +import BuildExternals.externals +import sbt.Keys._ +import sbt._ +import sbtunidoc.Plugin.UnidocKeys._ +import sbtunidoc.Plugin._ + +object Docs extends sbt.Build { + lazy val javadocSettings = Seq( + addCompilerPlugin( + "com.typesafe.genjavadoc" %% "genjavadoc-plugin" % "0.9" cross CrossVersion.full), + scalacOptions += s"-P:genjavadoc:out=${target.value}/java" + ) + + lazy val gearpumpUnidocSetting = scalaJavaUnidocSettings ++ Seq( + unidocProjectFilter in(ScalaUnidoc, unidoc) := projectsWithDoc, + unidocProjectFilter in(JavaUnidoc, unidoc) := projectsWithDoc, + + unidocAllSources in(ScalaUnidoc, unidoc) := { + ignoreUndocumentedPackages((unidocAllSources in(ScalaUnidoc, unidoc)).value) + }, + + // Skip class names containing $ and some internal packages in Javadocs + unidocAllSources in(JavaUnidoc, unidoc) := { + ignoreUndocumentedPackages((unidocAllSources in(JavaUnidoc, unidoc)).value) + } + ) + + private lazy val projectsWithDoc = { + val projects: Seq[ProjectReference] = Seq[ProjectReference]( + core, + streaming + ) ++ externals + + inProjects(projects: _*) + } + + private def ignoreUndocumentedPackages(packages: Seq[Seq[File]]): Seq[Seq[File]] = { + packages + .map(_.filterNot(_.getName.contains("$"))) + .map(_.filterNot(_.getCanonicalPath.contains("akka"))) + } +} http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/Pack.scala ---------------------------------------------------------------------- diff --git a/project/Pack.scala b/project/Pack.scala index 54b1d43..2eb0ca7 100644 --- a/project/Pack.scala +++ b/project/Pack.scala @@ -18,7 +18,9 @@ import sbt.Keys._ import sbt._ -import Build._ +import BuildGearpump._ +import BuildDashboard.services +import BuildExperiments.{cgroup, storm, yarn} import xerial.sbt.Pack._ object Pack extends sbt.Build { @@ -69,9 +71,14 @@ object Pack extends sbt.Build { "storm" -> "org.apache.gearpump.experiments.storm.StormRunner" ), packJvmOpts := Map( - "gear" -> Seq("-Djava.net.preferIPv4Stack=true", "-Dgearpump.home=${PROG_HOME}"), + "gear" -> Seq( + "-noverify", + "-Djava.net.preferIPv4Stack=true", + "-Dgearpump.home=${PROG_HOME}"), + "local" -> Seq( "-server", + "-noverify", "-Djava.net.preferIPv4Stack=true", "-DlogFilename=local", "-Dgearpump.home=${PROG_HOME}", @@ -79,6 +86,7 @@ object Pack extends sbt.Build { "master" -> Seq( "-server", + "-noverify", "-Djava.net.preferIPv4Stack=true", "-DlogFilename=master", "-Dgearpump.home=${PROG_HOME}", @@ -86,6 +94,7 @@ object Pack extends sbt.Build { "worker" -> Seq( "-server", + "-noverify", "-Djava.net.preferIPv4Stack=true", "-DlogFilename=worker", "-Dgearpump.home=${PROG_HOME}", @@ -93,23 +102,25 @@ object Pack extends sbt.Build { "services" -> Seq( "-server", + "-noverify", "-Djava.net.preferIPv4Stack=true", "-Dgearpump.home=${PROG_HOME}", "-Djava.rmi.server.hostname=localhost"), "yarnclient" -> Seq( "-server", + "-noverify", "-Djava.net.preferIPv4Stack=true", "-Dgearpump.home=${PROG_HOME}", "-Djava.rmi.server.hostname=localhost"), "storm" -> Seq( "-server", + "-noverify", "-Djava.net.preferIPv4Stack=true", "-Dgearpump.home=${PROG_HOME}") ), packLibDir := Map( - "lib" -> new ProjectsToPack(core.id, cgroup.id, streaming.id), "lib/yarn" -> new ProjectsToPack(gearpumpHadoop.id, yarn.id). exclude(services.id, core.id), "lib/services" -> new ProjectsToPack(services.id).exclude(core.id), @@ -120,7 +131,9 @@ object Pack extends sbt.Build { packResourceDir += (baseDirectory.value / ".." / "bin" -> "bin"), packResourceDir += (baseDirectory.value / ".." / "conf" -> "conf"), packResourceDir += (baseDirectory.value / ".." / "yarnconf" -> "conf/yarnconf"), - packResourceDir += (baseDirectory.value / ".." / "shaded" / "target" / + packResourceDir += (baseDirectory.value / ".." / "core" / "target" / + CrossVersion.binaryScalaVersion(scalaVersion.value) -> "lib"), + packResourceDir += (baseDirectory.value / ".." / "streaming" / "target" / CrossVersion.binaryScalaVersion(scalaVersion.value) -> "lib"), packResourceDir += (baseDirectory.value / ".." / "services" / "dashboard" -> "dashboard"), packResourceDir += (baseDirectory.value / ".." / "examples" / "target" / http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/build.properties ---------------------------------------------------------------------- diff --git a/project/build.properties b/project/build.properties index c4df008..8df00d1 100644 --- a/project/build.properties +++ b/project/build.properties @@ -16,4 +16,4 @@ # limitations under the License. # -sbt.version=0.13.11 +sbt.version=0.13.13 http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/plugins.sbt ---------------------------------------------------------------------- diff --git a/project/plugins.sbt b/project/plugins.sbt index 5a9740e..5f88b5e 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -25,7 +25,7 @@ addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.8") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3") -addSbtPlugin("io.gearpump.sbt" % "sbt-pack" % "0.7.6") +addSbtPlugin("io.gearpump.sbt" % "sbt-pack" % "0.7.7") addSbtPlugin("de.johoop" % "jacoco4sbt" % "2.1.6") http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/streaming/src/main/scala/org/apache/gearpump/streaming/appmaster/ClockService.scala ---------------------------------------------------------------------- diff --git a/streaming/src/main/scala/org/apache/gearpump/streaming/appmaster/ClockService.scala b/streaming/src/main/scala/org/apache/gearpump/streaming/appmaster/ClockService.scala index 68db354..2085953 100644 --- a/streaming/src/main/scala/org/apache/gearpump/streaming/appmaster/ClockService.scala +++ b/streaming/src/main/scala/org/apache/gearpump/streaming/appmaster/ClockService.scala @@ -23,7 +23,7 @@ import java.util.Date import java.util.concurrent.TimeUnit import akka.actor.{Actor, Cancellable, Stash} -import org.apache.gearpump.google.common.primitives.Longs +import com.google.common.primitives.Longs import org.apache.gearpump.TimeStamp import org.apache.gearpump.cluster.ClientToMaster.GetStallingTasks import org.apache.gearpump.streaming.AppMasterToMaster.StallingTasks http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/streaming/src/main/scala/org/apache/gearpump/streaming/dsl/window/impl/WindowRunner.scala ---------------------------------------------------------------------- diff --git a/streaming/src/main/scala/org/apache/gearpump/streaming/dsl/window/impl/WindowRunner.scala b/streaming/src/main/scala/org/apache/gearpump/streaming/dsl/window/impl/WindowRunner.scala index 9af5e61..640d090 100644 --- a/streaming/src/main/scala/org/apache/gearpump/streaming/dsl/window/impl/WindowRunner.scala +++ b/streaming/src/main/scala/org/apache/gearpump/streaming/dsl/window/impl/WindowRunner.scala @@ -22,10 +22,10 @@ import java.time.Instant import akka.actor.ActorSystem import org.apache.gearpump.Message import org.apache.gearpump.cluster.UserConfig -import org.apache.gearpump.gs.collections.api.block.procedure.Procedure -import org.apache.gearpump.gs.collections.impl.list.mutable.FastList -import org.apache.gearpump.gs.collections.impl.map.mutable.UnifiedMap -import org.apache.gearpump.gs.collections.impl.map.sorted.mutable.TreeSortedMap +import com.gs.collections.api.block.procedure.Procedure +import com.gs.collections.impl.list.mutable.FastList +import com.gs.collections.impl.map.mutable.UnifiedMap +import com.gs.collections.impl.map.sorted.mutable.TreeSortedMap import org.apache.gearpump.streaming.Constants._ import org.apache.gearpump.streaming.dsl.plan.functions.{EmitFunction, SingleInputFunction} import org.apache.gearpump.streaming.dsl.window.api.Discarding http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/streaming/src/main/scala/org/apache/gearpump/streaming/metrics/ProcessorAggregator.scala ---------------------------------------------------------------------- diff --git a/streaming/src/main/scala/org/apache/gearpump/streaming/metrics/ProcessorAggregator.scala b/streaming/src/main/scala/org/apache/gearpump/streaming/metrics/ProcessorAggregator.scala index d045def..8f8b7ab 100644 --- a/streaming/src/main/scala/org/apache/gearpump/streaming/metrics/ProcessorAggregator.scala +++ b/streaming/src/main/scala/org/apache/gearpump/streaming/metrics/ProcessorAggregator.scala @@ -20,12 +20,11 @@ package org.apache.gearpump.streaming.metrics import java.util +import com.google.common.collect.Iterators import com.typesafe.config.Config - import org.apache.gearpump.TimeStamp import org.apache.gearpump.cluster.ClientToMaster.ReadOption import org.apache.gearpump.cluster.MasterToClient.HistoryMetricsItem -import org.apache.gearpump.google.common.collect.Iterators import org.apache.gearpump.metrics.Metrics.{Histogram, Meter} import org.apache.gearpump.metrics.MetricsAggregator import org.apache.gearpump.streaming.metrics.ProcessorAggregator._ http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/streaming/src/main/scala/org/apache/gearpump/streaming/task/SerializerResolver.scala ---------------------------------------------------------------------- diff --git a/streaming/src/main/scala/org/apache/gearpump/streaming/task/SerializerResolver.scala b/streaming/src/main/scala/org/apache/gearpump/streaming/task/SerializerResolver.scala index 902c663..3b32163 100644 --- a/streaming/src/main/scala/org/apache/gearpump/streaming/task/SerializerResolver.scala +++ b/streaming/src/main/scala/org/apache/gearpump/streaming/task/SerializerResolver.scala @@ -18,7 +18,7 @@ package org.apache.gearpump.streaming.task -import org.apache.gearpump.esotericsoftware.kryo.util.{IntMap, ObjectMap} +import com.esotericsoftware.kryo.util.{IntMap, ObjectMap} import org.apache.gearpump.streaming.task.SerializerResolver.Registration private[task] class SerializerResolver { http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/streaming/src/main/scala/org/apache/gearpump/streaming/task/Subscription.scala ---------------------------------------------------------------------- diff --git a/streaming/src/main/scala/org/apache/gearpump/streaming/task/Subscription.scala b/streaming/src/main/scala/org/apache/gearpump/streaming/task/Subscription.scala index 16f9e93..4193fbf 100644 --- a/streaming/src/main/scala/org/apache/gearpump/streaming/task/Subscription.scala +++ b/streaming/src/main/scala/org/apache/gearpump/streaming/task/Subscription.scala @@ -20,7 +20,7 @@ package org.apache.gearpump.streaming.task import org.slf4j.Logger -import org.apache.gearpump.google.common.primitives.Shorts +import com.google.common.primitives.Shorts import org.apache.gearpump.streaming.partitioner.{MulticastPartitioner, Partitioner, UnicastPartitioner} import org.apache.gearpump.streaming.AppMasterToExecutor.MsgLostException import org.apache.gearpump.streaming.LifeTime http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/streaming/src/main/scala/org/apache/gearpump/streaming/task/TaskActor.scala ---------------------------------------------------------------------- diff --git a/streaming/src/main/scala/org/apache/gearpump/streaming/task/TaskActor.scala b/streaming/src/main/scala/org/apache/gearpump/streaming/task/TaskActor.scala index 9a52cc6..92f6672 100644 --- a/streaming/src/main/scala/org/apache/gearpump/streaming/task/TaskActor.scala +++ b/streaming/src/main/scala/org/apache/gearpump/streaming/task/TaskActor.scala @@ -23,10 +23,10 @@ import java.util import java.util.concurrent.TimeUnit import akka.actor._ +import com.gs.collections.impl.map.mutable.primitive.IntShortHashMap import org.apache.gearpump.streaming.source.Watermark import org.slf4j.Logger import org.apache.gearpump.cluster.UserConfig -import org.apache.gearpump.gs.collections.impl.map.mutable.primitive.IntShortHashMap import org.apache.gearpump.metrics.Metrics import org.apache.gearpump.serializer.SerializationFramework import org.apache.gearpump.streaming.AppMasterToExecutor._
