Move common project settings to build-level settings
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/00f4122a Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/00f4122a Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/00f4122a Branch: refs/heads/master Commit: 00f4122ae23c4c04a9706af0ea7e1ac1cb807d11 Parents: bf02de5 Author: Jakob Odersky <ja...@odersky.com> Authored: Mon Feb 13 16:16:29 2017 -0800 Committer: Jakob Odersky <ja...@odersky.com> Committed: Wed Feb 15 15:24:09 2017 -0800 ---------------------------------------------------------------------- Makefile | 5 +- build.sbt | 206 ++++++++++++++++++++++++------ kernel-api/build.sbt | 2 + kernel/build.sbt | 2 + plugins/build.sbt | 2 + project/CommonPlugin.scala | 41 ++++++ project/Dependencies.scala | 35 +++-- project/build.properties | 3 +- project/common.scala | 255 ------------------------------------- project/plugins.sbt | 2 +- protocol/build.sbt | 8 +- pyspark-interpreter/build.sbt | 1 + scala-interpreter/build.sbt | 38 +----- sparkr-interpreter/build.sbt | 1 + sql-interpreter/build.sbt | 1 + 15 files changed, 250 insertions(+), 352 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/Makefile ---------------------------------------------------------------------- diff --git a/Makefile b/Makefile index ec160e5..30bf04c 100644 --- a/Makefile +++ b/Makefile @@ -106,8 +106,9 @@ dev-binder: .binder-image target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): VM_WORKDIR=/src/toree-kernel target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): ${shell find ./*/src/main/**/*} target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): ${shell find ./*/build.sbt} -target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): dist/toree-legal project/build.properties build.sbt project/common.scala project/plugins.sbt - $(call RUN,$(ENV_OPTS) sbt toree/assembly) +target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): ${shell find ./project/*.scala} ${shell find ./project/*.sbt} +target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): dist/toree-legal project/build.properties build.sbt + $(call RUN,$(ENV_OPTS) sbt root/assembly) build: target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR) http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/build.sbt ---------------------------------------------------------------------- diff --git a/build.sbt b/build.sbt index a3d1f2a..e60a99e 100644 --- a/build.sbt +++ b/build.sbt @@ -15,25 +15,117 @@ * limitations under the License */ -import Common._ -import sbtunidoc.Plugin.UnidocKeys._ -import sbtunidoc.Plugin._ -import com.typesafe.sbt.SbtGit.{GitKeys => git} -import com.typesafe.sbt.SbtSite.site - -lazy val root = ToreeProject("toree", ".", doFork=false, needsSpark=true). - settings(unidocSettings:_*). - settings(site.settings:_*). - settings( - test in assembly := {}, - assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false), - scalacOptions in (ScalaUnidoc, unidoc) += "-Ymacro-no-expand", - git.gitRemoteRepo := "git://git.apache.org/incubator-toree.git", - aggregate in assembly := false, - unmanagedResourceDirectories in Compile += { baseDirectory.value / "dist/toree-legal" } - ).aggregate( +import scala.util.Properties + +// Version settings +version in ThisBuild := Properties.envOrElse("VERSION", "0.0.0-dev") + + (if ((isSnapshot in ThisBuild).value) "-SNAPSHOT" else "") +isSnapshot in ThisBuild := Properties.envOrElse("IS_SNAPSHOT","true").toBoolean +organization in ThisBuild := "org.apache.toree.kernel" +crossScalaVersions in ThisBuild := Seq("2.11.8") +scalaVersion in ThisBuild := (crossScalaVersions in ThisBuild).value.head +Dependencies.sparkVersion in ThisBuild := { + val envVar = "APACHE_SPARK_VERSION" + val defaultVersion = "2.0.0" + + Properties.envOrNone(envVar) match { + case None => + sLog.value.info(s"Using default Apache Spark version $defaultVersion!") + defaultVersion + case Some(version) => + sLog.value.info(s"Using Apache Spark version $version, provided from $envVar") + version + } +} + +// Compiler settings +scalacOptions in ThisBuild ++= Seq( + "-deprecation", + "-unchecked", + "-feature", + "-Xfatal-warnings", + "-language:reflectiveCalls", + "-target:jvm-1.6", + "-Xlint" // Scala 2.11.x only +) +// Java-based options for compilation (all tasks) +// NOTE: Providing a blank flag causes failures, only uncomment with options +//javacOptions in Compile ++= Seq(""), +// Java-based options for just the compile task +javacOptions in ThisBuild ++= Seq( + "-Xlint:all", // Enable all Java-based warnings + "-Xlint:-path", // Suppress path warnings since we get tons of them + "-Xlint:-options", + "-Xlint:-processing", + "-Werror", // Treat warnings as errors + "-source", "1.6", + "-target", "1.6" +) +// Options provided to forked JVMs through sbt, based on our .jvmopts file +javaOptions in ThisBuild ++= Seq( + "-Xms1024M", "-Xmx4096M", "-Xss2m", "-XX:MaxPermSize=1024M", + "-XX:ReservedCodeCacheSize=256M", "-XX:+TieredCompilation", + "-XX:+CMSPermGenSweepingEnabled", "-XX:+CMSClassUnloadingEnabled", + "-XX:+UseConcMarkSweepGC", "-XX:+HeapDumpOnOutOfMemoryError" +) +// Add additional test option to show time taken per test +testOptions in (ThisBuild, Test) += Tests.Argument("-oDF") + +// Build-wide dependencies +resolvers in ThisBuild ++= Seq( + "Apache Snapshots" at "http://repository.apache.org/snapshots/", + "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/" +) +updateOptions in ThisBuild := updateOptions.value.withCachedResolution(true) +libraryDependencies in ThisBuild ++= Seq( + Dependencies.scalaTest % "test", + Dependencies.mockito % "test", + Dependencies.jacksonDatabind % "test" +) + +ScoverageSbtPlugin.ScoverageKeys.coverageHighlighting in ThisBuild := false + +// Publish settings +useGpg in ThisBuild := true +pgpPassphrase in ThisBuild := Some(Properties.envOrElse("GPG_PASSWORD","").toArray) +publishTo in ThisBuild := { + if (isSnapshot.value) + Some("Apache Staging Repo" at "https://repository.apache.org/content/repositories/snapshots/") + else + Some("Apache Staging Repo" at "https://repository.apache.org/content/repositories/staging/") +} +mappings in packageBin in ThisBuild := Seq( + file("LICENSE") -> "LICENSE", + file("NOTICE") -> "NOTICE" +) +licenses in ThisBuild := Seq("Apache 2" -> url("http://www.apache.org/licenses/LICENSE-2.0.txt")) +pomExtra in ThisBuild := { + <parent> + <groupId>org.apache</groupId> + <artifactId>apache</artifactId> + <version>10</version> + </parent> + <url>http://toree.incubator.apache.org/</url> + <scm> + <url>g...@github.com:apache/incubator-toree.git</url> + <connection>scm:git:g...@github.com:apache/incubator-toree.git</connection> + <developerConnection> + scm:git:https://git-wip-us.apache.org/repos/asf/incubator-toree.git + </developerConnection> + <tag>HEAD</tag> + </scm> +} +credentials in ThisBuild+= Credentials(Path.userHome / ".ivy2" / ".credentials") + +// Project structure + +/** Root Toree project. */ +lazy val root = (project in file(".")) + .settings(name := "toree") + .aggregate( macros,protocol,plugins,communication,kernelApi,client,scalaInterpreter,sqlInterpreter,pysparkInterpreter,sparkrInterpreter,kernel - ).dependsOn( + ) + .dependsOn( macros,protocol,communication,kernelApi,client,scalaInterpreter,sqlInterpreter,pysparkInterpreter,sparkrInterpreter,kernel ) @@ -41,67 +133,105 @@ lazy val root = ToreeProject("toree", ".", doFork=false, needsSpark=true). * Project representing macros in Scala that must be compiled separately from * any other project using them. */ -lazy val macros = ToreeProject("macros") +lazy val macros = (project in file("macros")) + .settings(name := "toree-macros") /** * Project representing the IPython kernel message protocol in Scala. Used * by the client and kernel implementations. */ -lazy val protocol = ToreeProject("protocol").dependsOn(macros). - enablePlugins(BuildInfoPlugin).settings(buildInfoSettings:_*) +lazy val protocol = (project in file("protocol")) + .settings(name := "toree-protocol") + .dependsOn(macros) /** * Project representing base plugin system for the Toree infrastructure. */ -lazy val plugins = ToreeProject("plugins", doFork=true).dependsOn(macros) +lazy val plugins = (project in file("plugins")) + .settings(name := "toree-plugins") + .dependsOn(macros) /** * Project representing forms of communication used as input/output for the * client/kernel. */ -lazy val communication = ToreeProject("communication").dependsOn(macros, protocol) +lazy val communication = (project in file("communication")) + .settings(name := "toree-communication") + .dependsOn(macros, protocol) /** * Project representing the kernel-api code used by the Spark Kernel. Others can * import this to implement their own magics and plugins. */ -lazy val kernelApi = ToreeProject("kernel-api", needsSpark=true).dependsOn(macros, plugins) +lazy val kernelApi = (project in file("kernel-api")) + .settings(name := "toree-kernel-api") + .dependsOn(macros, plugins) /** * Project representing the client code for connecting to the kernel backend. */ -lazy val client = ToreeProject("client").dependsOn(macros, protocol, communication) +lazy val client = (project in file("client")) + .settings(name := "toree-client") + .dependsOn(macros, protocol, communication) /** * Project represents the scala interpreter used by the Spark Kernel. */ -lazy val scalaInterpreter = ToreeProject("scala-interpreter", needsSpark=true).dependsOn(plugins, protocol, kernelApi) +lazy val scalaInterpreter = (project in file("scala-interpreter")) + .settings(name := "toree-scala-interpreter") + .dependsOn(plugins, protocol, kernelApi) /** * Project represents the SQL interpreter used by the Spark Kernel. */ -lazy val sqlInterpreter = ToreeProject("sql-interpreter", needsSpark=true).dependsOn(plugins, protocol, kernelApi) +lazy val sqlInterpreter = (project in file("sql-interpreter")) + .settings(name := "toree-sql-interpreter") + .dependsOn(plugins, protocol, kernelApi) /** * Project represents the Python interpreter used by the Spark Kernel. */ -lazy val pysparkInterpreter = ToreeProject("pyspark-interpreter", needsSpark=true).dependsOn(plugins, protocol, kernelApi) +lazy val pysparkInterpreter = (project in file("pyspark-interpreter")) + .settings(name := "toree-pyspark-interpreter") + .dependsOn(plugins, protocol, kernelApi) /** * Project represents the R interpreter used by the Spark Kernel. */ -lazy val sparkrInterpreter = ToreeProject("sparkr-interpreter", needsSpark=true).dependsOn(plugins, protocol, kernelApi) +lazy val sparkrInterpreter = (project in file("sparkr-interpreter")) + .settings(name := "toree-sparkr-interpreter") + .dependsOn(plugins, protocol, kernelApi) /** * Project representing the kernel code for the Spark Kernel backend. */ -lazy val kernel = ToreeProject("kernel", doFork=true, needsSpark=true).dependsOn( - macros % "test->test;compile->compile", - protocol % "test->test;compile->compile", - communication % "test->test;compile->compile", - kernelApi % "test->test;compile->compile", - pysparkInterpreter % "test->test;compile->compile", - scalaInterpreter % "test->test;compile->compile", - sparkrInterpreter % "test->test;compile->compile", - sqlInterpreter % "test->test;compile->compile" +lazy val kernel = (project in file("kernel")) + .settings(name := "toree-kernel") + .dependsOn( + macros % "test->test;compile->compile", + protocol % "test->test;compile->compile", + communication % "test->test;compile->compile", + kernelApi % "test->test;compile->compile", + pysparkInterpreter % "test->test;compile->compile", + scalaInterpreter % "test->test;compile->compile", + sparkrInterpreter % "test->test;compile->compile", + sqlInterpreter % "test->test;compile->compile" + ) + +// Root project settings +enablePlugins(ScalaUnidocPlugin) +scalacOptions in (ScalaUnidoc, unidoc) ++= Seq( + "-Ymacro-expand:none", + "-skip-packages", Seq( + "akka", + "scala" + ).mkString(":"), + "-no-link-warnings" // Suppresses problems with Scaladoc @throws links ) + +libraryDependencies ++= Dependencies.sparkAll.value +unmanagedResourceDirectories in Compile += { baseDirectory.value / "dist/toree-legal" } + +test in assembly := {} +assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false) +aggregate in assembly := false http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/kernel-api/build.sbt ---------------------------------------------------------------------- diff --git a/kernel-api/build.sbt b/kernel-api/build.sbt index e1ab494..68be752 100644 --- a/kernel-api/build.sbt +++ b/kernel-api/build.sbt @@ -15,6 +15,8 @@ * limitations under the License */ +libraryDependencies ++= Dependencies.sparkAll.value + // // SCALA INTERPRETER DEPENDENCIES // http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/kernel/build.sbt ---------------------------------------------------------------------- diff --git a/kernel/build.sbt b/kernel/build.sbt index 752ad2a..5428b6e 100644 --- a/kernel/build.sbt +++ b/kernel/build.sbt @@ -14,6 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License */ +fork in Test := true +libraryDependencies ++= Dependencies.sparkAll.value // // TEST DEPENDENCIES http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/plugins/build.sbt ---------------------------------------------------------------------- diff --git a/plugins/build.sbt b/plugins/build.sbt index 99f1701..f41d40b 100644 --- a/plugins/build.sbt +++ b/plugins/build.sbt @@ -15,6 +15,8 @@ * limitations under the License */ +fork in Test := true + // Needed for type inspection libraryDependencies ++= Seq( Dependencies.scalaReflect.value, http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/project/CommonPlugin.scala ---------------------------------------------------------------------- diff --git a/project/CommonPlugin.scala b/project/CommonPlugin.scala new file mode 100644 index 0000000..a75a9a7 --- /dev/null +++ b/project/CommonPlugin.scala @@ -0,0 +1,41 @@ +import sbt._ +import sbt.Keys._ + +object CommonPlugin extends AutoPlugin { + + override def requires = plugins.JvmPlugin + override def trigger = allRequirements + + object autoImport { + lazy val UnitTest = config("unit") extend Test + lazy val IntegrationTest = config("integration") extend Test + lazy val SystemTest = config("system") extend Test + } + import autoImport._ + + override def projectSettings = { + inConfig(UnitTest)(Defaults.testSettings) ++ + inConfig(IntegrationTest)(Defaults.testSettings) ++ + inConfig(SystemTest)(Defaults.testSettings) ++ + Seq( + testOptions in UnitTest := Seq(Tests.Filter(unitFilter)), + testOptions in IntegrationTest := Seq(Tests.Filter(intFilter)), + testOptions in SystemTest := Seq(Tests.Filter(sysFilter)), + // Add a global resource directory with compile/ and test/ for resources in all projects + unmanagedResourceDirectories in Compile ++= Seq( + (baseDirectory in ThisBuild).value / "resources" / "compile" + ), + unmanagedResourceDirectories in Test ++= Seq( + (baseDirectory in ThisBuild).value / "resources" / "test" + ) + ) + } + + def sysFilter(name: String): Boolean = + (name endsWith "SpecForSystem") || (name startsWith "system.") + def intFilter(name: String): Boolean = + (name endsWith "SpecForIntegration") || (name startsWith "integration.") + def unitFilter(name: String): Boolean = + (name endsWith "Spec") && !intFilter(name) && !sysFilter(name) + +} http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/project/Dependencies.scala ---------------------------------------------------------------------- diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 2efdf35..508461f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -4,20 +4,7 @@ import scala.util.Properties object Dependencies { - val sparkVersion = Def.setting{ - //sLog.value.warn("danger!") - val envVar = "APACHE_SPARK_VERSION" - val defaultVersion = "2.0.0" - - Properties.envOrNone(envVar) match { - case None => - sLog.value.info(s"Using default Apache Spark version $defaultVersion!") - defaultVersion - case Some(version) => - sLog.value.info(s"Using Apache Spark version $version, provided from $envVar") - version - } - } + // Libraries val akkaActor = "com.typesafe.akka" %% "akka-actor" % "2.4.17" val akkaSlf4j = "com.typesafe.akka" %% "akka-slf4j" % "2.4.17" @@ -54,6 +41,7 @@ object Dependencies { val slf4jApi = "org.slf4j" % "slf4j-api" % "1.7.21" // MIT + val sparkVersion = settingKey[String]("Version of Apache Spark to use in Toree") // defined in root build val sparkCore = Def.setting{ "org.apache.spark" %% "spark-core" % sparkVersion.value } val sparkGraphX = Def.setting{ "org.apache.spark" %% "spark-graphx" % sparkVersion.value } val sparkMllib = Def.setting{ "org.apache.spark" %% "spark-mllib" % sparkVersion.value } @@ -63,4 +51,23 @@ object Dependencies { val springCore = "org.springframework" % "spring-core" % "4.1.1.RELEASE"// Apache v2 + // Projects + + val sparkAll = Def.setting{ + Seq( + sparkCore.value % "provided" excludeAll( + // Exclude netty (org.jboss.netty is for 3.2.2.Final only) + ExclusionRule( + organization = "org.jboss.netty", + name = "netty" + ) + ), + sparkGraphX.value % "provided", + sparkMllib.value % "provided", + sparkRepl.value % "provided", + sparkSql.value % "provided", + sparkStreaming.value % "provided" + ) + } + } http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/project/build.properties ---------------------------------------------------------------------- diff --git a/project/build.properties b/project/build.properties index 69fc25e..8dd784c 100644 --- a/project/build.properties +++ b/project/build.properties @@ -15,5 +15,4 @@ # limitations under the License # -sbt.version=0.13.9 - +sbt.version=0.13.13 http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/project/common.scala ---------------------------------------------------------------------- diff --git a/project/common.scala b/project/common.scala deleted file mode 100644 index 2e87135..0000000 --- a/project/common.scala +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License - */ -import org.apache.commons.io.FileUtils -import sbt._ -import sbt.Keys._ -import sbtbuildinfo._ -import sbtbuildinfo.BuildInfoKeys._ -import scoverage.ScoverageSbtPlugin -//import coursier.Keys._ -import com.typesafe.sbt.pgp.PgpKeys._ -import scala.util.{Try, Properties} - -import Dependencies.sparkVersion - -object Common { - // Parameters for publishing to artifact repositories - private val versionNumber = Properties.envOrElse("VERSION", "0.0.0-dev") - private val snapshot = Properties.envOrElse("IS_SNAPSHOT","true").toBoolean - private val gpgLocation = Properties.envOrElse("GPG","/usr/local/bin/gpg") - private val gpgPassword = Properties.envOrElse("GPG_PASSWORD","") - private val buildOrganization = "org.apache.toree.kernel" - private val buildVersion = if (snapshot) s"$versionNumber-SNAPSHOT" else versionNumber - private val buildScalaVersion = "2.11.8" -// private val buildScalaVersion = "2.10.6" - - val buildInfoSettings = Seq( - buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion, "sparkVersion" -> sparkVersion.value), - buildInfoPackage := buildOrganization, - buildInfoUsePackageAsPath := true, - buildInfoOptions += BuildInfoOption.BuildTime - ) - - def ToreeProject(projectName: String, doFork: Boolean = false, needsSpark: Boolean = false):Project = ( - ToreeProject(s"toree-$projectName", projectName, doFork, needsSpark) - ) - - def ToreeProject(projectName: String, projectDir: String, doFork: Boolean, needsSpark: Boolean):Project = ( - Project(projectName, file(projectDir)) - .configs( UnitTest ) - .configs( IntegrationTest ) - .configs( SystemTest ) - .configs( ScratchTest ) - .settings(commonSettings:_*) - .settings( inConfig(UnitTest)(Defaults.testTasks) : _*) - .settings( inConfig(IntegrationTest)(Defaults.testTasks) : _*) - .settings( inConfig(SystemTest)(Defaults.testTasks) : _*) - .settings( inConfig(ScratchTest)(Defaults.testTasks) : _*) - .settings( - testOptions in UnitTest := Seq(Tests.Filter(unitFilter)), - testOptions in IntegrationTest := Seq(Tests.Filter(intFilter)), - testOptions in SystemTest := Seq(Tests.Filter(sysFilter)), - testOptions in ScratchTest := Seq(Tests.Filter(scratchFilter)) - ).settings( - fork in Test := doFork, - fork in UnitTest := doFork, - fork in IntegrationTest := doFork, - fork in SystemTest := doFork, - fork in ScratchTest := doFork, - libraryDependencies ++= ( - if (needsSpark) Seq( - Dependencies.sparkCore.value % "provided" excludeAll( - // Exclude netty (org.jboss.netty is for 3.2.2.Final only) - ExclusionRule( - organization = "org.jboss.netty", - name = "netty" - ) - ), - Dependencies.sparkGraphX.value % "provided", - Dependencies.sparkMllib.value % "provided", - Dependencies.sparkRepl.value % "provided", - Dependencies.sparkSql.value % "provided", - Dependencies.sparkStreaming.value % "provided" - ) - else Nil - ) - ) - ) - - def scratchFilter(name: String): Boolean = - (name endsWith "SpecForScratch") || (name startsWith "scratch.") - def sysFilter(name: String): Boolean = - (name endsWith "SpecForSystem") || (name startsWith "system.") - def intFilter(name: String): Boolean = - (name endsWith "SpecForIntegration") || (name startsWith "integration.") - def unitFilter(name: String): Boolean = - (name endsWith "Spec") && !intFilter(name) && - !sysFilter(name) && !scratchFilter(name) - - lazy val UnitTest = config("unit") extend Test - lazy val IntegrationTest = config("integration") extend Test - lazy val SystemTest = config("system") extend Test - lazy val ScratchTest = config("scratch") extend Test - - val commonSettings: Seq[Def.Setting[_]] = Seq( - organization := buildOrganization, - useGpg := true, - gpgCommand := gpgLocation, - pgpPassphrase in Global := Some(gpgPassword.toArray), - version := buildVersion, - scalaVersion := buildScalaVersion, -// crossScalaVersions := Seq("2.10.5", "2.11.8"), - crossScalaVersions := Seq("2.11.8"), - isSnapshot := snapshot, - updateOptions := updateOptions.value.withCachedResolution(true), - resolvers ++= Seq( - "Apache Snapshots" at "http://repository.apache.org/snapshots/", - "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/" - ), - // Test dependencies - libraryDependencies ++= Seq( - Dependencies.scalaTest % "test", - Dependencies.mockito % "test", - Dependencies.jacksonDatabind % "test" - ), - ScoverageSbtPlugin.ScoverageKeys.coverageHighlighting := false, - pomExtra := - <parent> - <groupId>org.apache</groupId> - <artifactId>apache</artifactId> - <version>10</version> - </parent> - <licenses> - <license> - <name>Apache 2</name> - <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> - <distribution>repo</distribution> - </license> - </licenses> - <url>http://toree.incubator.apache.org/</url> - <scm> - <connection>scm:git:g...@github.com:apache/incubator-toree.git</connection> - <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-toree.git</developerConnection> - <url>scm:git:g...@github.com:apache/incubator-toree.git</url> - <tag>HEAD</tag> - </scm>, - - mappings in packageBin in Compile += file("LICENSE") -> "LICENSE", - mappings in packageBin in Compile += file("NOTICE") -> "NOTICE", - -// coursierVerbosity := { -// val level = Try(Integer.valueOf(Properties.envOrElse( -// "TOREE_RESOLUTION_VERBOSITY", "1") -// ).toInt).getOrElse(1) -// -// scala.Console.out.println( -// s"[INFO] Toree Resolution Verbosity Level = $level" -// ) -// -// level -// }, - - scalacOptions in (Compile, doc) ++= Seq( - // Ignore packages (for Scaladoc) not from our project - "-skip-packages", Seq( - "akka", - "scala" - ).mkString(":") - ), - - // Scala-based options for compilation - scalacOptions ++= Seq( - "-deprecation", - "-unchecked", "-feature", - //"-Xlint", // Scala 2.11.x only - "-Xfatal-warnings", - //"-Ywarn-all", - "-language:reflectiveCalls", - "-target:jvm-1.6" - ), - - // Java-based options for compilation (all tasks) - // NOTE: Providing a blank flag causes failures, only uncomment with options - //javacOptions in Compile ++= Seq(""), - - // Java-based options for just the compile task - javacOptions in (Compile, compile) ++= Seq( - "-Xlint:all", // Enable all Java-based warnings - "-Xlint:-path", // Suppress path warnings since we get tons of them - "-Xlint:-options", - "-Xlint:-processing", - "-Werror", // Treat warnings as errors - "-source", "1.6", - "-target", "1.6" - ), - - scalacOptions in (Compile, doc) ++= Seq( - "-no-link-warnings" // Suppresses problems with Scaladoc @throws links - ), - - // Options provided to forked JVMs through sbt, based on our .jvmopts file - javaOptions ++= Seq( - "-Xms1024M", "-Xmx4096M", "-Xss2m", "-XX:MaxPermSize=1024M", - "-XX:ReservedCodeCacheSize=256M", "-XX:+TieredCompilation", - "-XX:+CMSPermGenSweepingEnabled", "-XX:+CMSClassUnloadingEnabled", - "-XX:+UseConcMarkSweepGC", "-XX:+HeapDumpOnOutOfMemoryError" - ), - - // Add additional test option to show time taken per test - testOptions in Test += Tests.Argument("-oDF"), - - // Add a global resource directory with compile/ and test/ for resources in all projects - unmanagedResourceDirectories in Compile += file("resources/compile"), - unmanagedResourceDirectories in Test += file("resources/test"), - - // Publish Settings - publishTo := { - if (isSnapshot.value) - Some("Apache Staging Repo" at "https://repository.apache.org/content/repositories/snapshots/") - else - Some("Apache Staging Repo" at "https://repository.apache.org/content/repositories/staging/") - }, - credentials += Credentials(Path.userHome / ".ivy2" / ".credentials"), - - // Add rebuild ivy xml to the following tasks - compile <<= (compile in Compile) dependsOn (rebuildIvyXml dependsOn deliverLocal) - ) ++ rebuildIvyXmlSettings // Include our rebuild ivy xml settings - - // ========================================================================== - // = REBUILD IVY XML SETTINGS BELOW - // ========================================================================== - - lazy val rebuildIvyXml = TaskKey[Unit]( - "rebuild-ivy-xml", - "Rebuilds the ivy xml using deliver-local and copies it to src " + - "resource directories" - ) - - // TODO: Figure out how to retrieve the configuration being used to avoid - // this duplication - lazy val rebuildIvyXmlSettings = Seq( - rebuildIvyXml := { - val s: TaskStreams = streams.value - val inputFile = (crossTarget.value / s"ivy-${version.value}.xml").getAbsoluteFile - val outputFile = - ((resourceDirectory in Compile).value / s"${name.value}-ivy.xml").getAbsoluteFile - s.log.info(s"Copying ${inputFile.getPath} to ${outputFile.getPath}") - FileUtils.copyFile(inputFile, outputFile) - } - ) - -} http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/project/plugins.sbt ---------------------------------------------------------------------- diff --git a/project/plugins.sbt b/project/plugins.sbt index 61c6905..2b299da 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -27,7 +27,7 @@ resolvers += "Apache Snapshots" at "http://repository.apache.org/snapshots/" addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.6.0") // Provides the ability to generate unifed documentation for multiple projects -addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.3.1") +addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.0") // Provides the ability to list dependencies in a readable format using // `sbt dependencyTree`; there are other commands provided as well http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/protocol/build.sbt ---------------------------------------------------------------------- diff --git a/protocol/build.sbt b/protocol/build.sbt index ca70698..0ed902e 100644 --- a/protocol/build.sbt +++ b/protocol/build.sbt @@ -25,6 +25,8 @@ libraryDependencies ++= Seq( Dependencies.slf4jApi ) -// -// TEST DEPENDENCIES -// +enablePlugins(BuildInfoPlugin) +buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion, "sparkVersion" -> Dependencies.sparkVersion.value) +buildInfoPackage := organization.value +buildInfoUsePackageAsPath := true +buildInfoOptions += BuildInfoOption.BuildTime http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/pyspark-interpreter/build.sbt ---------------------------------------------------------------------- diff --git a/pyspark-interpreter/build.sbt b/pyspark-interpreter/build.sbt index 9444250..ac89909 100644 --- a/pyspark-interpreter/build.sbt +++ b/pyspark-interpreter/build.sbt @@ -14,3 +14,4 @@ * See the License for the specific language governing permissions and * limitations under the License */ +libraryDependencies ++= Dependencies.sparkAll.value http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/scala-interpreter/build.sbt ---------------------------------------------------------------------- diff --git a/scala-interpreter/build.sbt b/scala-interpreter/build.sbt index 5d6c95f..c07b032 100644 --- a/scala-interpreter/build.sbt +++ b/scala-interpreter/build.sbt @@ -16,41 +16,5 @@ import sbt.Tests.{Group, SubProcess} * See the License for the specific language governing permissions and * limitations under the License */ -// The chunk below is used to ensure that the test tagged with SbtForked ACTUALLY runs with -// a forked jvm, - -def isAnnotatedWithRequiresSpark(definition: xsbti.api.Definition): Boolean = { - definition.annotations().exists { annotation: xsbti.api.Annotation => - annotation.base match { - case proj: xsbti.api.Projection if (proj.id() == "SbtForked") => true - case _ => false - } - } -} - -// Note the type TaskKey[Seq[String]] must be explicitly specified otherwise an error occurs -lazy val testsAnnotatedWithRequiresSpark: TaskKey[Seq[String]] = taskKey[Seq[String]]("Returns list of FQCNs of tests annotated with SbtForked") -testsAnnotatedWithRequiresSpark := { - val analysis = (compile in Test).value - analysis.apis.internal.values.flatMap({ source => - source.api().definitions().filter(isAnnotatedWithRequiresSpark).map(_.name()) - }).toSeq -} - - -def forkedJvmPerTest(testDefs: Seq[TestDefinition], testsToFork: Seq[String]) = { - - val (forkedTests, otherTests) = testDefs.partition { testDef => testsToFork.contains(testDef.name) } - - val otherTestsGroup = new Group(name = "Single JVM tests", tests = otherTests, runPolicy = SubProcess(javaOptions = Seq.empty[String])) - val forkedTestGroups = forkedTests map { test => - new Group( - name = test.name, - tests = Seq(test), - runPolicy = SubProcess(javaOptions = Seq.empty[String])) - } - Seq(otherTestsGroup) ++ forkedTestGroups -} - -testGrouping in Test <<= (definedTests in Test, testsAnnotatedWithRequiresSpark) map forkedJvmPerTest +libraryDependencies ++= Dependencies.sparkAll.value http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/sparkr-interpreter/build.sbt ---------------------------------------------------------------------- diff --git a/sparkr-interpreter/build.sbt b/sparkr-interpreter/build.sbt index 9444250..ac89909 100644 --- a/sparkr-interpreter/build.sbt +++ b/sparkr-interpreter/build.sbt @@ -14,3 +14,4 @@ * See the License for the specific language governing permissions and * limitations under the License */ +libraryDependencies ++= Dependencies.sparkAll.value http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/sql-interpreter/build.sbt ---------------------------------------------------------------------- diff --git a/sql-interpreter/build.sbt b/sql-interpreter/build.sbt index 9444250..ac89909 100644 --- a/sql-interpreter/build.sbt +++ b/sql-interpreter/build.sbt @@ -14,3 +14,4 @@ * See the License for the specific language governing permissions and * limitations under the License */ +libraryDependencies ++= Dependencies.sparkAll.value