Repository: incubator-predictionio Updated Branches: refs/heads/feature/xbuild 2bfe640ba -> 36d8c6939
More build and test fixes Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/36d8c693 Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/36d8c693 Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/36d8c693 Branch: refs/heads/feature/xbuild Commit: 36d8c69396dde9c1eb381bcfdfcd24cbd84c1b38 Parents: 2bfe640 Author: Donald Szeto <[email protected]> Authored: Fri Mar 17 16:40:11 2017 -0700 Committer: Donald Szeto <[email protected]> Committed: Fri Mar 17 16:40:11 2017 -0700 ---------------------------------------------------------------------- build.sbt | 14 ++++++++------ core/build.sbt | 9 --------- data/build.sbt | 10 ---------- e2/build.sbt | 2 -- project/PIOBuild.scala | 9 ++++++--- storage/elasticsearch/build.sbt | 28 ++++++++-------------------- storage/elasticsearch1/build.sbt | 16 ++-------------- storage/hbase/build.sbt | 12 +----------- storage/hdfs/build.sbt | 18 +++++++----------- storage/jdbc/build.sbt | 12 +----------- storage/localfs/build.sbt | 14 ++------------ tests/build_docker.sh | 7 +++++-- tests/docker-files/env-conf/pio-env.sh | 7 ++++++- tools/build.sbt | 16 ---------------- 14 files changed, 46 insertions(+), 128 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/36d8c693/build.sbt ---------------------------------------------------------------------- diff --git a/build.sbt b/build.sbt index 4287e50..fc47f6a 100644 --- a/build.sbt +++ b/build.sbt @@ -67,19 +67,21 @@ javacOptions in (ThisBuild, compile) ++= Seq("-source", "1.7", "-target", "1.7", // Ignore differentiation of Spark patch levels sparkVersion in ThisBuild := sys.props.getOrElse("spark.version", "1.6.3") +sparkBinaryVersion in ThisBuild := binaryVersion(sparkVersion.value) + akkaVersion in ThisBuild := sys.props.getOrElse( "akka.version", - scalaSparkDepsVersion(versionPrefix(scalaVersion.value))(versionPrefix(sparkVersion.value))("akka")) + scalaSparkDepsVersion(scalaBinaryVersion.value)(sparkBinaryVersion.value)("akka")) lazy val es = sys.props.getOrElse("elasticsearch.version", "1.7.6") elasticsearchVersion in ThisBuild := es -json4sVersion in ThisBuild := scalaSparkDepsVersion(versionPrefix(scalaVersion.value))(versionPrefix(sparkVersion.value))("json4s") +json4sVersion in ThisBuild := scalaSparkDepsVersion(scalaBinaryVersion.value)(sparkBinaryVersion.value)("json4s") hadoopVersion in ThisBuild := sys.props.getOrElse( "hadoop.version", - scalaSparkDepsVersion(versionPrefix(scalaVersion.value))(versionPrefix(sparkVersion.value))("hadoop")) + scalaSparkDepsVersion(scalaBinaryVersion.value)(sparkBinaryVersion.value)("hadoop")) val pioBuildInfoSettings = buildInfoSettings ++ Seq( sourceGenerators in Compile <+= buildInfo, @@ -102,7 +104,7 @@ val conf = file("conf") val commonSettings = Seq( autoAPIMappings := true, unmanagedClasspath in Test += conf, - unmanagedClasspath in Test += baseDirectory.value / s"../storage/jdbc/target/scala-${versionPrefix(scalaVersion.value)}/classes") + unmanagedClasspath in Test += baseDirectory.value.getParentFile / s"storage/jdbc/target/scala-${scalaBinaryVersion.value}/classes") val commonTestSettings = Seq( libraryDependencies ++= Seq( @@ -150,7 +152,7 @@ val data = (project in file("data")). settings(commonTestSettings: _*). settings(genjavadocSettings: _*). settings(unmanagedSourceDirectories in Compile += - sourceDirectory.value / s"main/spark-${versionMajor(sparkVersion.value)}"). + sourceDirectory.value / s"main/spark-${majorVersion(sparkVersion.value)}"). disablePlugins(sbtassembly.AssemblyPlugin) val core = (project in file("core")). @@ -176,7 +178,7 @@ val e2 = (project in file("e2")). settings(genjavadocSettings: _*). disablePlugins(sbtassembly.AssemblyPlugin) -val dataEs = if (versionMajor(es) == 1) dataElasticsearch1 else dataElasticsearch +val dataEs = if (majorVersion(es) == 1) dataElasticsearch1 else dataElasticsearch val storageSubprojects = Seq( dataEs, http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/36d8c693/core/build.sbt ---------------------------------------------------------------------- diff --git a/core/build.sbt b/core/build.sbt index d040d4c..b95a957 100644 --- a/core/build.sbt +++ b/core/build.sbt @@ -22,19 +22,10 @@ name := "apache-predictionio-core" libraryDependencies ++= Seq( "com.github.scopt" %% "scopt" % "3.5.0", "com.google.code.gson" % "gson" % "2.5", - "com.google.guava" % "guava" % "19.0", - "com.twitter" %% "chill" % "0.7.2" - exclude("com.esotericsoftware.minlog", "minlog"), "com.twitter" %% "chill-bijection" % "0.7.2", "de.javakaffee" % "kryo-serializers" % "0.37", - "commons-io" % "commons-io" % "2.4", - "io.spray" %% "spray-can" % "1.3.3", - "io.spray" %% "spray-routing" % "1.3.3", "net.jodah" % "typetools" % "0.3.1", "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided", - "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided", - "org.clapper" %% "grizzled-slf4j" % "1.0.2", - "org.json4s" %% "json4s-native" % json4sVersion.value, "org.json4s" %% "json4s-ext" % json4sVersion.value, "org.scalaj" %% "scalaj-http" % "1.1.6", "org.slf4j" % "slf4j-log4j12" % "1.7.18", http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/36d8c693/data/build.sbt ---------------------------------------------------------------------- diff --git a/data/build.sbt b/data/build.sbt index 7c34f23..ddd085d 100644 --- a/data/build.sbt +++ b/data/build.sbt @@ -22,22 +22,12 @@ name := "apache-predictionio-data" libraryDependencies ++= Seq( "com.github.nscala-time" %% "nscala-time" % "2.6.0", "com.google.guava" % "guava" % "19.0", - "commons-codec" % "commons-codec" % "1.9", "io.spray" %% "spray-can" % "1.3.3", "io.spray" %% "spray-routing" % "1.3.3", - "org.apache.hadoop" % "hadoop-common" % hadoopVersion.value - exclude("javax.servlet", "servlet-api"), - "org.apache.zookeeper" % "zookeeper" % "3.4.7" - exclude("org.slf4j", "slf4j-api") - exclude("org.slf4j", "slf4j-log4j12"), - "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided", "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided", "org.clapper" %% "grizzled-slf4j" % "1.0.2", "org.json4s" %% "json4s-native" % json4sVersion.value, - "org.json4s" %% "json4s-ext" % json4sVersion.value, - "org.slf4j" % "slf4j-log4j12" % "1.7.18", "org.scalikejdbc" %% "scalikejdbc" % "2.3.2", - "com.typesafe.akka" %% "akka-actor" % akkaVersion.value, "io.spray" %% "spray-testkit" % "1.3.3" % "test", "org.scalatest" %% "scalatest" % "2.1.7" % "test", "org.specs2" %% "specs2" % "2.3.13" % "test") http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/36d8c693/e2/build.sbt ---------------------------------------------------------------------- diff --git a/e2/build.sbt b/e2/build.sbt index f30536a..0774a55 100644 --- a/e2/build.sbt +++ b/e2/build.sbt @@ -22,9 +22,7 @@ name := "apache-predictionio-e2" parallelExecution in Test := false libraryDependencies ++= Seq( - "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided", "org.apache.spark" %% "spark-mllib" % sparkVersion.value % "provided", - "org.clapper" %% "grizzled-slf4j" % "1.0.2", "org.scalatest" %% "scalatest" % "2.2.5" % "test") pomExtra := childrenPomExtra.value http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/36d8c693/project/PIOBuild.scala ---------------------------------------------------------------------- diff --git a/project/PIOBuild.scala b/project/PIOBuild.scala index 69ac189..30fca65 100644 --- a/project/PIOBuild.scala +++ b/project/PIOBuild.scala @@ -21,13 +21,16 @@ object PIOBuild { val elasticsearchVersion = settingKey[String]("The version of Elasticsearch used for building") val json4sVersion = settingKey[String]("The version of JSON4S used for building") val sparkVersion = settingKey[String]("The version of Apache Spark used for building") + val sparkBinaryVersion = settingKey[String]("The binary version of Apache Spark used for building") val hadoopVersion = settingKey[String]("The version of Apache Hadoop used for building") val akkaVersion = settingKey[String]("The version of Akka used for building") + val childrenPomExtra = settingKey[scala.xml.NodeSeq]("Extra POM data for children projects") + val elasticsearchSparkArtifact = settingKey[String]("Name of Elasticsearch-Spark artifact used for building") - def versionPrefix(versionString: String): String = versionString.split('.').take(2).mkString(".") - def versionMajor(versionString: String): Int = versionString.split('.')(0).toInt - def versionMinor(versionString: String): Int = versionString.split('.')(1).toInt + def binaryVersion(versionString: String): String = versionString.split('.').take(2).mkString(".") + def majorVersion(versionString: String): Int = versionString.split('.')(0).toInt + def minorVersion(versionString: String): Int = versionString.split('.')(1).toInt lazy val printBuildInfo = taskKey[Unit]("Print build information") } http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/36d8c693/storage/elasticsearch/build.sbt ---------------------------------------------------------------------- diff --git a/storage/elasticsearch/build.sbt b/storage/elasticsearch/build.sbt index a7d3ab5..52dc965 100644 --- a/storage/elasticsearch/build.sbt +++ b/storage/elasticsearch/build.sbt @@ -19,34 +19,22 @@ import PIOBuild._ name := "apache-predictionio-data-elasticsearch" -elasticsearchVersion := "5.2.1" +elasticsearchSparkArtifact := (if (majorVersion(sparkVersion.value) == 2) "elasticsearch-spark-20" else "elasticsearch-spark-13") libraryDependencies ++= Seq( "org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided", - "org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided", - "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided", - "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided", - "org.elasticsearch.client" % "rest" % elasticsearchVersion.value, - "org.elasticsearch" %% "elasticsearch-spark-13" % elasticsearchVersion.value - exclude("org.apache.spark", "spark-sql_2.10") - exclude("org.apache.spark", "spark-streaming_2.10"), - "org.elasticsearch" % "elasticsearch-hadoop-mr" % elasticsearchVersion.value, - "org.scalatest" %% "scalatest" % "2.1.7" % "test", - "org.specs2" %% "specs2" % "2.3.13" % "test") + "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided", + "org.elasticsearch.client" % "rest" % elasticsearchVersion.value, + "org.elasticsearch" %% elasticsearchSparkArtifact.value % elasticsearchVersion.value + exclude("org.apache.spark", "*"), + "org.elasticsearch" % "elasticsearch-hadoop-mr" % elasticsearchVersion.value, + "org.scalatest" %% "scalatest" % "2.1.7" % "test") parallelExecution in Test := false pomExtra := childrenPomExtra.value -assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true) - -assemblyMergeStrategy in assembly := { - case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat - case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) -} +assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false) assemblyShadeRules in assembly := Seq( ShadeRule.rename("org.apache.http.**" -> "shadeio.data.http.@1").inAll http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/36d8c693/storage/elasticsearch1/build.sbt ---------------------------------------------------------------------- diff --git a/storage/elasticsearch1/build.sbt b/storage/elasticsearch1/build.sbt index c78c75d..bb68b11 100644 --- a/storage/elasticsearch1/build.sbt +++ b/storage/elasticsearch1/build.sbt @@ -19,28 +19,16 @@ import PIOBuild._ name := "apache-predictionio-data-elasticsearch1" -elasticsearchVersion := "1.7.3" - libraryDependencies ++= Seq( "org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided", - "org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided", "org.elasticsearch" % "elasticsearch" % elasticsearchVersion.value, - "org.scalatest" %% "scalatest" % "2.1.7" % "test", - "org.specs2" %% "specs2" % "2.3.13" % "test") + "org.scalatest" %% "scalatest" % "2.1.7" % "test") parallelExecution in Test := false pomExtra := childrenPomExtra.value -assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true) - -assemblyMergeStrategy in assembly := { - case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat - case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) -} +assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false) // skip test in assembly test in assembly := {} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/36d8c693/storage/hbase/build.sbt ---------------------------------------------------------------------- diff --git a/storage/hbase/build.sbt b/storage/hbase/build.sbt index 22b53d8..19aa126 100644 --- a/storage/hbase/build.sbt +++ b/storage/hbase/build.sbt @@ -21,7 +21,6 @@ name := "apache-predictionio-data-hbase" libraryDependencies ++= Seq( "org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided", - "org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided", "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided", "org.apache.hbase" % "hbase-common" % "0.98.5-hadoop2", "org.apache.hbase" % "hbase-client" % "0.98.5-hadoop2" @@ -34,22 +33,13 @@ libraryDependencies ++= Seq( exclude("org.mortbay.jetty", "servlet-api-2.5") exclude("org.mortbay.jetty", "jsp-api-2.1") exclude("org.mortbay.jetty", "jsp-2.1"), - "org.scalatest" %% "scalatest" % "2.1.7" % "test", "org.specs2" %% "specs2" % "2.3.13" % "test") parallelExecution in Test := false pomExtra := childrenPomExtra.value -assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true) - -assemblyMergeStrategy in assembly := { - case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat - case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) -} +assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false) // skip test in assembly test in assembly := {} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/36d8c693/storage/hdfs/build.sbt ---------------------------------------------------------------------- diff --git a/storage/hdfs/build.sbt b/storage/hdfs/build.sbt index 2942e56..8bae2a8 100644 --- a/storage/hdfs/build.sbt +++ b/storage/hdfs/build.sbt @@ -20,24 +20,20 @@ import PIOBuild._ name := "apache-predictionio-data-hdfs" libraryDependencies ++= Seq( - "org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided", + "org.apache.hadoop" % "hadoop-common" % hadoopVersion.value + exclude("commons-beanutils", "*"), "org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided", - "org.scalatest" %% "scalatest" % "2.1.7" % "test", - "org.specs2" %% "specs2" % "2.3.13" % "test") + "org.scalatest" %% "scalatest" % "2.1.7" % "test") parallelExecution in Test := false pomExtra := childrenPomExtra.value -assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true) +assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false) -assemblyMergeStrategy in assembly := { - case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat - case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) -} +assemblyOption in assembly := (assemblyOption in assembly).value.copy( + includeScala = false, + excludedJars = (fullClasspath in assembly).value.filter {_.data.getName startsWith "apache-predictionio"}) // skip test in assembly test in assembly := {} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/36d8c693/storage/jdbc/build.sbt ---------------------------------------------------------------------- diff --git a/storage/jdbc/build.sbt b/storage/jdbc/build.sbt index b0ff2fd..dc98e21 100644 --- a/storage/jdbc/build.sbt +++ b/storage/jdbc/build.sbt @@ -21,26 +21,16 @@ name := "apache-predictionio-data-jdbc" libraryDependencies ++= Seq( "org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided", - "org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided", "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided", "org.scalikejdbc" %% "scalikejdbc" % "2.3.5", "org.postgresql" % "postgresql" % "9.4-1204-jdbc41" % "test", - "org.scalatest" %% "scalatest" % "2.1.7" % "test", "org.specs2" %% "specs2" % "2.3.13" % "test") parallelExecution in Test := false pomExtra := childrenPomExtra.value -assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true) - -assemblyMergeStrategy in assembly := { - case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat - case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) -} +assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false) // skip test in assembly test in assembly := {} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/36d8c693/storage/localfs/build.sbt ---------------------------------------------------------------------- diff --git a/storage/localfs/build.sbt b/storage/localfs/build.sbt index 90a61db..f8c6b8e 100644 --- a/storage/localfs/build.sbt +++ b/storage/localfs/build.sbt @@ -21,23 +21,13 @@ name := "apache-predictionio-data-localfs" libraryDependencies ++= Seq( "org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided", - "org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided", - "org.scalatest" %% "scalatest" % "2.1.7" % "test", - "org.specs2" %% "specs2" % "2.3.13" % "test") + "org.scalatest" %% "scalatest" % "2.1.7" % "test") parallelExecution in Test := false pomExtra := childrenPomExtra.value -assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true) - -assemblyMergeStrategy in assembly := { - case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat - case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) -} +assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false) // skip test in assembly test in assembly := {} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/36d8c693/tests/build_docker.sh ---------------------------------------------------------------------- diff --git a/tests/build_docker.sh b/tests/build_docker.sh index 9b6c2c6..af30cb8 100755 --- a/tests/build_docker.sh +++ b/tests/build_docker.sh @@ -37,9 +37,11 @@ fi -Dspark.version=$PIO_SPARK_VERSION \ -Dhadoop.version=$PIO_HADOOP_VERSION \ -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION -sbt/sbt clean +sbt/sbt clean storage/clean +rm -rf assembly mkdir assembly cp dist/lib/*.jar assembly/ +rm -rf lib/spark mkdir -p lib/spark cp dist/lib/spark/*.jar lib/spark rm *.tar.gz @@ -51,4 +53,5 @@ docker build -t predictionio/pio-testing $DIR \ --build-arg SPARK_DIR=$SPARK_DIR \ --build-arg PGSQL_JAR=$PGSQL_JAR \ --build-arg PIO_SCALA_VERSION=$PIO_SCALA_VERSION \ - --build-arg PIO_SPARK_VERSION=$PIO_SPARK_VERSION + --build-arg PIO_SPARK_VERSION=$PIO_SPARK_VERSION \ + --build-arg PIO_ELASTICSEARCH_VERSION=$PIO_ELASTICSEARCH_VERSION http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/36d8c693/tests/docker-files/env-conf/pio-env.sh ---------------------------------------------------------------------- diff --git a/tests/docker-files/env-conf/pio-env.sh b/tests/docker-files/env-conf/pio-env.sh index 153c4ee..16f1fef 100644 --- a/tests/docker-files/env-conf/pio-env.sh +++ b/tests/docker-files/env-conf/pio-env.sh @@ -88,7 +88,12 @@ PIO_STORAGE_SOURCES_ELASTICSEARCH_TYPE=elasticsearch #PIO_STORAGE_SOURCES_ELASTICSEARCH_CLUSTERNAME=pio PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=elasticsearch PIO_STORAGE_SOURCES_ELASTICSEARCH_SCHEMES=http -if [ "$PIO_ELASTICSEARCH_VERSION" = "1" ]; then +if [ ! -z "$PIO_ELASTICSEARCH_VERSION" ]; then + ES_MAJOR=`echo $PIO_ELASTICSEARCH_VERSION | awk -F. '{print $1}'` +else + ES_MAJOR=1 +fi +if [ "$ES_MAJOR" = "1" ]; then PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9300 else PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9200 http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/36d8c693/tools/build.sbt ---------------------------------------------------------------------- diff --git a/tools/build.sbt b/tools/build.sbt index b61a20a..483a591 100644 --- a/tools/build.sbt +++ b/tools/build.sbt @@ -21,22 +21,10 @@ import sbtassembly.AssemblyPlugin.autoImport._ name := "apache-predictionio-tools" libraryDependencies ++= Seq( - "com.github.scopt" %% "scopt" % "3.5.0", - "io.spray" %% "spray-can" % "1.3.3", - "io.spray" %% "spray-routing" % "1.3.3", "me.lessis" % "semverfi_2.10" % "0.1.3", - "org.apache.hadoop" % "hadoop-common" % hadoopVersion.value, - "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion.value, - "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided", "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided", - "org.clapper" %% "grizzled-slf4j" % "1.0.2", - "org.json4s" %% "json4s-native" % json4sVersion.value, - "org.json4s" %% "json4s-ext" % json4sVersion.value, - "org.scalaj" %% "scalaj-http" % "1.1.6", - "com.typesafe.akka" %% "akka-actor" % akkaVersion.value, "com.typesafe.akka" %% "akka-slf4j" % akkaVersion.value, "io.spray" %% "spray-testkit" % "1.3.3" % "test", - "org.postgresql" % "postgresql" % "9.4-1204-jdbc41" % "test", "org.specs2" %% "specs2" % "2.3.13" % "test") dependencyOverrides += "org.slf4j" % "slf4j-log4j12" % "1.7.18" @@ -51,12 +39,8 @@ assemblyMergeStrategy in assembly := { excludedJars in assembly <<= (fullClasspath in assembly) map { cp => cp filter { _.data.getName match { - case "asm-3.1.jar" => true - case "commons-beanutils-1.7.0.jar" => true case "reflectasm-1.10.1.jar" => true - case "commons-beanutils-core-1.8.0.jar" => true case "kryo-3.0.3.jar" => true - case "slf4j-log4j12-1.7.5.jar" => true case _ => false }} }
